diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 4b3c96fd33..1926c46f4a 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -48,6 +48,11 @@ func RegisterRenderers() { }, }) + markup.RegisterRenderer(&frontendRenderer{ + name: "jupyter-notebook", + patterns: []string{"*.ipynb"}, + }) + for _, renderer := range setting.ExternalMarkupRenderers { markup.RegisterRenderer(&Renderer{renderer}) } diff --git a/modules/markup/external/frontend.go b/modules/markup/external/frontend.go index 7327503d28..a68d265bd4 100644 --- a/modules/markup/external/frontend.go +++ b/modules/markup/external/frontend.go @@ -4,12 +4,16 @@ package external import ( + "bytes" "encoding/base64" "io" + "strings" "unicode/utf8" "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/public" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -58,6 +62,46 @@ func (p *frontendRenderer) GetExternalRendererOptions() (ret markup.ExternalRend return ret } +type notebookCell struct { + CellType string `json:"cell_type"` + Source []string `json:"source"` + Outputs []map[string]any `json:"outputs,omitempty"` +} + +type notebookData struct { + Cells []notebookCell `json:"cells"` + Metadata map[string]any `json:"metadata,omitempty"` + Nbformat int `json:"nbformat,omitempty"` + NbformatMinor int `json:"nbformat_minor,omitempty"` +} + +func preprocessJupyterNotebook(ctx *markup.RenderContext, input io.Reader) ([]byte, error) { + content, err := io.ReadAll(input) + if err != nil { + return nil, err + } + + var nb notebookData + if err := json.Unmarshal(content, &nb); err != nil { + return content, nil + } + + for i := range nb.Cells { + if nb.Cells[i].CellType == "markdown" { + var sourceBuilder strings.Builder + for _, line := range nb.Cells[i].Source { + sourceBuilder.WriteString(line) + } + var buf bytes.Buffer + if err := markdown.RenderRaw(ctx, bytes.NewReader([]byte(sourceBuilder.String())), &buf); err == nil { + nb.Cells[i].Source = []string{buf.String()} + } + } + } + + return json.Marshal(nb) +} + func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { if ctx.RenderOptions.StandalonePageOptions == nil { opts := p.GetExternalRendererOptions() @@ -69,6 +113,13 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou return err } + if p.name == "jupyter-notebook" { + preprocessed, err := preprocessJupyterNotebook(ctx, bytes.NewReader(content)) + if err == nil { + content = preprocessed + } + } + contentEncoding, contentString := "text", util.UnsafeBytesToString(content) if !utf8.Valid(content) { contentEncoding = "base64" @@ -81,6 +132,7 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou +
@@ -88,6 +140,7 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou `, + public.AssetURI("css/index.css"), p.name, ctx.RenderOptions.RelativePath, contentEncoding, contentString, public.AssetURI("js/external-render-frontend.js")) diff --git a/tests/e2e/jupyter-render.test.ts b/tests/e2e/jupyter-render.test.ts new file mode 100644 index 0000000000..6f5dc14c14 --- /dev/null +++ b/tests/e2e/jupyter-render.test.ts @@ -0,0 +1,223 @@ +import {env} from 'node:process'; +import {expect, test} from '@playwright/test'; +import type {APIRequestContext} from '@playwright/test'; +import {login, apiCreateRepo, apiDeleteRepo, assertFlushWithParent, assertNoJsError, randomString, baseUrl, apiHeaders} from './utils.ts'; + +// Helper to create multiple files in a single commit +async function apiCreateFiles(request: APIRequestContext, owner: string, repo: string, files: Record) { + const branch = 'main'; + + // Get the latest commit SHA + const branchInfo = await request.get(`${baseUrl()}/api/v1/repos/${owner}/${repo}/branches/${branch}`); + const branchData = await branchInfo.json(); + const latestCommitSha = branchData.commit.id; + + // Create file changes + const fileChanges = Object.entries(files).map(([path, content]) => ({ + operation: 'create', + path, + content: globalThis.btoa(content), + })); + + // Create all files in one commit + await request.post(`${baseUrl()}/api/v1/repos/${owner}/${repo}/contents`, { + headers: apiHeaders(), + data: { + branch, + files: fileChanges, + message: 'Add test notebooks', + sha: latestCommitSha, + }, + }); +} + +test('jupyter notebook - all scenarios', async ({page, request}) => { + test.setTimeout(25000); + const repoName = `e2e-jupyter-all-${randomString(8)}`; + const owner = env.GITEA_TEST_E2E_USER; + + await Promise.all([ + apiCreateRepo(request, {name: repoName}), + login(page), + ]); + + try { + // Define all notebooks + const mainNotebook = JSON.stringify({ + cells: [ + { + cell_type: 'markdown', + source: ['# Test Notebook\n', 'This is **markdown** with `code`.'], + }, + { + cell_type: 'code', + execution_count: 1, + source: ['print("Hello World")'], + outputs: [{output_type: 'stream', name: 'stdout', text: ['Hello World\n']}], + }, + { + cell_type: 'code', + execution_count: 2, + source: ['import matplotlib.pyplot as plt'], + outputs: [{ + output_type: 'execute_result', + data: { + 'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', + }, + }], + }, + { + cell_type: 'code', + execution_count: 3, + source: ['x = 5'], + outputs: [{ + output_type: 'execute_result', + data: { + 'text/latex': ['$$x^2 + y^2 = z^2$$'], + 'text/plain': ['x^2 + y^2 = z^2'], + }, + }], + }, + ], + metadata: {}, + nbformat: 4, + nbformat_minor: 5, + }); + + const noOutputNotebook = JSON.stringify({ + cells: [ + { + cell_type: 'code', + source: ['# Code with no output'], + outputs: [], + }, + ], + metadata: {}, + nbformat: 4, + nbformat_minor: 5, + }); + + const errorNotebook = JSON.stringify({ + cells: [ + { + cell_type: 'code', + source: ['raise ValueError("Test error")'], + outputs: [{ + output_type: 'error', + ename: 'ValueError', + evalue: 'Test error', + traceback: ['ValueError: Test error'], + }], + }, + ], + metadata: {}, + nbformat: 4, + nbformat_minor: 5, + }); + + const mixedNotebook = JSON.stringify({ + cells: [ + { + cell_type: 'code', + source: ['print("text")'], + outputs: [ + {output_type: 'stream', name: 'stdout', text: ['text\n']}, + { + output_type: 'execute_result', + data: { + 'text/html': ['HTML output'], + 'text/plain': ['HTML output'], + }, + }, + { + output_type: 'execute_result', + data: { + 'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', + }, + }, + ], + }, + ], + metadata: {}, + nbformat: 4, + nbformat_minor: 5, + }); + + // Create all files in a single commit (fast, no race condition) + await apiCreateFiles(request, owner, repoName, { + 'test.ipynb': mainNotebook, + 'no-output.ipynb': noOutputNotebook, + 'error.ipynb': errorNotebook, + 'mixed.ipynb': mixedNotebook, + }); + + // Test 1: Main notebook rendering + await page.goto(`/${owner}/${repoName}/src/branch/main/test.ipynb`); + + let iframe = page.locator('iframe.external-render-iframe'); + await expect(iframe).toBeVisible(); + + let frame = page.frameLocator('iframe.external-render-iframe'); + let viewer = frame.locator('#frontend-render-viewer'); + + await Promise.all([ + expect(viewer.locator('.cell.markdown h1')).toContainText('Test Notebook'), + expect(viewer.locator('.cell.markdown strong')).toContainText('markdown'), + expect(viewer.locator('.cell.code .input code').first()).toContainText('print("Hello World")'), + expect(viewer.locator('.cell.code .output pre').first()).toContainText('Hello World'), + expect(viewer.locator('.cell.code .output img')).toBeVisible(), + expect(viewer.locator('.cell.code .input .prompt').first()).toContainText('In [1]:'), + expect(viewer.locator('.cell.code .output .prompt').first()).toContainText('Out[2]:'), + expect(viewer.locator('.cell.code')).toHaveCount(3), + ]); + + await expect.poll(async () => (await iframe.boundingBox())!.height).toBeGreaterThan(200); + await assertFlushWithParent(iframe, page.locator('.file-view')); + await assertNoJsError(page); + + // Test 2: No outputs + await page.goto(`/${owner}/${repoName}/src/branch/main/no-output.ipynb`); + + iframe = page.locator('iframe.external-render-iframe'); + await expect(iframe).toBeVisible(); + + frame = page.frameLocator('iframe.external-render-iframe'); + viewer = frame.locator('#frontend-render-viewer'); + + await Promise.all([ + expect(viewer.locator('.cell.code')).toBeVisible(), + expect(viewer.locator('.cell.code .output')).toBeHidden(), + ]); + await assertNoJsError(page); + + // Test 3: Error output + await page.goto(`/${owner}/${repoName}/src/branch/main/error.ipynb`); + + iframe = page.locator('iframe.external-render-iframe'); + await expect(iframe).toBeVisible(); + + frame = page.frameLocator('iframe.external-render-iframe'); + viewer = frame.locator('#frontend-render-viewer'); + + await expect(viewer.locator('.error-output')).toContainText('ValueError: Test error'); + await assertNoJsError(page); + + // Test 4: Mixed outputs + await page.goto(`/${owner}/${repoName}/src/branch/main/mixed.ipynb`); + + iframe = page.locator('iframe.external-render-iframe'); + await expect(iframe).toBeVisible(); + + frame = page.frameLocator('iframe.external-render-iframe'); + viewer = frame.locator('#frontend-render-viewer'); + + await Promise.all([ + expect(viewer.locator('.cell.code .output pre').first()).toContainText('text'), + expect(viewer.locator('.cell.code .output b')).toContainText('HTML output'), + expect(viewer.locator('.cell.code .output img')).toBeVisible(), + ]); + await assertNoJsError(page); + } finally { + await apiDeleteRepo(request, owner, repoName); + } +}); diff --git a/web_src/css/features/jupyter.css b/web_src/css/features/jupyter.css new file mode 100644 index 0000000000..f407acd1d6 --- /dev/null +++ b/web_src/css/features/jupyter.css @@ -0,0 +1,182 @@ +.jupyter-notebook { + padding: 20px; + background: var(--color-body); + color: var(--color-text); + font-family: inherit; +} + +/* Cell containers */ +.jupyter-notebook .cell { + margin-bottom: 20px; + overflow: hidden; +} + +/* Markdown cells */ +.jupyter-notebook .cell.markdown { + background: var(--color-body); + border: none; +} + +.jupyter-notebook .cell.markdown .input { + padding: 6px 12px; + line-height: 1.6; + background: var(--color-body); + color: var(--color-text); +} + +.jupyter-notebook .cell.markdown h1, +.jupyter-notebook .cell.markdown h2, +.jupyter-notebook .cell.markdown h3 { + margin-top: 24px; + margin-bottom: 16px; + font-weight: var(--font-weight-semibold); + line-height: 1.25; + color: var(--color-text); +} + +.jupyter-notebook .cell.markdown h1 { + font-size: 1.875em; + border-bottom: 1px solid var(--color-secondary-alpha-20); + padding-bottom: 0.3em; +} + +.jupyter-notebook .cell.markdown h2 { + font-size: 1.5em; +} + +.jupyter-notebook .cell.markdown h3 { + font-size: 1.25em; +} + +.jupyter-notebook .cell.markdown p { + margin-top: 0; + margin-bottom: 16px; +} + +.jupyter-notebook .cell.markdown code { + padding: 0.2em 0.4em; + margin: 0; + font-size: 85%; + background-color: var(--color-secondary-alpha-20); + border-radius: 3px; + font-family: var(--fonts-monospace); +} + +.jupyter-notebook .cell.markdown table { + border-collapse: collapse; + width: 100%; + margin: 16px 0; + font-size: 13px; +} + +.jupyter-notebook .cell.markdown table th, +.jupyter-notebook .cell.markdown table td { + border: 1px solid var(--color-secondary); + padding: 6px 13px; + text-align: left; +} + +.jupyter-notebook .cell.markdown table th { + background: var(--color-secondary-alpha-20); + font-weight: var(--font-weight-semibold); +} + +.jupyter-notebook .cell.markdown table tr:nth-child(even) { + background: var(--color-secondary-alpha-10); +} + +/* Code cells */ +.jupyter-notebook .cell.code { + background: transparent; + border: none; +} + +.jupyter-notebook .cell.code .input { + display: flex; + background-color: var(--color-code-bg); + border: 1px solid var(--color-secondary-alpha-20); + border-radius: 4px; +} + +.jupyter-notebook .cell.code .prompt { + padding: 10px 16px; + color: var(--color-text-light-2); + font-family: var(--fonts-monospace); + font-size: 13px; + white-space: nowrap; + user-select: none; + min-width: 80px; +} + +.jupyter-notebook .cell.code .input pre { + flex: 1; + margin: 0; + padding: 10px 16px 10px 0; + font-family: var(--fonts-monospace); + font-size: 13px; + line-height: 1.5; + overflow-x: auto; + color: var(--color-text); + background: transparent; +} + +.jupyter-notebook .cell.code .input code { + display: block; + font-family: var(--fonts-monospace); +} + +/* Code outputs */ +.jupyter-notebook .cell.code .output { + background: var(--color-body); + color: var(--color-text); +} + +.jupyter-notebook .cell.code .output .prompt { + padding: 10px 16px; + color: var(--color-text-light-2); + font-family: var(--fonts-monospace); + font-size: 13px; + min-width: 80px; +} + +.jupyter-notebook .cell.code .output pre { + margin: 0; + padding: 10px 16px; + font-family: var(--fonts-monospace); + font-size: 13px; + line-height: 1.5; + overflow-x: auto; + color: var(--color-text); + background: var(--color-body); + white-space: pre-wrap; + overflow-wrap: break-word; +} + +.jupyter-notebook .cell.code .output table { + border-collapse: collapse; + margin: 10px 16px; + font-size: 13px; +} + +.jupyter-notebook .cell.code .output table th, +.jupyter-notebook .cell.code .output table td { + border: 1px solid var(--color-secondary); + padding: 6px 13px; + text-align: left; +} + +.jupyter-notebook .cell.code .output table th { + background: var(--color-secondary-alpha-20); + font-weight: var(--font-weight-semibold); +} + +.jupyter-notebook .cell.code .output table tr:nth-child(even) { + background: var(--color-secondary-alpha-10); +} + +.jupyter-notebook .cell.code .output img { + max-width: 100%; + height: auto; + display: block; + margin: 10px 16px; +} diff --git a/web_src/css/index.css b/web_src/css/index.css index c23e3e1c19..0d19cd8fc9 100644 --- a/web_src/css/index.css +++ b/web_src/css/index.css @@ -48,6 +48,7 @@ @import "./features/expander.css"; @import "./features/cropper.css"; @import "./features/console.css"; +@import "./features/jupyter.css"; @import "./features/captcha.css"; @import "./markup/content.css"; diff --git a/web_src/js/external-render-frontend.ts b/web_src/js/external-render-frontend.ts index 9d969bcf90..bd2c548741 100644 --- a/web_src/js/external-render-frontend.ts +++ b/web_src/js/external-render-frontend.ts @@ -8,6 +8,7 @@ type LazyLoadFunc = () => Promise<{frontendRender: FrontendRenderFunc}>; const frontendPlugins: Record = { 'viewer-3d': () => import('./render/plugins/frontend-viewer-3d.ts'), 'openapi-swagger': () => import('./render/plugins/frontend-openapi-swagger.ts'), + 'jupyter-notebook': () => import('./render/plugins/frontend-jupyter-notebook.ts'), }; class Options implements FrontendRenderOptions { diff --git a/web_src/js/render/plugins/frontend-jupyter-notebook.ts b/web_src/js/render/plugins/frontend-jupyter-notebook.ts new file mode 100644 index 0000000000..562efdcec8 --- /dev/null +++ b/web_src/js/render/plugins/frontend-jupyter-notebook.ts @@ -0,0 +1,184 @@ +import type {FrontendRenderFunc} from '../plugin.ts'; + +export const frontendRender: FrontendRenderFunc = async (opts) => { + try { + const notebook = JSON.parse(opts.contentString()); + + if (!notebook.cells || !Array.isArray(notebook.cells)) { + throw new Error('Invalid notebook format: missing or invalid cells array'); + } + + const container = document.createElement('div'); + container.className = 'jupyter-notebook'; + + let executionCount = 1; + + for (const cell of notebook.cells) { + if (!cell.cell_type) continue; + + const cellDiv = document.createElement('div'); + cellDiv.className = `cell ${cell.cell_type}`; + + if (cell.cell_type === 'markdown') { + const inputDiv = document.createElement('div'); + inputDiv.className = 'input'; + const source = Array.isArray(cell.source) ? cell.source.join('') : (cell.source || ''); + inputDiv.innerHTML = source; + cellDiv.append(inputDiv); + } else if (cell.cell_type === 'code') { + const inputDiv = document.createElement('div'); + inputDiv.className = 'input'; + + const prompt = document.createElement('div'); + prompt.className = 'prompt'; + prompt.textContent = `In [${cell.execution_count || executionCount}]:`; + inputDiv.append(prompt); + + const pre = document.createElement('pre'); + const code = document.createElement('code'); + code.className = 'language-python'; + const source = Array.isArray(cell.source) ? cell.source.join('') : (cell.source || ''); + code.textContent = source; + pre.append(code); + inputDiv.append(pre); + cellDiv.append(inputDiv); + + if (cell.outputs && Array.isArray(cell.outputs) && cell.outputs.length > 0) { + const outputDiv = document.createElement('div'); + outputDiv.className = 'output'; + + const hasExecutionResult = cell.outputs.some((o: any) => o.output_type === 'execute_result'); + if (hasExecutionResult) { + const outPrompt = document.createElement('div'); + outPrompt.className = 'prompt'; + outPrompt.textContent = `Out[${cell.execution_count || executionCount}]:`; + outputDiv.append(outPrompt); + } + + for (const output of cell.outputs) { + try { + if (output.data) { + if (output.data['image/png']) { + const img = document.createElement('img'); + const imgData = Array.isArray(output.data['image/png']) ? + output.data['image/png'].join('') : output.data['image/png']; + img.src = `data:image/png;base64,${imgData}`; + img.style.maxWidth = '100%'; + outputDiv.append(img); + } else if (output.data['image/jpeg']) { + const img = document.createElement('img'); + const imgData = Array.isArray(output.data['image/jpeg']) ? + output.data['image/jpeg'].join('') : output.data['image/jpeg']; + img.src = `data:image/jpeg;base64,${imgData}`; + img.style.maxWidth = '100%'; + outputDiv.append(img); + } else if (output.data['image/svg+xml']) { + const svgDiv = document.createElement('div'); + const svgData = Array.isArray(output.data['image/svg+xml']) ? + output.data['image/svg+xml'].join('') : output.data['image/svg+xml']; + svgDiv.innerHTML = svgData; + outputDiv.append(svgDiv); + } else if (output.data['text/html']) { + const htmlDiv = document.createElement('div'); + const htmlData = Array.isArray(output.data['text/html']) ? + output.data['text/html'].join('') : output.data['text/html']; + htmlDiv.innerHTML = htmlData; + outputDiv.append(htmlDiv); + } else if (output.data['application/javascript']) { + const jsDiv = document.createElement('div'); + jsDiv.className = 'js-output-warning'; + jsDiv.textContent = '[JavaScript output - execution disabled for security]'; + jsDiv.style.color = 'var(--color-text-light-2)'; + jsDiv.style.fontStyle = 'italic'; + outputDiv.append(jsDiv); + } else if (output.data['application/vnd.plotly.v1+json']) { + const plotlyDiv = document.createElement('div'); + plotlyDiv.className = 'plotly-output-warning'; + plotlyDiv.textContent = '[Plotly output - interactive plots not supported]'; + plotlyDiv.style.color = 'var(--color-text-light-2)'; + plotlyDiv.style.fontStyle = 'italic'; + outputDiv.append(plotlyDiv); + } else if (output.data['application/vnd.jupyter.widget-view+json']) { + const widgetDiv = document.createElement('div'); + widgetDiv.className = 'widget-output-warning'; + widgetDiv.textContent = '[Jupyter widget - interactive widgets not supported]'; + widgetDiv.style.color = 'var(--color-text-light-2)'; + widgetDiv.style.fontStyle = 'italic'; + outputDiv.append(widgetDiv); + } else if (output.data['text/latex']) { + const latex = Array.isArray(output.data['text/latex']) ? + output.data['text/latex'].join('') : output.data['text/latex']; + const pre = document.createElement('pre'); + const mathCode = document.createElement('code'); + mathCode.className = 'language-math display'; + mathCode.textContent = latex.replace(/^\$\$|\$\$$/g, ''); + pre.append(mathCode); + outputDiv.append(pre); + } else if (output.data['text/plain']) { + const textPre = document.createElement('pre'); + const plainText = Array.isArray(output.data['text/plain']) ? + output.data['text/plain'].join('') : output.data['text/plain']; + textPre.textContent = plainText; + outputDiv.append(textPre); + } + } else if (output.text) { + const textPre = document.createElement('pre'); + const text = Array.isArray(output.text) ? output.text.join('') : output.text; + textPre.textContent = text; + outputDiv.append(textPre); + } else if (output.output_type === 'stream' && output.name) { + const streamPre = document.createElement('pre'); + streamPre.className = `stream-${output.name}`; + const streamText = Array.isArray(output.text) ? output.text.join('') : (output.text || ''); + streamPre.textContent = streamText; + outputDiv.append(streamPre); + } else if (output.output_type === 'error') { + const errorPre = document.createElement('pre'); + errorPre.className = 'error-output'; + errorPre.style.color = '#d32f2f'; + const traceback = Array.isArray(output.traceback) ? output.traceback.join('\n') : + (output.ename && output.evalue ? `${output.ename}: ${output.evalue}` : 'Error'); + errorPre.textContent = traceback; + outputDiv.append(errorPre); + } + } catch (outputError) { + console.warn('Failed to render output:', outputError); + const errorDiv = document.createElement('div'); + errorDiv.textContent = '[Output rendering failed]'; + errorDiv.style.color = 'var(--color-text-light-2)'; + errorDiv.style.fontStyle = 'italic'; + outputDiv.append(errorDiv); + } + } + + if (outputDiv.children.length > 0) { + cellDiv.append(outputDiv); + } + } + + executionCount++; + } + + container.append(cellDiv); + } + + opts.container.append(container); + + const {initMarkupCodeMath} = await import('../../markup/math.ts'); + await initMarkupCodeMath(container); + + return true; + } catch (error) { + console.error('Jupyter notebook rendering failed:', error); + const errorDiv = document.createElement('div'); + errorDiv.style.padding = '20px'; + errorDiv.style.color = '#d32f2f'; + const errorTitle = document.createElement('strong'); + errorTitle.textContent = 'Failed to render notebook:'; + errorDiv.append(errorTitle); + errorDiv.append(document.createElement('br')); + errorDiv.append(document.createTextNode(error.message)); + opts.container.append(errorDiv); + return false; + } +};