0
0
mirror of https://github.com/go-gitea/gitea.git synced 2026-05-12 02:33:50 +02:00

feat:Render .ipynb files natively

This commit is contained in:
karthik.bhandary 2026-04-26 12:59:48 +00:00
parent ea82743dc7
commit d83c602da2
7 changed files with 649 additions and 0 deletions

View File

@ -48,6 +48,11 @@ func RegisterRenderers() {
},
})
markup.RegisterRenderer(&frontendRenderer{
name: "jupyter-notebook",
patterns: []string{"*.ipynb"},
})
for _, renderer := range setting.ExternalMarkupRenderers {
markup.RegisterRenderer(&Renderer{renderer})
}

View File

@ -4,12 +4,16 @@
package external
import (
"bytes"
"encoding/base64"
"io"
"strings"
"unicode/utf8"
"code.gitea.io/gitea/modules/htmlutil"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/public"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -58,6 +62,46 @@ func (p *frontendRenderer) GetExternalRendererOptions() (ret markup.ExternalRend
return ret
}
type notebookCell struct {
CellType string `json:"cell_type"`
Source []string `json:"source"`
Outputs []map[string]any `json:"outputs,omitempty"`
}
type notebookData struct {
Cells []notebookCell `json:"cells"`
Metadata map[string]any `json:"metadata,omitempty"`
Nbformat int `json:"nbformat,omitempty"`
NbformatMinor int `json:"nbformat_minor,omitempty"`
}
func preprocessJupyterNotebook(ctx *markup.RenderContext, input io.Reader) ([]byte, error) {
content, err := io.ReadAll(input)
if err != nil {
return nil, err
}
var nb notebookData
if err := json.Unmarshal(content, &nb); err != nil {
return content, nil
}
for i := range nb.Cells {
if nb.Cells[i].CellType == "markdown" {
var sourceBuilder strings.Builder
for _, line := range nb.Cells[i].Source {
sourceBuilder.WriteString(line)
}
var buf bytes.Buffer
if err := markdown.RenderRaw(ctx, bytes.NewReader([]byte(sourceBuilder.String())), &buf); err == nil {
nb.Cells[i].Source = []string{buf.String()}
}
}
}
return json.Marshal(nb)
}
func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
if ctx.RenderOptions.StandalonePageOptions == nil {
opts := p.GetExternalRendererOptions()
@ -69,6 +113,13 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou
return err
}
if p.name == "jupyter-notebook" {
preprocessed, err := preprocessJupyterNotebook(ctx, bytes.NewReader(content))
if err == nil {
content = preprocessed
}
}
contentEncoding, contentString := "text", util.UnsafeBytesToString(content)
if !utf8.Valid(content) {
contentEncoding = "base64"
@ -81,6 +132,7 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou
<head>
<!-- external-render-helper will be injected here by the markup render -->
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="%s">
</head>
<body>
<div id="frontend-render-viewer" data-frontend-renders="%s" data-file-tree-path="%s"></div>
@ -88,6 +140,7 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou
<script nonce type="module" src="%s"></script>
</body>
</html>`,
public.AssetURI("css/index.css"),
p.name, ctx.RenderOptions.RelativePath,
contentEncoding, contentString,
public.AssetURI("js/external-render-frontend.js"))

View File

@ -0,0 +1,223 @@
import {env} from 'node:process';
import {expect, test} from '@playwright/test';
import type {APIRequestContext} from '@playwright/test';
import {login, apiCreateRepo, apiDeleteRepo, assertFlushWithParent, assertNoJsError, randomString, baseUrl, apiHeaders} from './utils.ts';
// Helper to create multiple files in a single commit
async function apiCreateFiles(request: APIRequestContext, owner: string, repo: string, files: Record<string, string>) {
const branch = 'main';
// Get the latest commit SHA
const branchInfo = await request.get(`${baseUrl()}/api/v1/repos/${owner}/${repo}/branches/${branch}`);
const branchData = await branchInfo.json();
const latestCommitSha = branchData.commit.id;
// Create file changes
const fileChanges = Object.entries(files).map(([path, content]) => ({
operation: 'create',
path,
content: globalThis.btoa(content),
}));
// Create all files in one commit
await request.post(`${baseUrl()}/api/v1/repos/${owner}/${repo}/contents`, {
headers: apiHeaders(),
data: {
branch,
files: fileChanges,
message: 'Add test notebooks',
sha: latestCommitSha,
},
});
}
test('jupyter notebook - all scenarios', async ({page, request}) => {
test.setTimeout(25000);
const repoName = `e2e-jupyter-all-${randomString(8)}`;
const owner = env.GITEA_TEST_E2E_USER;
await Promise.all([
apiCreateRepo(request, {name: repoName}),
login(page),
]);
try {
// Define all notebooks
const mainNotebook = JSON.stringify({
cells: [
{
cell_type: 'markdown',
source: ['# Test Notebook\n', 'This is **markdown** with `code`.'],
},
{
cell_type: 'code',
execution_count: 1,
source: ['print("Hello World")'],
outputs: [{output_type: 'stream', name: 'stdout', text: ['Hello World\n']}],
},
{
cell_type: 'code',
execution_count: 2,
source: ['import matplotlib.pyplot as plt'],
outputs: [{
output_type: 'execute_result',
data: {
'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
},
}],
},
{
cell_type: 'code',
execution_count: 3,
source: ['x = 5'],
outputs: [{
output_type: 'execute_result',
data: {
'text/latex': ['$$x^2 + y^2 = z^2$$'],
'text/plain': ['x^2 + y^2 = z^2'],
},
}],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
const noOutputNotebook = JSON.stringify({
cells: [
{
cell_type: 'code',
source: ['# Code with no output'],
outputs: [],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
const errorNotebook = JSON.stringify({
cells: [
{
cell_type: 'code',
source: ['raise ValueError("Test error")'],
outputs: [{
output_type: 'error',
ename: 'ValueError',
evalue: 'Test error',
traceback: ['ValueError: Test error'],
}],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
const mixedNotebook = JSON.stringify({
cells: [
{
cell_type: 'code',
source: ['print("text")'],
outputs: [
{output_type: 'stream', name: 'stdout', text: ['text\n']},
{
output_type: 'execute_result',
data: {
'text/html': ['<b>HTML output</b>'],
'text/plain': ['HTML output'],
},
},
{
output_type: 'execute_result',
data: {
'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
},
},
],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
// Create all files in a single commit (fast, no race condition)
await apiCreateFiles(request, owner, repoName, {
'test.ipynb': mainNotebook,
'no-output.ipynb': noOutputNotebook,
'error.ipynb': errorNotebook,
'mixed.ipynb': mixedNotebook,
});
// Test 1: Main notebook rendering
await page.goto(`/${owner}/${repoName}/src/branch/main/test.ipynb`);
let iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
let frame = page.frameLocator('iframe.external-render-iframe');
let viewer = frame.locator('#frontend-render-viewer');
await Promise.all([
expect(viewer.locator('.cell.markdown h1')).toContainText('Test Notebook'),
expect(viewer.locator('.cell.markdown strong')).toContainText('markdown'),
expect(viewer.locator('.cell.code .input code').first()).toContainText('print("Hello World")'),
expect(viewer.locator('.cell.code .output pre').first()).toContainText('Hello World'),
expect(viewer.locator('.cell.code .output img')).toBeVisible(),
expect(viewer.locator('.cell.code .input .prompt').first()).toContainText('In [1]:'),
expect(viewer.locator('.cell.code .output .prompt').first()).toContainText('Out[2]:'),
expect(viewer.locator('.cell.code')).toHaveCount(3),
]);
await expect.poll(async () => (await iframe.boundingBox())!.height).toBeGreaterThan(200);
await assertFlushWithParent(iframe, page.locator('.file-view'));
await assertNoJsError(page);
// Test 2: No outputs
await page.goto(`/${owner}/${repoName}/src/branch/main/no-output.ipynb`);
iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
frame = page.frameLocator('iframe.external-render-iframe');
viewer = frame.locator('#frontend-render-viewer');
await Promise.all([
expect(viewer.locator('.cell.code')).toBeVisible(),
expect(viewer.locator('.cell.code .output')).toBeHidden(),
]);
await assertNoJsError(page);
// Test 3: Error output
await page.goto(`/${owner}/${repoName}/src/branch/main/error.ipynb`);
iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
frame = page.frameLocator('iframe.external-render-iframe');
viewer = frame.locator('#frontend-render-viewer');
await expect(viewer.locator('.error-output')).toContainText('ValueError: Test error');
await assertNoJsError(page);
// Test 4: Mixed outputs
await page.goto(`/${owner}/${repoName}/src/branch/main/mixed.ipynb`);
iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
frame = page.frameLocator('iframe.external-render-iframe');
viewer = frame.locator('#frontend-render-viewer');
await Promise.all([
expect(viewer.locator('.cell.code .output pre').first()).toContainText('text'),
expect(viewer.locator('.cell.code .output b')).toContainText('HTML output'),
expect(viewer.locator('.cell.code .output img')).toBeVisible(),
]);
await assertNoJsError(page);
} finally {
await apiDeleteRepo(request, owner, repoName);
}
});

View File

@ -0,0 +1,182 @@
.jupyter-notebook {
padding: 20px;
background: var(--color-body);
color: var(--color-text);
font-family: inherit;
}
/* Cell containers */
.jupyter-notebook .cell {
margin-bottom: 20px;
overflow: hidden;
}
/* Markdown cells */
.jupyter-notebook .cell.markdown {
background: var(--color-body);
border: none;
}
.jupyter-notebook .cell.markdown .input {
padding: 6px 12px;
line-height: 1.6;
background: var(--color-body);
color: var(--color-text);
}
.jupyter-notebook .cell.markdown h1,
.jupyter-notebook .cell.markdown h2,
.jupyter-notebook .cell.markdown h3 {
margin-top: 24px;
margin-bottom: 16px;
font-weight: var(--font-weight-semibold);
line-height: 1.25;
color: var(--color-text);
}
.jupyter-notebook .cell.markdown h1 {
font-size: 1.875em;
border-bottom: 1px solid var(--color-secondary-alpha-20);
padding-bottom: 0.3em;
}
.jupyter-notebook .cell.markdown h2 {
font-size: 1.5em;
}
.jupyter-notebook .cell.markdown h3 {
font-size: 1.25em;
}
.jupyter-notebook .cell.markdown p {
margin-top: 0;
margin-bottom: 16px;
}
.jupyter-notebook .cell.markdown code {
padding: 0.2em 0.4em;
margin: 0;
font-size: 85%;
background-color: var(--color-secondary-alpha-20);
border-radius: 3px;
font-family: var(--fonts-monospace);
}
.jupyter-notebook .cell.markdown table {
border-collapse: collapse;
width: 100%;
margin: 16px 0;
font-size: 13px;
}
.jupyter-notebook .cell.markdown table th,
.jupyter-notebook .cell.markdown table td {
border: 1px solid var(--color-secondary);
padding: 6px 13px;
text-align: left;
}
.jupyter-notebook .cell.markdown table th {
background: var(--color-secondary-alpha-20);
font-weight: var(--font-weight-semibold);
}
.jupyter-notebook .cell.markdown table tr:nth-child(even) {
background: var(--color-secondary-alpha-10);
}
/* Code cells */
.jupyter-notebook .cell.code {
background: transparent;
border: none;
}
.jupyter-notebook .cell.code .input {
display: flex;
background-color: var(--color-code-bg);
border: 1px solid var(--color-secondary-alpha-20);
border-radius: 4px;
}
.jupyter-notebook .cell.code .prompt {
padding: 10px 16px;
color: var(--color-text-light-2);
font-family: var(--fonts-monospace);
font-size: 13px;
white-space: nowrap;
user-select: none;
min-width: 80px;
}
.jupyter-notebook .cell.code .input pre {
flex: 1;
margin: 0;
padding: 10px 16px 10px 0;
font-family: var(--fonts-monospace);
font-size: 13px;
line-height: 1.5;
overflow-x: auto;
color: var(--color-text);
background: transparent;
}
.jupyter-notebook .cell.code .input code {
display: block;
font-family: var(--fonts-monospace);
}
/* Code outputs */
.jupyter-notebook .cell.code .output {
background: var(--color-body);
color: var(--color-text);
}
.jupyter-notebook .cell.code .output .prompt {
padding: 10px 16px;
color: var(--color-text-light-2);
font-family: var(--fonts-monospace);
font-size: 13px;
min-width: 80px;
}
.jupyter-notebook .cell.code .output pre {
margin: 0;
padding: 10px 16px;
font-family: var(--fonts-monospace);
font-size: 13px;
line-height: 1.5;
overflow-x: auto;
color: var(--color-text);
background: var(--color-body);
white-space: pre-wrap;
overflow-wrap: break-word;
}
.jupyter-notebook .cell.code .output table {
border-collapse: collapse;
margin: 10px 16px;
font-size: 13px;
}
.jupyter-notebook .cell.code .output table th,
.jupyter-notebook .cell.code .output table td {
border: 1px solid var(--color-secondary);
padding: 6px 13px;
text-align: left;
}
.jupyter-notebook .cell.code .output table th {
background: var(--color-secondary-alpha-20);
font-weight: var(--font-weight-semibold);
}
.jupyter-notebook .cell.code .output table tr:nth-child(even) {
background: var(--color-secondary-alpha-10);
}
.jupyter-notebook .cell.code .output img {
max-width: 100%;
height: auto;
display: block;
margin: 10px 16px;
}

View File

@ -48,6 +48,7 @@
@import "./features/expander.css";
@import "./features/cropper.css";
@import "./features/console.css";
@import "./features/jupyter.css";
@import "./features/captcha.css";
@import "./markup/content.css";

View File

@ -8,6 +8,7 @@ type LazyLoadFunc = () => Promise<{frontendRender: FrontendRenderFunc}>;
const frontendPlugins: Record<string, LazyLoadFunc> = {
'viewer-3d': () => import('./render/plugins/frontend-viewer-3d.ts'),
'openapi-swagger': () => import('./render/plugins/frontend-openapi-swagger.ts'),
'jupyter-notebook': () => import('./render/plugins/frontend-jupyter-notebook.ts'),
};
class Options implements FrontendRenderOptions {

View File

@ -0,0 +1,184 @@
import type {FrontendRenderFunc} from '../plugin.ts';
export const frontendRender: FrontendRenderFunc = async (opts) => {
try {
const notebook = JSON.parse(opts.contentString());
if (!notebook.cells || !Array.isArray(notebook.cells)) {
throw new Error('Invalid notebook format: missing or invalid cells array');
}
const container = document.createElement('div');
container.className = 'jupyter-notebook';
let executionCount = 1;
for (const cell of notebook.cells) {
if (!cell.cell_type) continue;
const cellDiv = document.createElement('div');
cellDiv.className = `cell ${cell.cell_type}`;
if (cell.cell_type === 'markdown') {
const inputDiv = document.createElement('div');
inputDiv.className = 'input';
const source = Array.isArray(cell.source) ? cell.source.join('') : (cell.source || '');
inputDiv.innerHTML = source;
cellDiv.append(inputDiv);
} else if (cell.cell_type === 'code') {
const inputDiv = document.createElement('div');
inputDiv.className = 'input';
const prompt = document.createElement('div');
prompt.className = 'prompt';
prompt.textContent = `In [${cell.execution_count || executionCount}]:`;
inputDiv.append(prompt);
const pre = document.createElement('pre');
const code = document.createElement('code');
code.className = 'language-python';
const source = Array.isArray(cell.source) ? cell.source.join('') : (cell.source || '');
code.textContent = source;
pre.append(code);
inputDiv.append(pre);
cellDiv.append(inputDiv);
if (cell.outputs && Array.isArray(cell.outputs) && cell.outputs.length > 0) {
const outputDiv = document.createElement('div');
outputDiv.className = 'output';
const hasExecutionResult = cell.outputs.some((o: any) => o.output_type === 'execute_result');
if (hasExecutionResult) {
const outPrompt = document.createElement('div');
outPrompt.className = 'prompt';
outPrompt.textContent = `Out[${cell.execution_count || executionCount}]:`;
outputDiv.append(outPrompt);
}
for (const output of cell.outputs) {
try {
if (output.data) {
if (output.data['image/png']) {
const img = document.createElement('img');
const imgData = Array.isArray(output.data['image/png']) ?
output.data['image/png'].join('') : output.data['image/png'];
img.src = `data:image/png;base64,${imgData}`;
img.style.maxWidth = '100%';
outputDiv.append(img);
} else if (output.data['image/jpeg']) {
const img = document.createElement('img');
const imgData = Array.isArray(output.data['image/jpeg']) ?
output.data['image/jpeg'].join('') : output.data['image/jpeg'];
img.src = `data:image/jpeg;base64,${imgData}`;
img.style.maxWidth = '100%';
outputDiv.append(img);
} else if (output.data['image/svg+xml']) {
const svgDiv = document.createElement('div');
const svgData = Array.isArray(output.data['image/svg+xml']) ?
output.data['image/svg+xml'].join('') : output.data['image/svg+xml'];
svgDiv.innerHTML = svgData;
outputDiv.append(svgDiv);
} else if (output.data['text/html']) {
const htmlDiv = document.createElement('div');
const htmlData = Array.isArray(output.data['text/html']) ?
output.data['text/html'].join('') : output.data['text/html'];
htmlDiv.innerHTML = htmlData;
outputDiv.append(htmlDiv);
} else if (output.data['application/javascript']) {
const jsDiv = document.createElement('div');
jsDiv.className = 'js-output-warning';
jsDiv.textContent = '[JavaScript output - execution disabled for security]';
jsDiv.style.color = 'var(--color-text-light-2)';
jsDiv.style.fontStyle = 'italic';
outputDiv.append(jsDiv);
} else if (output.data['application/vnd.plotly.v1+json']) {
const plotlyDiv = document.createElement('div');
plotlyDiv.className = 'plotly-output-warning';
plotlyDiv.textContent = '[Plotly output - interactive plots not supported]';
plotlyDiv.style.color = 'var(--color-text-light-2)';
plotlyDiv.style.fontStyle = 'italic';
outputDiv.append(plotlyDiv);
} else if (output.data['application/vnd.jupyter.widget-view+json']) {
const widgetDiv = document.createElement('div');
widgetDiv.className = 'widget-output-warning';
widgetDiv.textContent = '[Jupyter widget - interactive widgets not supported]';
widgetDiv.style.color = 'var(--color-text-light-2)';
widgetDiv.style.fontStyle = 'italic';
outputDiv.append(widgetDiv);
} else if (output.data['text/latex']) {
const latex = Array.isArray(output.data['text/latex']) ?
output.data['text/latex'].join('') : output.data['text/latex'];
const pre = document.createElement('pre');
const mathCode = document.createElement('code');
mathCode.className = 'language-math display';
mathCode.textContent = latex.replace(/^\$\$|\$\$$/g, '');
pre.append(mathCode);
outputDiv.append(pre);
} else if (output.data['text/plain']) {
const textPre = document.createElement('pre');
const plainText = Array.isArray(output.data['text/plain']) ?
output.data['text/plain'].join('') : output.data['text/plain'];
textPre.textContent = plainText;
outputDiv.append(textPre);
}
} else if (output.text) {
const textPre = document.createElement('pre');
const text = Array.isArray(output.text) ? output.text.join('') : output.text;
textPre.textContent = text;
outputDiv.append(textPre);
} else if (output.output_type === 'stream' && output.name) {
const streamPre = document.createElement('pre');
streamPre.className = `stream-${output.name}`;
const streamText = Array.isArray(output.text) ? output.text.join('') : (output.text || '');
streamPre.textContent = streamText;
outputDiv.append(streamPre);
} else if (output.output_type === 'error') {
const errorPre = document.createElement('pre');
errorPre.className = 'error-output';
errorPre.style.color = '#d32f2f';
const traceback = Array.isArray(output.traceback) ? output.traceback.join('\n') :
(output.ename && output.evalue ? `${output.ename}: ${output.evalue}` : 'Error');
errorPre.textContent = traceback;
outputDiv.append(errorPre);
}
} catch (outputError) {
console.warn('Failed to render output:', outputError);
const errorDiv = document.createElement('div');
errorDiv.textContent = '[Output rendering failed]';
errorDiv.style.color = 'var(--color-text-light-2)';
errorDiv.style.fontStyle = 'italic';
outputDiv.append(errorDiv);
}
}
if (outputDiv.children.length > 0) {
cellDiv.append(outputDiv);
}
}
executionCount++;
}
container.append(cellDiv);
}
opts.container.append(container);
const {initMarkupCodeMath} = await import('../../markup/math.ts');
await initMarkupCodeMath(container);
return true;
} catch (error) {
console.error('Jupyter notebook rendering failed:', error);
const errorDiv = document.createElement('div');
errorDiv.style.padding = '20px';
errorDiv.style.color = '#d32f2f';
const errorTitle = document.createElement('strong');
errorTitle.textContent = 'Failed to render notebook:';
errorDiv.append(errorTitle);
errorDiv.append(document.createElement('br'));
errorDiv.append(document.createTextNode(error.message));
opts.container.append(errorDiv);
return false;
}
};