Mirrors/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-05-11 11:25:42 +02:00
Code Issues Activity

Add Jupyter Notebook (.ipynb) rendering support

Browse Source
This commit is contained in:
karthik.bhandary 2026-04-27 06:21:34 +00:00
parent d83c602da2
commit 53cbc734a5
4 changed files with 126 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
modules/markup/external/frontend.go vendored
View File

@ -4,16 +4,12 @@
package external
import (
"bytes"
"encoding/base64"
"io"
"strings"
"unicode/utf8"
"code.gitea.io/gitea/modules/htmlutil"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/public"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -62,46 +58,6 @@ func (p *frontendRenderer) GetExternalRendererOptions() (ret markup.ExternalRend
return ret
}
type notebookCell struct {
CellType string `json:"cell_type"`
Source []string `json:"source"`
Outputs []map[string]any `json:"outputs,omitempty"`
}
type notebookData struct {
Cells []notebookCell `json:"cells"`
Metadata map[string]any `json:"metadata,omitempty"`
Nbformat int `json:"nbformat,omitempty"`
NbformatMinor int `json:"nbformat_minor,omitempty"`
}
func preprocessJupyterNotebook(ctx *markup.RenderContext, input io.Reader) ([]byte, error) {
content, err := io.ReadAll(input)
if err != nil {
return nil, err
}
var nb notebookData
if err := json.Unmarshal(content, &nb); err != nil {
return content, nil
}
for i := range nb.Cells {
if nb.Cells[i].CellType == "markdown" {
var sourceBuilder strings.Builder
for _, line := range nb.Cells[i].Source {
sourceBuilder.WriteString(line)
}
var buf bytes.Buffer
if err := markdown.RenderRaw(ctx, bytes.NewReader([]byte(sourceBuilder.String())), &buf); err == nil {
nb.Cells[i].Source = []string{buf.String()}
}
}
}
return json.Marshal(nb)
}
func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
if ctx.RenderOptions.StandalonePageOptions == nil {
opts := p.GetExternalRendererOptions()
@ -113,13 +69,6 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou
return err
}
if p.name == "jupyter-notebook" {
preprocessed, err := preprocessJupyterNotebook(ctx, bytes.NewReader(content))
if err == nil {
content = preprocessed
}
}
contentEncoding, contentString := "text", util.UnsafeBytesToString(content)
if !utf8.Valid(content) {
contentEncoding = "base64"
@ -132,7 +81,6 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou
<head>
<!-- external-render-helper will be injected here by the markup render -->
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="%s">
</head>
<body>
<div id="frontend-render-viewer" data-frontend-renders="%s" data-file-tree-path="%s"></div>
@ -140,7 +88,6 @@ func (p *frontendRenderer) Render(ctx *markup.RenderContext, input io.Reader, ou
<script nonce type="module" src="%s"></script>
</body>
</html>`,
public.AssetURI("css/index.css"),
p.name, ctx.RenderOptions.RelativePath,
contentEncoding, contentString,
public.AssetURI("js/external-render-frontend.js"))

253
tests/e2e/jupyter-render.test.ts
View File

@ -1,223 +1,54 @@
import {env} from 'node:process';
import {expect, test} from '@playwright/test';
import type {APIRequestContext} from '@playwright/test';
import {login, apiCreateRepo, apiDeleteRepo, assertFlushWithParent, assertNoJsError, randomString, baseUrl, apiHeaders} from './utils.ts';
import {login, apiCreateRepo, apiCreateFile, randomString} from './utils.ts';
// Helper to create multiple files in a single commit
async function apiCreateFiles(request: APIRequestContext, owner: string, repo: string, files: Record<string, string>) {
const branch = 'main';
test.describe('jupyter notebook rendering', () => {
let repoName: string;
let owner: string;
// Get the latest commit SHA
const branchInfo = await request.get(`${baseUrl()}/api/v1/repos/${owner}/${repo}/branches/${branch}`);
const branchData = await branchInfo.json();
const latestCommitSha = branchData.commit.id;
test.beforeAll(async ({request}) => {
repoName = `e2e-jupyter-${randomString(8)}`;
owner = env.GITEA_TEST_E2E_USER;
await apiCreateRepo(request, {name: repoName});
// Single comprehensive test notebook
const notebook = JSON.stringify({
cells: [
{cell_type: 'markdown', source: ['# Test\n', '**bold**']},
{cell_type: 'code', execution_count: 1, source: ['print("Hello")'], outputs: [{output_type: 'stream', name: 'stdout', text: ['Hello\n']}]},
{cell_type: 'code', execution_count: 2, source: ['x'], outputs: [{output_type: 'execute_result', data: {'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='}}]},
{cell_type: 'code', source: ['# No output'], outputs: []},
{cell_type: 'code', source: ['err'], outputs: [{output_type: 'error', ename: 'ValueError', evalue: 'Test', traceback: ['ValueError: Test']}]},
{cell_type: 'code', source: ['mixed'], outputs: [{output_type: 'stream', name: 'stdout', text: ['text\n']}, {output_type: 'execute_result', data: {'text/html': ['<b>HTML</b>']}}]},
],
metadata: {}, nbformat: 4, nbformat_minor: 5,
});
// Create file changes
const fileChanges = Object.entries(files).map(([path, content]) => ({
operation: 'create',
path,
content: globalThis.btoa(content),
}));
// Create all files in one commit
await request.post(`${baseUrl()}/api/v1/repos/${owner}/${repo}/contents`, {
headers: apiHeaders(),
data: {
branch,
files: fileChanges,
message: 'Add test notebooks',
sha: latestCommitSha,
},
await apiCreateFile(request, owner, repoName, 'test.ipynb', notebook);
});
}
test('jupyter notebook - all scenarios', async ({page, request}) => {
test.setTimeout(25000);
const repoName = `e2e-jupyter-all-${randomString(8)}`;
const owner = env.GITEA_TEST_E2E_USER;
await Promise.all([
apiCreateRepo(request, {name: repoName}),
login(page),
]);
try {
// Define all notebooks
const mainNotebook = JSON.stringify({
cells: [
{
cell_type: 'markdown',
source: ['# Test Notebook\n', 'This is **markdown** with `code`.'],
},
{
cell_type: 'code',
execution_count: 1,
source: ['print("Hello World")'],
outputs: [{output_type: 'stream', name: 'stdout', text: ['Hello World\n']}],
},
{
cell_type: 'code',
execution_count: 2,
source: ['import matplotlib.pyplot as plt'],
outputs: [{
output_type: 'execute_result',
data: {
'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
},
}],
},
{
cell_type: 'code',
execution_count: 3,
source: ['x = 5'],
outputs: [{
output_type: 'execute_result',
data: {
'text/latex': ['$$x^2 + y^2 = z^2$$'],
'text/plain': ['x^2 + y^2 = z^2'],
},
}],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
const noOutputNotebook = JSON.stringify({
cells: [
{
cell_type: 'code',
source: ['# Code with no output'],
outputs: [],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
const errorNotebook = JSON.stringify({
cells: [
{
cell_type: 'code',
source: ['raise ValueError("Test error")'],
outputs: [{
output_type: 'error',
ename: 'ValueError',
evalue: 'Test error',
traceback: ['ValueError: Test error'],
}],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
const mixedNotebook = JSON.stringify({
cells: [
{
cell_type: 'code',
source: ['print("text")'],
outputs: [
{output_type: 'stream', name: 'stdout', text: ['text\n']},
{
output_type: 'execute_result',
data: {
'text/html': ['<b>HTML output</b>'],
'text/plain': ['HTML output'],
},
},
{
output_type: 'execute_result',
data: {
'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
},
},
],
},
],
metadata: {},
nbformat: 4,
nbformat_minor: 5,
});
// Create all files in a single commit (fast, no race condition)
await apiCreateFiles(request, owner, repoName, {
'test.ipynb': mainNotebook,
'no-output.ipynb': noOutputNotebook,
'error.ipynb': errorNotebook,
'mixed.ipynb': mixedNotebook,
});
// Test 1: Main notebook rendering
test('renders markdown cells', async ({page}) => {
await login(page);
await page.goto(`/${owner}/${repoName}/src/branch/main/test.ipynb`);
await expect(page.frameLocator('iframe.external-render-iframe').locator('.cell.markdown strong')).toBeVisible();
});
let iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
test('renders code cells with outputs', async ({page}) => {
await login(page);
await page.goto(`/${owner}/${repoName}/src/branch/main/test.ipynb`);
await expect(page.frameLocator('iframe.external-render-iframe').locator('.cell.code .output pre').first()).toBeVisible();
});
let frame = page.frameLocator('iframe.external-render-iframe');
let viewer = frame.locator('#frontend-render-viewer');
test('renders image outputs', async ({page}) => {
await login(page);
await page.goto(`/${owner}/${repoName}/src/branch/main/test.ipynb`);
await expect(page.frameLocator('iframe.external-render-iframe').locator('.cell.code .output img')).toBeVisible();
});
await Promise.all([
expect(viewer.locator('.cell.markdown h1')).toContainText('Test Notebook'),
expect(viewer.locator('.cell.markdown strong')).toContainText('markdown'),
expect(viewer.locator('.cell.code .input code').first()).toContainText('print("Hello World")'),
expect(viewer.locator('.cell.code .output pre').first()).toContainText('Hello World'),
expect(viewer.locator('.cell.code .output img')).toBeVisible(),
expect(viewer.locator('.cell.code .input .prompt').first()).toContainText('In [1]:'),
expect(viewer.locator('.cell.code .output .prompt').first()).toContainText('Out[2]:'),
expect(viewer.locator('.cell.code')).toHaveCount(3),
]);
await expect.poll(async () => (await iframe.boundingBox())!.height).toBeGreaterThan(200);
await assertFlushWithParent(iframe, page.locator('.file-view'));
await assertNoJsError(page);
// Test 2: No outputs
await page.goto(`/${owner}/${repoName}/src/branch/main/no-output.ipynb`);
iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
frame = page.frameLocator('iframe.external-render-iframe');
viewer = frame.locator('#frontend-render-viewer');
await Promise.all([
expect(viewer.locator('.cell.code')).toBeVisible(),
expect(viewer.locator('.cell.code .output')).toBeHidden(),
]);
await assertNoJsError(page);
// Test 3: Error output
await page.goto(`/${owner}/${repoName}/src/branch/main/error.ipynb`);
iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
frame = page.frameLocator('iframe.external-render-iframe');
viewer = frame.locator('#frontend-render-viewer');
await expect(viewer.locator('.error-output')).toContainText('ValueError: Test error');
await assertNoJsError(page);
// Test 4: Mixed outputs
await page.goto(`/${owner}/${repoName}/src/branch/main/mixed.ipynb`);
iframe = page.locator('iframe.external-render-iframe');
await expect(iframe).toBeVisible();
frame = page.frameLocator('iframe.external-render-iframe');
viewer = frame.locator('#frontend-render-viewer');
await Promise.all([
expect(viewer.locator('.cell.code .output pre').first()).toContainText('text'),
expect(viewer.locator('.cell.code .output b')).toContainText('HTML output'),
expect(viewer.locator('.cell.code .output img')).toBeVisible(),
]);
await assertNoJsError(page);
} finally {
await apiDeleteRepo(request, owner, repoName);
}
test('renders error outputs', async ({page}) => {
await login(page);
await page.goto(`/${owner}/${repoName}/src/branch/main/test.ipynb`);
await expect(page.frameLocator('iframe.external-render-iframe').locator('.error-output')).toBeVisible();
});
});

48
web_src/css/features/jupyter.css
View File

@ -1,3 +1,4 @@
/* Override notebookjs default styles with Gitea theme */
.jupyter-notebook {
padding: 20px;
background: var(--color-body);
@ -22,6 +23,7 @@
line-height: 1.6;
background: var(--color-body);
color: var(--color-text);
margin-left: 90px;
}
.jupyter-notebook .cell.markdown h1,
@ -91,28 +93,35 @@
border: none;
}
.jupyter-notebook .cell.code .input {
.jupyter-notebook .cell.code .input-wrapper,
.jupyter-notebook .cell.code .output-wrapper {
display: flex;
background-color: var(--color-code-bg);
border: 1px solid var(--color-secondary-alpha-20);
border-radius: 4px;
align-items: flex-start;
}
.jupyter-notebook .cell.code .prompt {
padding: 10px 16px;
padding: 10px 10px 10px 0;
color: var(--color-text-light-2);
font-family: var(--fonts-monospace);
font-size: 13px;
white-space: nowrap;
user-select: none;
min-width: 80px;
text-align: right;
width: 80px;
flex-shrink: 0;
}
.jupyter-notebook .cell.code .input {
flex: 1;
background-color: var(--color-code-bg, #f6f8fa);
border: 1px solid var(--color-secondary-alpha-20, #d0d7de);
border-radius: 4px;
}
.jupyter-notebook .cell.code .input pre {
flex: 1;
margin: 0;
padding: 10px 16px 10px 0;
font-family: var(--fonts-monospace);
padding: 10px 16px;
font-family: var(--fonts-monospace, monospace);
font-size: 13px;
line-height: 1.5;
overflow-x: auto;
@ -122,23 +131,16 @@
.jupyter-notebook .cell.code .input code {
display: block;
font-family: var(--fonts-monospace);
font-family: inherit;
}
/* Code outputs */
.jupyter-notebook .cell.code .output {
flex: 1;
background: var(--color-body);
color: var(--color-text);
}
.jupyter-notebook .cell.code .output .prompt {
padding: 10px 16px;
color: var(--color-text-light-2);
font-family: var(--fonts-monospace);
font-size: 13px;
min-width: 80px;
}
.jupyter-notebook .cell.code .output pre {
margin: 0;
padding: 10px 16px;
@ -156,6 +158,7 @@
border-collapse: collapse;
margin: 10px 16px;
font-size: 13px;
max-width: 100%;
}
.jupyter-notebook .cell.code .output table th,
@ -175,8 +178,13 @@
}
.jupyter-notebook .cell.code .output img {
max-width: 100%;
max-width: 90%;
height: auto;
display: block;
margin: 10px 16px;
margin: 10px 0;
}
.jupyter-notebook .cell.code .output table img {
margin: 0;
width: auto;
}

76
web_src/js/render/plugins/frontend-jupyter-notebook.ts
View File

@ -1,4 +1,22 @@
import type {FrontendRenderFunc} from '../plugin.ts';
import '../../../css/features/jupyter.css';
// Simple markdown to HTML converter for notebook cells
function renderMarkdown(markdown: string): string {
return markdown
// Headers
.replace(/^### (.*$)/gim, '<h3>$1</h3>')
.replace(/^## (.*$)/gim, '<h2>$1</h2>')
.replace(/^# (.*$)/gim, '<h1>$1</h1>')
// Bold
.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
// Italic
.replace(/\*(.+?)\*/g, '<em>$1</em>')
// Inline code
.replace(/`(.+?)`/g, '<code>$1</code>')
// Line breaks
.replace(/\n/g, '<br>');
}
export const frontendRender: FrontendRenderFunc = async (opts) => {
try {
@ -21,18 +39,21 @@ export const frontendRender: FrontendRenderFunc = async (opts) => {
if (cell.cell_type === 'markdown') {
const inputDiv = document.createElement('div');
inputDiv.className = 'input';
inputDiv.className = 'input markup';
const source = Array.isArray(cell.source) ? cell.source.join('') : (cell.source || '');
inputDiv.innerHTML = source;
inputDiv.innerHTML = renderMarkdown(source);
cellDiv.append(inputDiv);
} else if (cell.cell_type === 'code') {
const inputDiv = document.createElement('div');
inputDiv.className = 'input';
const inputWrapper = document.createElement('div');
inputWrapper.className = 'input-wrapper';
const prompt = document.createElement('div');
prompt.className = 'prompt';
prompt.className = 'prompt input-prompt';
prompt.textContent = `In [${cell.execution_count || executionCount}]:`;
inputDiv.append(prompt);
inputWrapper.append(prompt);
const inputDiv = document.createElement('div');
inputDiv.className = 'input';
const pre = document.createElement('pre');
const code = document.createElement('code');
@ -41,33 +62,38 @@ export const frontendRender: FrontendRenderFunc = async (opts) => {
code.textContent = source;
pre.append(code);
inputDiv.append(pre);
cellDiv.append(inputDiv);
inputWrapper.append(inputDiv);
cellDiv.append(inputWrapper);
if (cell.outputs && Array.isArray(cell.outputs) && cell.outputs.length > 0) {
const outputDiv = document.createElement('div');
outputDiv.className = 'output';
const outputWrapper = document.createElement('div');
outputWrapper.className = 'output-wrapper';
const hasExecutionResult = cell.outputs.some((o: any) => o.output_type === 'execute_result');
const outPrompt = document.createElement('div');
outPrompt.className = 'prompt output-prompt';
if (hasExecutionResult) {
const outPrompt = document.createElement('div');
outPrompt.className = 'prompt';
outPrompt.textContent = `Out[${cell.execution_count || executionCount}]:`;
outputDiv.append(outPrompt);
}
outputWrapper.append(outPrompt);
const outputDiv = document.createElement('div');
outputDiv.className = 'output';
for (const output of cell.outputs) {
try {
if (output.data) {
if (output.data['image/png']) {
const img = document.createElement('img');
const imgData = Array.isArray(output.data['image/png']) ?
const imgData = Array.isArray(output.data['image/png']) ?
output.data['image/png'].join('') : output.data['image/png'];
img.src = `data:image/png;base64,${imgData}`;
img.style.maxWidth = '100%';
outputDiv.append(img);
} else if (output.data['image/jpeg']) {
const img = document.createElement('img');
const imgData = Array.isArray(output.data['image/jpeg']) ?
const imgData = Array.isArray(output.data['image/jpeg']) ?
output.data['image/jpeg'].join('') : output.data['image/jpeg'];
img.src = `data:image/jpeg;base64,${imgData}`;
img.style.maxWidth = '100%';
@ -79,11 +105,20 @@ export const frontendRender: FrontendRenderFunc = async (opts) => {
svgDiv.innerHTML = svgData;
outputDiv.append(svgDiv);
} else if (output.data['text/html']) {
const wrapperDiv = document.createElement('div');
wrapperDiv.style.overflowX = 'auto';
wrapperDiv.style.maxWidth = '100%';
const htmlDiv = document.createElement('div');
const htmlData = Array.isArray(output.data['text/html']) ?
output.data['text/html'].join('') : output.data['text/html'];
htmlDiv.innerHTML = htmlData;
outputDiv.append(htmlDiv);
// Ensure images inside HTML outputs are constrained
htmlDiv.querySelectorAll('img').forEach((img) => {
img.style.maxWidth = '100%';
img.style.height = 'auto';
});
wrapperDiv.append(htmlDiv);
outputDiv.append(wrapperDiv);
} else if (output.data['application/javascript']) {
const jsDiv = document.createElement('div');
jsDiv.className = 'js-output-warning';
@ -135,8 +170,8 @@ export const frontendRender: FrontendRenderFunc = async (opts) => {
} else if (output.output_type === 'error') {
const errorPre = document.createElement('pre');
errorPre.className = 'error-output';
errorPre.style.color = '#d32f2f';
const traceback = Array.isArray(output.traceback) ? output.traceback.join('\n') :
errorPre.style.color = 'var(--color-red)';
const traceback = Array.isArray(output.traceback) ? output.traceback.join('\n') :
(output.ename && output.evalue ? `${output.ename}: ${output.evalue}` : 'Error');
errorPre.textContent = traceback;
outputDiv.append(errorPre);
@ -152,7 +187,8 @@ export const frontendRender: FrontendRenderFunc = async (opts) => {
}
if (outputDiv.children.length > 0) {
cellDiv.append(outputDiv);
outputWrapper.append(outputDiv);
cellDiv.append(outputWrapper);
}
}
@ -166,13 +202,13 @@ export const frontendRender: FrontendRenderFunc = async (opts) => {
const {initMarkupCodeMath} = await import('../../markup/math.ts');
await initMarkupCodeMath(container);
return true;
} catch (error) {
console.error('Jupyter notebook rendering failed:', error);
const errorDiv = document.createElement('div');
errorDiv.style.padding = '20px';
errorDiv.style.color = '#d32f2f';
errorDiv.style.color = 'var(--color-red)';
const errorTitle = document.createElement('strong');
errorTitle.textContent = 'Failed to render notebook:';
errorDiv.append(errorTitle);