From b76807817573c8c4b83627cfb917f43d17a64503 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:44:18 +0200 Subject: [PATCH 1/6] feat(editor): broaden language detection in web code editor The CodeMirror language registry only ships a narrow set of extensions and filenames per language, so common config and DSL files (.gitconfig, Brewfile, Vagrantfile, Containerfile, Cargo.lock, *.gemspec, *.tcc, Snakefile, etc.) render as plain text in the file editor. Pull authoritative extension/filename data from github-linguist via a new `make generate-codemirror-languages` script, write a curated subset to `assets/codemirror-languages.json`, and wire it into the editor as overrides on top of `@codemirror/language-data`. A small set of manual entries fill gaps Linguist classifies under separate languages (.editorconfig, .gitconfig, .npmrc) or doesn't list at all (*.conf, Snakefile, Containerfile.*, Dockerfile.*, Makefile.am, BSDmakefile). The derived data structures are memoised at module scope so the work runs once per page session, and the JSON moves into the dynamic `importCodemirror()` chunk so it doesn't bloat the entry bundle. Co-Authored-By: Claude (Opus 4.7) --- Makefile | 4 + assets/codemirror-languages.json | 1004 ++++++++++++++++++++++++ tools/generate-codemirror-languages.ts | 151 ++++ web_src/js/modules/codeeditor/main.ts | 53 +- 4 files changed, 1207 insertions(+), 5 deletions(-) create mode 100644 assets/codemirror-languages.json create mode 100755 tools/generate-codemirror-languages.ts diff --git a/Makefile b/Makefile index 27b2c30295..7db366c57f 100644 --- a/Makefile +++ b/Makefile @@ -661,6 +661,10 @@ generate-gitignore: ## update gitignore files generate-images: | node_modules ## generate images cd tools && node generate-images.ts $(TAGS) +.PHONY: generate-codemirror-languages +generate-codemirror-languages: | node_modules ## refresh assets/codemirror-languages.json from github-linguist + node tools/generate-codemirror-languages.ts + .PHONY: generate-manpage generate-manpage: ## generate manpage @[ -f gitea ] || make backend diff --git a/assets/codemirror-languages.json b/assets/codemirror-languages.json new file mode 100644 index 0000000000..5bab971eb7 --- /dev/null +++ b/assets/codemirror-languages.json @@ -0,0 +1,1004 @@ +[ + { + "name": "C", + "extensions": [ + "c", + "cats", + "h", + "idc" + ], + "filenames": [] + }, + { + "name": "C#", + "extensions": [ + "cs", + "cake", + "csx", + "linq" + ], + "filenames": [] + }, + { + "name": "C++", + "extensions": [ + "cpp", + "c++", + "cc", + "cp", + "cppm", + "cxx", + "h", + "h++", + "hh", + "hpp", + "hxx", + "inl", + "ino", + "ipp", + "ixx", + "re", + "tcc", + "tpp", + "txx" + ], + "filenames": [] + }, + { + "name": "Clojure", + "extensions": [ + "clj", + "bb", + "boot", + "cl2", + "cljc", + "cljs", + "cljscm", + "cljx", + "hic" + ], + "filenames": [ + "riemann.config" + ] + }, + { + "name": "CMake", + "extensions": [ + "cmake" + ], + "filenames": [ + "CMakeLists.txt" + ] + }, + { + "name": "Cobol", + "extensions": [ + "cob", + "cbl", + "ccp", + "cobol", + "cpy" + ], + "filenames": [] + }, + { + "name": "CoffeeScript", + "extensions": [ + "coffee", + "_coffee", + "cake", + "cjsx", + "iced" + ], + "filenames": [ + "Cakefile" + ] + }, + { + "name": "Common Lisp", + "extensions": [ + "lisp", + "asd", + "cl", + "l", + "lsp", + "ny", + "podsl", + "sexp" + ], + "filenames": [] + }, + { + "name": "Crystal", + "extensions": [ + "cr" + ], + "filenames": [] + }, + { + "name": "CSS", + "extensions": [ + "css" + ], + "filenames": [] + }, + { + "name": "Cython", + "extensions": [ + "pyx", + "pxd", + "pxi" + ], + "filenames": [] + }, + { + "name": "D", + "extensions": [ + "d", + "di" + ], + "filenames": [] + }, + { + "name": "Dart", + "extensions": [ + "dart" + ], + "filenames": [] + }, + { + "name": "diff", + "extensions": [ + "diff", + "patch" + ], + "filenames": [] + }, + { + "name": "Dockerfile", + "extensions": [ + "dockerfile", + "containerfile" + ], + "filenames": [ + "Containerfile", + "Dockerfile" + ] + }, + { + "name": "Elm", + "extensions": [ + "elm" + ], + "filenames": [] + }, + { + "name": "Erlang", + "extensions": [ + "erl", + "app", + "es", + "escript", + "hrl", + "xrl", + "yrl" + ], + "filenames": [ + "Emakefile", + "rebar.config", + "rebar.config.lock", + "rebar.lock" + ] + }, + { + "name": "F#", + "extensions": [ + "fs", + "fsi", + "fsx" + ], + "filenames": [] + }, + { + "name": "Fortran", + "extensions": [ + "f", + "f77", + "for", + "fpp" + ], + "filenames": [] + }, + { + "name": "Go", + "extensions": [ + "go" + ], + "filenames": [] + }, + { + "name": "Groovy", + "extensions": [ + "groovy", + "grt", + "gtpl", + "gvy" + ], + "filenames": [ + "Jenkinsfile" + ] + }, + { + "name": "Haskell", + "extensions": [ + "hs", + "hs-boot", + "hsc" + ], + "filenames": [] + }, + { + "name": "HTML", + "extensions": [ + "html", + "hta", + "htm", + "inc", + "xht", + "xhtml" + ], + "filenames": [] + }, + { + "name": "Java", + "extensions": [ + "java", + "jav", + "jsh" + ], + "filenames": [] + }, + { + "name": "JavaScript", + "extensions": [ + "js", + "cjs", + "jsx", + "mjs" + ], + "filenames": [ + "Jakefile" + ] + }, + { + "name": "JSON", + "extensions": [ + "json", + "4DForm", + "4DProject", + "avsc", + "geojson", + "gltf", + "har", + "ice", + "JSON-tmLanguage", + "jsonl", + "mcmeta", + "sarif", + "tact", + "tfstate", + "topojson", + "webapp", + "webmanifest", + "yy", + "yyp" + ], + "filenames": [ + ".all-contributorsrc", + ".arcconfig", + ".auto-changelog", + ".c8rc", + ".htmlhintrc", + ".imgbotconfig", + ".nycrc", + ".tern-config", + ".tern-project", + ".watchmanconfig", + "MODULE.bazel.lock", + "Package.resolved", + "Pipfile.lock", + "bun.lock", + "composer.lock", + "deno.lock", + "flake.lock", + "mcmod.info" + ] + }, + { + "name": "Julia", + "extensions": [ + "jl" + ], + "filenames": [] + }, + { + "name": "Kotlin", + "extensions": [ + "kt", + "ktm", + "kts" + ], + "filenames": [] + }, + { + "name": "LaTeX", + "extensions": [ + "tex", + "aux", + "bbx", + "cbx", + "cls", + "dtx", + "ins", + "lbx", + "ltx", + "mkii", + "mkiv", + "mkvi", + "sty", + "toc" + ], + "filenames": [] + }, + { + "name": "LESS", + "extensions": [ + "less" + ], + "filenames": [] + }, + { + "name": "LiveScript", + "extensions": [ + "ls", + "_ls" + ], + "filenames": [ + "Slakefile" + ] + }, + { + "name": "Lua", + "extensions": [ + "lua", + "nse", + "p8", + "pd_lua", + "rbxs", + "rockspec", + "wlua" + ], + "filenames": [ + ".luacheckrc" + ] + }, + { + "name": "Markdown", + "extensions": [ + "md", + "livemd", + "markdown", + "mdown", + "mdwn", + "mkd", + "mkdn", + "mkdown", + "ronn", + "scd", + "workbook" + ], + "filenames": [ + "contents.lr" + ] + }, + { + "name": "Nginx", + "extensions": [ + "nginx", + "nginxconf", + "vhost" + ], + "filenames": [ + "nginx.conf" + ] + }, + { + "name": "OCaml", + "extensions": [ + "ml", + "eliom", + "eliomi", + "ml4", + "mli", + "mll", + "mly" + ], + "filenames": [] + }, + { + "name": "Pascal", + "extensions": [ + "pas", + "dfm", + "dpr", + "inc", + "lpr", + "pascal", + "pp" + ], + "filenames": [] + }, + { + "name": "Perl", + "extensions": [ + "pl", + "al", + "perl", + "ph", + "plx", + "pm", + "psgi", + "t" + ], + "filenames": [ + ".latexmkrc", + "Makefile.PL", + "Rexfile", + "ack", + "cpanfile", + "latexmkrc" + ] + }, + { + "name": "PHP", + "extensions": [ + "php", + "aw", + "ctp", + "php3", + "php4", + "php5", + "phps", + "phpt" + ], + "filenames": [ + ".php", + ".php_cs", + ".php_cs.dist", + "Phakefile" + ] + }, + { + "name": "PowerShell", + "extensions": [ + "ps1", + "psd1", + "psm1" + ], + "filenames": [] + }, + { + "name": "Properties files", + "extensions": [ + "ini", + "cfg", + "cnf", + "dof", + "lektorproject", + "prefs", + "pro", + "properties", + "url" + ], + "filenames": [ + ".buckconfig", + ".coveragerc", + ".flake8", + ".pylintrc", + "HOSTS", + "buildozer.spec", + "hosts", + "pylintrc", + "vlcrc" + ] + }, + { + "name": "ProtoBuf", + "extensions": [ + "proto" + ], + "filenames": [] + }, + { + "name": "Pug", + "extensions": [ + "jade", + "pug" + ], + "filenames": [] + }, + { + "name": "Puppet", + "extensions": [ + "pp" + ], + "filenames": [ + "Modulefile" + ] + }, + { + "name": "Python", + "extensions": [ + "py", + "gyp", + "gypi", + "lmi", + "py3", + "pyde", + "pyi", + "pyp", + "pyt", + "pyw", + "rpy", + "tac", + "wsgi", + "xpy" + ], + "filenames": [ + ".gclient", + "DEPS", + "SConscript", + "SConstruct", + "wscript" + ] + }, + { + "name": "R", + "extensions": [ + "r", + "rd", + "rsx" + ], + "filenames": [ + ".Rprofile", + "expr-dist" + ] + }, + { + "name": "Ruby", + "extensions": [ + "rb", + "builder", + "eye", + "gemspec", + "god", + "jbuilder", + "mspec", + "pluginspec", + "podspec", + "prawn", + "rabl", + "rake", + "rbi", + "rbuild", + "rbw", + "rbx", + "ru", + "ruby", + "thor", + "watchr" + ], + "filenames": [ + ".irbrc", + ".pryrc", + ".simplecov", + "Appraisals", + "Berksfile", + "Brewfile", + "Buildfile", + "Capfile", + "Dangerfile", + "Deliverfile", + "Fastfile", + "Gemfile", + "Guardfile", + "Jarfile", + "Mavenfile", + "Podfile", + "Puppetfile", + "Rakefile", + "Snapfile", + "Steepfile", + "Thorfile", + "Vagrantfile", + "buildfile" + ] + }, + { + "name": "Rust", + "extensions": [ + "rs" + ], + "filenames": [] + }, + { + "name": "Sass", + "extensions": [ + "sass" + ], + "filenames": [] + }, + { + "name": "Scala", + "extensions": [ + "scala", + "kojo", + "sbt", + "sc" + ], + "filenames": [] + }, + { + "name": "Scheme", + "extensions": [ + "scm", + "sch", + "sld", + "sls", + "sps", + "ss" + ], + "filenames": [] + }, + { + "name": "SCSS", + "extensions": [ + "scss" + ], + "filenames": [] + }, + { + "name": "Shell", + "extensions": [ + "sh", + "bash", + "bats", + "command", + "ksh", + "sbatch", + "slurm", + "tmux", + "tool", + "trigger", + "zsh", + "zsh-theme" + ], + "filenames": [ + ".bash_aliases", + ".bash_functions", + ".bash_history", + ".bash_logout", + ".bash_profile", + ".bashrc", + ".cshrc", + ".envrc", + ".flaskenv", + ".kshrc", + ".login", + ".profile", + ".tmux.conf", + ".xinitrc", + ".xsession", + ".zlogin", + ".zlogout", + ".zprofile", + ".zshenv", + ".zshrc", + "9fs", + "PKGBUILD", + "bash_aliases", + "bash_logout", + "bash_profile", + "bashrc", + "cshrc", + "gradlew", + "kshrc", + "login", + "man", + "mvnw", + "profile", + "tmux.conf", + "xinitrc", + "xsession", + "zlogin", + "zlogout", + "zprofile", + "zshenv", + "zshrc" + ] + }, + { + "name": "Smalltalk", + "extensions": [ + "st", + "cs" + ], + "filenames": [] + }, + { + "name": "SQL", + "extensions": [ + "sql", + "ddl", + "inc", + "mysql", + "prc", + "tab", + "udf", + "viw" + ], + "filenames": [] + }, + { + "name": "Stylus", + "extensions": [ + "styl" + ], + "filenames": [] + }, + { + "name": "Swift", + "extensions": [ + "swift" + ], + "filenames": [] + }, + { + "name": "SystemVerilog", + "extensions": [ + "sv", + "svh", + "vh" + ], + "filenames": [] + }, + { + "name": "Tcl", + "extensions": [ + "tcl", + "adp", + "sdc", + "tm", + "xdc" + ], + "filenames": [ + "owh", + "starfield" + ] + }, + { + "name": "TOML", + "extensions": [ + "toml" + ], + "filenames": [ + "Cargo.lock", + "Cargo.toml.orig", + "Gopkg.lock", + "Pipfile", + "mise.local.lock", + "mise.lock", + "pdm.lock", + "poetry.lock", + "uv.lock" + ] + }, + { + "name": "TSX", + "extensions": [ + "tsx" + ], + "filenames": [] + }, + { + "name": "TypeScript", + "extensions": [ + "ts", + "cts", + "mts" + ], + "filenames": [] + }, + { + "name": "Verilog", + "extensions": [ + "v", + "veo" + ], + "filenames": [] + }, + { + "name": "VHDL", + "extensions": [ + "vhdl", + "vhd", + "vhf", + "vhi", + "vho", + "vhs", + "vht", + "vhw" + ], + "filenames": [] + }, + { + "name": "Vue", + "extensions": [ + "vue" + ], + "filenames": [] + }, + { + "name": "WebAssembly", + "extensions": [ + "wast", + "wat" + ], + "filenames": [] + }, + { + "name": "XML", + "extensions": [ + "xml", + "adml", + "admx", + "ant", + "axaml", + "axml", + "builds", + "ccproj", + "ccxml", + "clixml", + "cproject", + "cscfg", + "csdef", + "csl", + "csproj", + "ct", + "depproj", + "dita", + "ditamap", + "ditaval", + "dotsettings", + "filters", + "fsproj", + "fxml", + "glade", + "gml", + "gmx", + "gpx", + "grxml", + "gst", + "hzp", + "icls", + "iml", + "ivy", + "jelly", + "kml", + "launch", + "mdpolicy", + "mjml", + "mm", + "mod", + "mojo", + "mxml", + "natvis", + "ncl", + "ndproj", + "nproj", + "nuspec", + "odd", + "osm", + "pkgproj", + "pluginspec", + "proj", + "props", + "ps1xml", + "psc1", + "pt", + "pubxml", + "qhelp", + "rdf", + "res", + "resx", + "rs", + "rss", + "sch", + "scxml", + "sfproj", + "shproj", + "slnx", + "srdf", + "storyboard", + "sublime-snippet", + "sw", + "targets", + "tml", + "typ", + "ui", + "urdf", + "ux", + "vbproj", + "vcxproj", + "vsixmanifest", + "vssettings", + "vstemplate", + "vxml", + "wixproj", + "workflow", + "wsdl", + "wsf", + "wxi", + "wxl", + "wxs", + "x3d", + "xacro", + "xaml", + "xib", + "xlf", + "xliff", + "xmi", + "xmp", + "xproj", + "xsd", + "xspec", + "xul", + "zcml" + ], + "filenames": [ + ".classpath", + ".cproject", + ".project", + "App.config", + "NuGet.config", + "Settings.StyleCop", + "Web.Debug.config", + "Web.Release.config", + "Web.config", + "packages.config" + ] + }, + { + "name": "YAML", + "extensions": [ + "yml", + "mir", + "reek", + "rviz", + "sublime-syntax", + "syntax", + "yaml", + "yaml-tmlanguage" + ], + "filenames": [ + ".clang-format", + ".clang-tidy", + ".clangd", + ".gemrc", + "CITATION.cff", + "glide.lock", + "pixi.lock", + "yarn.lock" + ] + } +] diff --git a/tools/generate-codemirror-languages.ts b/tools/generate-codemirror-languages.ts new file mode 100755 index 0000000000..522da8b7d3 --- /dev/null +++ b/tools/generate-codemirror-languages.ts @@ -0,0 +1,151 @@ +#!/usr/bin/env node +import {load as parseYaml} from 'js-yaml'; +import {writeFile} from 'node:fs/promises'; + +const LINGUIST_URL = 'https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml'; + +// Map github-linguist language names to the names CodeMirror's @codemirror/language-data +// uses. Only languages that we want to load with extended extension/filename data are +// listed; everything else falls through to language-data's defaults at runtime. +const linguistToCm: Record = { + 'C': 'C', + 'C++': 'C++', + 'C#': 'C#', + 'CMake': 'CMake', + 'COBOL': 'Cobol', + 'CSS': 'CSS', + 'Clojure': 'Clojure', + 'CoffeeScript': 'CoffeeScript', + 'Common Lisp': 'Common Lisp', + 'Crystal': 'Crystal', + 'Cython': 'Cython', + 'D': 'D', + 'Dart': 'Dart', + 'Diff': 'diff', + 'Dockerfile': 'Dockerfile', + 'Elm': 'Elm', + 'Erlang': 'Erlang', + 'F#': 'F#', + 'Fortran': 'Fortran', + 'Go': 'Go', + 'Groovy': 'Groovy', + 'HTML': 'HTML', + 'Haskell': 'Haskell', + 'INI': 'Properties files', + 'JSON': 'JSON', + 'Java': 'Java', + 'JavaScript': 'JavaScript', + 'Julia': 'Julia', + 'Kotlin': 'Kotlin', + 'Less': 'LESS', + 'LiveScript': 'LiveScript', + 'Lua': 'Lua', + 'Markdown': 'Markdown', + 'Nginx': 'Nginx', + 'OCaml': 'OCaml', + 'PHP': 'PHP', + 'Pascal': 'Pascal', + 'Perl': 'Perl', + 'PowerShell': 'PowerShell', + 'Protocol Buffer': 'ProtoBuf', + 'Pug': 'Pug', + 'Puppet': 'Puppet', + 'Python': 'Python', + 'R': 'R', + 'Ruby': 'Ruby', + 'Rust': 'Rust', + 'SCSS': 'SCSS', + 'SQL': 'SQL', + 'Sass': 'Sass', + 'Scala': 'Scala', + 'Scheme': 'Scheme', + 'Shell': 'Shell', + 'Smalltalk': 'Smalltalk', + 'Stylus': 'Stylus', + 'Swift': 'Swift', + 'SystemVerilog': 'SystemVerilog', + 'TOML': 'TOML', + 'TSX': 'TSX', + 'Tcl': 'Tcl', + 'TeX': 'LaTeX', + 'TypeScript': 'TypeScript', + 'VHDL': 'VHDL', + 'Verilog': 'Verilog', + 'Vue': 'Vue', + 'WebAssembly': 'WebAssembly', + 'XML': 'XML', + 'YAML': 'YAML', +}; + +// Per-language extensions to drop. Use only for extensions that would actively collide +// with another language (e.g. .inc claimed by both PHP and C++) or where the syntax is +// genuinely incompatible with the CodeMirror mode (e.g. .csh vs sh). +const excludeExt: Record = { + 'C++': ['inc'], + 'INI': ['frm'], + 'JavaScript': ['_js', 'bones', 'es', 'es6', 'frag', 'gs', 'jake', 'javascript', 'jsb', 'jscad', 'jsfl', 'jslib', 'jsm', 'jspre', 'jss', 'njs', 'pac', 'sjs', 'ssjs', 'xsjs', 'xsjslib'], + 'Lua': ['fcgi'], + 'PHP': ['fcgi', 'inc'], + 'Perl': ['cgi', 'fcgi'], + 'Python': ['cgi', 'fcgi', 'spec'], + 'Ruby': ['fcgi', 'spec'], + 'Shell': ['cgi', 'csh', 'fcgi'], + 'XML': ['inc', 'jsproj', 'tmpl', 'ts', 'tsx'], +}; + +type LinguistEntry = { + type: string; + extensions?: string[]; + filenames?: string[]; +}; + +type CmLanguage = { + name: string; + extensions: string[]; + filenames: string[]; +}; + +async function main() { + const res = await fetch(LINGUIST_URL); // eslint-disable-line no-restricted-globals -- node build script, not browser code + if (!res.ok) throw new Error(`fetch ${LINGUIST_URL} failed: ${res.status}`); + const linguist = parseYaml(await res.text()) as Record; + + const out: CmLanguage[] = []; + const missing: string[] = []; + for (const [linguistName, cmName] of Object.entries(linguistToCm)) { + const entry = linguist[linguistName]; + if (!entry) { + missing.push(linguistName); + continue; + } + const exExt = new Set(excludeExt[linguistName]); + // CodeMirror's matchFilename uses /\.([^.]+)$/ to extract the suffix, so multi-dot + // extensions like ".cmake.in" cannot match as extensions and are dropped here. + const extensions = (entry.extensions ?? []) + .map((e) => e.replace(/^\./, '')) + .filter((e) => !e.includes('.') && !exExt.has(e)); + const filenames = entry.filenames ?? []; + out.push({ + name: cmName, + extensions: Array.from(new Set(extensions)), + filenames: Array.from(new Set(filenames)), + }); + } + + if (missing.length) { + console.warn(`linguist entries not found: ${missing.join(', ')}`); + } + + out.sort((a, b) => a.name.localeCompare(b.name)); + + const outPath = new URL('../assets/codemirror-languages.json', import.meta.url); + await writeFile(outPath, `${JSON.stringify(out, null, 2)}\n`); + console.info(`wrote ${out.length} languages to ${outPath.pathname}`); +} + +try { + await main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/web_src/js/modules/codeeditor/main.ts b/web_src/js/modules/codeeditor/main.ts index e847f357cb..793a4e9491 100644 --- a/web_src/js/modules/codeeditor/main.ts +++ b/web_src/js/modules/codeeditor/main.ts @@ -43,8 +43,10 @@ export type CodemirrorEditor = { export type CodemirrorModules = Awaited>; +type LinguistLanguage = {name: string; extensions: string[]; filenames: string[]}; + async function importCodemirror() { - const [autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap] = await Promise.all([ + const [autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap, linguistJson] = await Promise.all([ import('@codemirror/autocomplete'), import('@codemirror/commands'), import('@codemirror/language'), @@ -56,8 +58,42 @@ async function importCodemirror() { import('@lezer/highlight'), import('@replit/codemirror-indentation-markers'), import('@replit/codemirror-vscode-keymap'), + import('../../../../assets/codemirror-languages.json', {with: {type: 'json'}}), ]); - return {autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap}; + return {autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap, linguistLanguages: linguistJson.default as LinguistLanguage[]}; +} + +const manualFilenames: Record = { + 'Properties files': ['.editorconfig', '.gitconfig', '.npmrc'], + 'Python': ['Snakefile'], +}; +const manualExtensions: Record = { + 'Properties files': ['conf'], +}; +const handledByCustomEntry = new Set(['Dockerfile', 'Markdown']); +const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const filenameUnion = (filenames: string[]) => + filenames.length ? new RegExp(`^(${filenames.map(escapeRegex).join('|')})$`) : undefined; + +let baseLanguagesCache: LanguageDescription[] | null = null; +function buildBaseLanguages(cm: CodemirrorModules): LanguageDescription[] { + if (baseLanguagesCache) return baseLanguagesCache; + const loadByName = new Map( + cm.languageData.languages.map((l: LanguageDescription) => [l.name, l.load.bind(l)]), + ); + const overrides = cm.linguistLanguages + .filter((l) => loadByName.has(l.name) && !handledByCustomEntry.has(l.name)) + .map((l) => cm.language.LanguageDescription.of({ + name: l.name, + extensions: [...l.extensions, ...(manualExtensions[l.name] ?? [])], + filename: filenameUnion([...l.filenames, ...(manualFilenames[l.name] ?? [])]), + load: loadByName.get(l.name)!, + })); + const overrideNames = new Set(overrides.map((o) => o.name)); + const fallback = cm.languageData.languages.filter( + (l: LanguageDescription) => !overrideNames.has(l.name) && !handledByCustomEntry.has(l.name), + ); + return baseLanguagesCache = [...overrides, ...fallback]; } function togglePreviewDisplay(previewable: boolean): void { @@ -85,13 +121,20 @@ export async function createCodeEditor(textarea: HTMLTextAreaElement, filenameIn const previewableExts = new Set(config.previewableExtensions || []); const lineWrapExts = config.lineWrapExtensions || []; const cm = await importCodemirror(); + const markdown = cm.linguistLanguages.find((l) => l.name === 'Markdown'); + const dockerfile = cm.linguistLanguages.find((l) => l.name === 'Dockerfile'); const languageDescriptions: LanguageDescription[] = [ - ...cm.languageData.languages.filter((l: LanguageDescription) => l.name !== 'Markdown'), + ...buildBaseLanguages(cm), cm.language.LanguageDescription.of({ - name: 'Markdown', extensions: ['md', 'markdown', 'mkd'], + name: 'Markdown', extensions: markdown?.extensions ?? ['md', 'markdown', 'mkd'], load: async () => (await import('@codemirror/lang-markdown')).markdown({codeLanguages: languageDescriptions}), }), + cm.language.LanguageDescription.of({ + name: 'Dockerfile', extensions: dockerfile?.extensions ?? ['dockerfile', 'containerfile'], + filename: /^(Containerfile|Dockerfile)(\..+)?$/i, + load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/dockerfile')).dockerFile)), + }), cm.language.LanguageDescription.of({ name: 'Elixir', extensions: ['ex', 'exs'], load: async () => (await import('codemirror-lang-elixir')).elixir(), @@ -105,7 +148,7 @@ export async function createCodeEditor(textarea: HTMLTextAreaElement, filenameIn load: async () => (await import('@replit/codemirror-lang-svelte')).svelte(), }), cm.language.LanguageDescription.of({ - name: 'Makefile', filename: /^(GNUm|M|m)akefile$/, + name: 'Makefile', extensions: ['mk', 'mak', 'make'], filename: /^(GNU|BSD)?[Mm]akefile(\..+)?$/, load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)), }), cm.language.LanguageDescription.of({ From f905f5e138acfd6677ffa13726bb867572506b1a Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:47:21 +0200 Subject: [PATCH 2/6] fix(editor): use static import for codemirror language data The dynamic `import(...json, {with: {type: 'json'}})` form left a literal `?import` URL in the entry chunk under rolldown, which the backend then served with the wrong MIME type. Static import bundles the data inline correctly. Co-Authored-By: Claude (Opus 4.7) --- web_src/js/modules/codeeditor/main.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/web_src/js/modules/codeeditor/main.ts b/web_src/js/modules/codeeditor/main.ts index 793a4e9491..e703203038 100644 --- a/web_src/js/modules/codeeditor/main.ts +++ b/web_src/js/modules/codeeditor/main.ts @@ -7,6 +7,7 @@ import type {PaletteCommand} from './command-palette.ts'; import {contextMenu, collectSymbols, selectAllOccurrences} from './context-menu.ts'; import {createJsonLinter, createSyntaxErrorLinter} from './linter.ts'; import {clickableUrls, goToDefinitionAt, trimTrailingWhitespaceFromView} from './utils.ts'; +import linguistLanguages from '../../../../assets/codemirror-languages.json' with {type: 'json'}; import type {LanguageDescription, LanguageSupport} from '@codemirror/language'; import type {Compartment, Extension} from '@codemirror/state'; import type {EditorView, ViewUpdate} from '@codemirror/view'; @@ -43,10 +44,8 @@ export type CodemirrorEditor = { export type CodemirrorModules = Awaited>; -type LinguistLanguage = {name: string; extensions: string[]; filenames: string[]}; - async function importCodemirror() { - const [autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap, linguistJson] = await Promise.all([ + const [autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap] = await Promise.all([ import('@codemirror/autocomplete'), import('@codemirror/commands'), import('@codemirror/language'), @@ -58,9 +57,8 @@ async function importCodemirror() { import('@lezer/highlight'), import('@replit/codemirror-indentation-markers'), import('@replit/codemirror-vscode-keymap'), - import('../../../../assets/codemirror-languages.json', {with: {type: 'json'}}), ]); - return {autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap, linguistLanguages: linguistJson.default as LinguistLanguage[]}; + return {autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap}; } const manualFilenames: Record = { @@ -81,7 +79,7 @@ function buildBaseLanguages(cm: CodemirrorModules): LanguageDescription[] { const loadByName = new Map( cm.languageData.languages.map((l: LanguageDescription) => [l.name, l.load.bind(l)]), ); - const overrides = cm.linguistLanguages + const overrides = linguistLanguages .filter((l) => loadByName.has(l.name) && !handledByCustomEntry.has(l.name)) .map((l) => cm.language.LanguageDescription.of({ name: l.name, @@ -121,8 +119,8 @@ export async function createCodeEditor(textarea: HTMLTextAreaElement, filenameIn const previewableExts = new Set(config.previewableExtensions || []); const lineWrapExts = config.lineWrapExtensions || []; const cm = await importCodemirror(); - const markdown = cm.linguistLanguages.find((l) => l.name === 'Markdown'); - const dockerfile = cm.linguistLanguages.find((l) => l.name === 'Dockerfile'); + const markdown = linguistLanguages.find((l) => l.name === 'Markdown'); + const dockerfile = linguistLanguages.find((l) => l.name === 'Dockerfile'); const languageDescriptions: LanguageDescription[] = [ ...buildBaseLanguages(cm), From 693752eb49f1bc1bd3e69f576d84615dc31290ec Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:49:52 +0200 Subject: [PATCH 3/6] chore(editor): rename constant, simplify Makefile help comment Co-Authored-By: Claude (Opus 4.7) --- Makefile | 2 +- tools/generate-codemirror-languages.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7db366c57f..b1fdfce6cc 100644 --- a/Makefile +++ b/Makefile @@ -662,7 +662,7 @@ generate-images: | node_modules ## generate images cd tools && node generate-images.ts $(TAGS) .PHONY: generate-codemirror-languages -generate-codemirror-languages: | node_modules ## refresh assets/codemirror-languages.json from github-linguist +generate-codemirror-languages: | node_modules ## generate codemirror-languages.json node tools/generate-codemirror-languages.ts .PHONY: generate-manpage diff --git a/tools/generate-codemirror-languages.ts b/tools/generate-codemirror-languages.ts index 522da8b7d3..ebf9e496a5 100755 --- a/tools/generate-codemirror-languages.ts +++ b/tools/generate-codemirror-languages.ts @@ -2,7 +2,7 @@ import {load as parseYaml} from 'js-yaml'; import {writeFile} from 'node:fs/promises'; -const LINGUIST_URL = 'https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml'; +const linguistUrl = 'https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml'; // Map github-linguist language names to the names CodeMirror's @codemirror/language-data // uses. Only languages that we want to load with extended extension/filename data are @@ -106,8 +106,8 @@ type CmLanguage = { }; async function main() { - const res = await fetch(LINGUIST_URL); // eslint-disable-line no-restricted-globals -- node build script, not browser code - if (!res.ok) throw new Error(`fetch ${LINGUIST_URL} failed: ${res.status}`); + const res = await fetch(linguistUrl); // eslint-disable-line no-restricted-globals -- node build script, not browser code + if (!res.ok) throw new Error(`fetch ${linguistUrl} failed: ${res.status}`); const linguist = parseYaml(await res.text()) as Record; const out: CmLanguage[] = []; From 023a11a3f96b9dd1b1f92126251201af1cc3dc9b Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:52:13 +0200 Subject: [PATCH 4/6] =?UTF-8?q?refactor(editor):=20collapse=20linguist?= =?UTF-8?q?=E2=86=92cm=20map=20into=20a=20list=20of=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 61 of the 67 entries had identical keys and values; replace the verbose `Record` with a list where bare strings imply no rename and tuples carry the few that do. Co-Authored-By: Claude (Opus 4.7) --- tools/generate-codemirror-languages.ts | 89 +++++--------------------- 1 file changed, 16 insertions(+), 73 deletions(-) diff --git a/tools/generate-codemirror-languages.ts b/tools/generate-codemirror-languages.ts index ebf9e496a5..d2bdf14d72 100755 --- a/tools/generate-codemirror-languages.ts +++ b/tools/generate-codemirror-languages.ts @@ -4,78 +4,20 @@ import {writeFile} from 'node:fs/promises'; const linguistUrl = 'https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml'; -// Map github-linguist language names to the names CodeMirror's @codemirror/language-data -// uses. Only languages that we want to load with extended extension/filename data are -// listed; everything else falls through to language-data's defaults at runtime. -const linguistToCm: Record = { - 'C': 'C', - 'C++': 'C++', - 'C#': 'C#', - 'CMake': 'CMake', - 'COBOL': 'Cobol', - 'CSS': 'CSS', - 'Clojure': 'Clojure', - 'CoffeeScript': 'CoffeeScript', - 'Common Lisp': 'Common Lisp', - 'Crystal': 'Crystal', - 'Cython': 'Cython', - 'D': 'D', - 'Dart': 'Dart', - 'Diff': 'diff', - 'Dockerfile': 'Dockerfile', - 'Elm': 'Elm', - 'Erlang': 'Erlang', - 'F#': 'F#', - 'Fortran': 'Fortran', - 'Go': 'Go', - 'Groovy': 'Groovy', - 'HTML': 'HTML', - 'Haskell': 'Haskell', - 'INI': 'Properties files', - 'JSON': 'JSON', - 'Java': 'Java', - 'JavaScript': 'JavaScript', - 'Julia': 'Julia', - 'Kotlin': 'Kotlin', - 'Less': 'LESS', - 'LiveScript': 'LiveScript', - 'Lua': 'Lua', - 'Markdown': 'Markdown', - 'Nginx': 'Nginx', - 'OCaml': 'OCaml', - 'PHP': 'PHP', - 'Pascal': 'Pascal', - 'Perl': 'Perl', - 'PowerShell': 'PowerShell', - 'Protocol Buffer': 'ProtoBuf', - 'Pug': 'Pug', - 'Puppet': 'Puppet', - 'Python': 'Python', - 'R': 'R', - 'Ruby': 'Ruby', - 'Rust': 'Rust', - 'SCSS': 'SCSS', - 'SQL': 'SQL', - 'Sass': 'Sass', - 'Scala': 'Scala', - 'Scheme': 'Scheme', - 'Shell': 'Shell', - 'Smalltalk': 'Smalltalk', - 'Stylus': 'Stylus', - 'Swift': 'Swift', - 'SystemVerilog': 'SystemVerilog', - 'TOML': 'TOML', - 'TSX': 'TSX', - 'Tcl': 'Tcl', - 'TeX': 'LaTeX', - 'TypeScript': 'TypeScript', - 'VHDL': 'VHDL', - 'Verilog': 'Verilog', - 'Vue': 'Vue', - 'WebAssembly': 'WebAssembly', - 'XML': 'XML', - 'YAML': 'YAML', -}; +// Languages to extract from github-linguist. A bare string means the linguist name +// matches CodeMirror's @codemirror/language-data name; a tuple is [linguist, cm] when +// they differ. Anything not listed falls through to language-data's defaults at runtime. +const languages: Array = [ + 'C', 'C++', 'C#', 'CMake', ['COBOL', 'Cobol'], 'CSS', 'Clojure', 'CoffeeScript', + 'Common Lisp', 'Crystal', 'Cython', 'D', 'Dart', ['Diff', 'diff'], 'Dockerfile', + 'Elm', 'Erlang', 'F#', 'Fortran', 'Go', 'Groovy', 'HTML', 'Haskell', + ['INI', 'Properties files'], 'JSON', 'Java', 'JavaScript', 'Julia', 'Kotlin', + ['Less', 'LESS'], 'LiveScript', 'Lua', 'Markdown', 'Nginx', 'OCaml', 'PHP', 'Pascal', + 'Perl', 'PowerShell', ['Protocol Buffer', 'ProtoBuf'], 'Pug', 'Puppet', 'Python', 'R', + 'Ruby', 'Rust', 'SCSS', 'SQL', 'Sass', 'Scala', 'Scheme', 'Shell', 'Smalltalk', + 'Stylus', 'Swift', 'SystemVerilog', 'TOML', 'TSX', 'Tcl', ['TeX', 'LaTeX'], + 'TypeScript', 'VHDL', 'Verilog', 'Vue', 'WebAssembly', 'XML', 'YAML', +]; // Per-language extensions to drop. Use only for extensions that would actively collide // with another language (e.g. .inc claimed by both PHP and C++) or where the syntax is @@ -112,7 +54,8 @@ async function main() { const out: CmLanguage[] = []; const missing: string[] = []; - for (const [linguistName, cmName] of Object.entries(linguistToCm)) { + for (const lang of languages) { + const [linguistName, cmName] = typeof lang === 'string' ? [lang, lang] : lang; const entry = linguist[linguistName]; if (!entry) { missing.push(linguistName); From 5764fc25463fd658994ea01daaf519964e7f2d57 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:58:30 +0200 Subject: [PATCH 5/6] refactor(editor): auto-match against @codemirror/language-data Drop the hand-curated 67-name list; intersect Linguist's full set against language-data's exported names instead. A small renames map covers the six places where the two registries disagree on naming. Adds 34 niche languages we hadn't curated (Cypher, Eiffel, Erlang, Mathematica, etc.) at +4 KB on the JSON. Co-Authored-By: Claude (Opus 4.7) --- assets/codemirror-languages.json | 283 +++++++++++++++++++++++++ tools/generate-codemirror-languages.ts | 40 ++-- 2 files changed, 297 insertions(+), 26 deletions(-) diff --git a/assets/codemirror-languages.json b/assets/codemirror-languages.json index 5bab971eb7..0e4639c8ba 100644 --- a/assets/codemirror-languages.json +++ b/assets/codemirror-languages.json @@ -1,4 +1,28 @@ [ + { + "name": "APL", + "extensions": [ + "apl", + "dyalog" + ], + "filenames": [] + }, + { + "name": "ASN.1", + "extensions": [ + "asn", + "asn1" + ], + "filenames": [] + }, + { + "name": "Brainfuck", + "extensions": [ + "b", + "bf" + ], + "filenames": [] + }, { "name": "C", "extensions": [ @@ -108,6 +132,13 @@ ], "filenames": [] }, + { + "name": "CQL", + "extensions": [ + "cql" + ], + "filenames": [] + }, { "name": "Crystal", "extensions": [ @@ -122,6 +153,14 @@ ], "filenames": [] }, + { + "name": "Cypher", + "extensions": [ + "cyp", + "cypher" + ], + "filenames": [] + }, { "name": "Cython", "extensions": [ @@ -165,6 +204,45 @@ "Dockerfile" ] }, + { + "name": "Dylan", + "extensions": [ + "dylan", + "dyl", + "intr", + "lid" + ], + "filenames": [] + }, + { + "name": "EBNF", + "extensions": [ + "ebnf" + ], + "filenames": [] + }, + { + "name": "ECL", + "extensions": [ + "ecl", + "eclxml" + ], + "filenames": [] + }, + { + "name": "edn", + "extensions": [ + "edn" + ], + "filenames": [] + }, + { + "name": "Eiffel", + "extensions": [ + "e" + ], + "filenames": [] + }, { "name": "Elm", "extensions": [ @@ -199,6 +277,30 @@ ], "filenames": [] }, + { + "name": "Factor", + "extensions": [ + "factor" + ], + "filenames": [ + ".factor-boot-rc", + ".factor-rc" + ] + }, + { + "name": "Forth", + "extensions": [ + "fth", + "4th", + "f", + "for", + "forth", + "fr", + "frt", + "fs" + ], + "filenames": [] + }, { "name": "Fortran", "extensions": [ @@ -209,6 +311,14 @@ ], "filenames": [] }, + { + "name": "Gherkin", + "extensions": [ + "feature", + "story" + ], + "filenames": [] + }, { "name": "Go", "extensions": [ @@ -237,6 +347,14 @@ ], "filenames": [] }, + { + "name": "Haxe", + "extensions": [ + "hx", + "hxsl" + ], + "filenames": [] + }, { "name": "HTML", "extensions": [ @@ -249,6 +367,28 @@ ], "filenames": [] }, + { + "name": "HTTP", + "extensions": [ + "http" + ], + "filenames": [] + }, + { + "name": "HXML", + "extensions": [ + "hxml" + ], + "filenames": [] + }, + { + "name": "IDL", + "extensions": [ + "pro", + "dlm" + ], + "filenames": [] + }, { "name": "Java", "extensions": [ @@ -270,6 +410,15 @@ "Jakefile" ] }, + { + "name": "Jinja", + "extensions": [ + "jinja", + "j2", + "jinja2" + ], + "filenames": [] + }, { "name": "JSON", "extensions": [ @@ -357,6 +506,13 @@ ], "filenames": [] }, + { + "name": "Liquid", + "extensions": [ + "liquid" + ], + "filenames": [] + }, { "name": "LiveScript", "extensions": [ @@ -401,6 +557,13 @@ "contents.lr" ] }, + { + "name": "Modelica", + "extensions": [ + "mo" + ], + "filenames": [] + }, { "name": "Nginx", "extensions": [ @@ -412,6 +575,29 @@ "nginx.conf" ] }, + { + "name": "NSIS", + "extensions": [ + "nsi", + "nsh" + ], + "filenames": [] + }, + { + "name": "Objective-C", + "extensions": [ + "m", + "h" + ], + "filenames": [] + }, + { + "name": "Objective-C++", + "extensions": [ + "mm" + ], + "filenames": [] + }, { "name": "OCaml", "extensions": [ @@ -425,6 +611,13 @@ ], "filenames": [] }, + { + "name": "Oz", + "extensions": [ + "oz" + ], + "filenames": [] + }, { "name": "Pascal", "extensions": [ @@ -478,6 +671,28 @@ "Phakefile" ] }, + { + "name": "PLSQL", + "extensions": [ + "pls", + "bdy", + "ddl", + "fnc", + "pck", + "pkb", + "pks", + "plb", + "plsql", + "prc", + "spc", + "sql", + "tpb", + "tps", + "trg", + "vw" + ], + "filenames": [] + }, { "name": "PowerShell", "extensions": [ @@ -574,6 +789,13 @@ "expr-dist" ] }, + { + "name": "RPM Spec", + "extensions": [ + "spec" + ], + "filenames": [] + }, { "name": "Ruby", "extensions": [ @@ -631,6 +853,13 @@ ], "filenames": [] }, + { + "name": "SAS", + "extensions": [ + "sas" + ], + "filenames": [] + }, { "name": "Sass", "extensions": [ @@ -727,6 +956,13 @@ "zshrc" ] }, + { + "name": "Sieve", + "extensions": [ + "sieve" + ], + "filenames": [] + }, { "name": "Smalltalk", "extensions": [ @@ -735,6 +971,14 @@ ], "filenames": [] }, + { + "name": "SPARQL", + "extensions": [ + "sparql", + "rq" + ], + "filenames": [] + }, { "name": "SQL", "extensions": [ @@ -749,6 +993,13 @@ ], "filenames": [] }, + { + "name": "Squirrel", + "extensions": [ + "nut" + ], + "filenames": [] + }, { "name": "Stylus", "extensions": [ @@ -786,6 +1037,13 @@ "starfield" ] }, + { + "name": "Textile", + "extensions": [ + "textile" + ], + "filenames": [] + }, { "name": "TOML", "extensions": [ @@ -810,6 +1068,13 @@ ], "filenames": [] }, + { + "name": "Turtle", + "extensions": [ + "ttl" + ], + "filenames": [] + }, { "name": "TypeScript", "extensions": [ @@ -819,6 +1084,13 @@ ], "filenames": [] }, + { + "name": "VBScript", + "extensions": [ + "vbs" + ], + "filenames": [] + }, { "name": "Verilog", "extensions": [ @@ -978,6 +1250,17 @@ "packages.config" ] }, + { + "name": "XQuery", + "extensions": [ + "xquery", + "xq", + "xql", + "xqm", + "xqy" + ], + "filenames": [] + }, { "name": "YAML", "extensions": [ diff --git a/tools/generate-codemirror-languages.ts b/tools/generate-codemirror-languages.ts index d2bdf14d72..9461580197 100755 --- a/tools/generate-codemirror-languages.ts +++ b/tools/generate-codemirror-languages.ts @@ -1,23 +1,19 @@ #!/usr/bin/env node import {load as parseYaml} from 'js-yaml'; import {writeFile} from 'node:fs/promises'; +import {languages as cmLanguages} from '@codemirror/language-data'; const linguistUrl = 'https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml'; -// Languages to extract from github-linguist. A bare string means the linguist name -// matches CodeMirror's @codemirror/language-data name; a tuple is [linguist, cm] when -// they differ. Anything not listed falls through to language-data's defaults at runtime. -const languages: Array = [ - 'C', 'C++', 'C#', 'CMake', ['COBOL', 'Cobol'], 'CSS', 'Clojure', 'CoffeeScript', - 'Common Lisp', 'Crystal', 'Cython', 'D', 'Dart', ['Diff', 'diff'], 'Dockerfile', - 'Elm', 'Erlang', 'F#', 'Fortran', 'Go', 'Groovy', 'HTML', 'Haskell', - ['INI', 'Properties files'], 'JSON', 'Java', 'JavaScript', 'Julia', 'Kotlin', - ['Less', 'LESS'], 'LiveScript', 'Lua', 'Markdown', 'Nginx', 'OCaml', 'PHP', 'Pascal', - 'Perl', 'PowerShell', ['Protocol Buffer', 'ProtoBuf'], 'Pug', 'Puppet', 'Python', 'R', - 'Ruby', 'Rust', 'SCSS', 'SQL', 'Sass', 'Scala', 'Scheme', 'Shell', 'Smalltalk', - 'Stylus', 'Swift', 'SystemVerilog', 'TOML', 'TSX', 'Tcl', ['TeX', 'LaTeX'], - 'TypeScript', 'VHDL', 'Verilog', 'Vue', 'WebAssembly', 'XML', 'YAML', -]; +// Linguist names that don't match the corresponding @codemirror/language-data name. +const renames: Record = { + 'COBOL': 'Cobol', + 'Diff': 'diff', + 'INI': 'Properties files', + 'Less': 'LESS', + 'Protocol Buffer': 'ProtoBuf', + 'TeX': 'LaTeX', +}; // Per-language extensions to drop. Use only for extensions that would actively collide // with another language (e.g. .inc claimed by both PHP and C++) or where the syntax is @@ -52,15 +48,11 @@ async function main() { if (!res.ok) throw new Error(`fetch ${linguistUrl} failed: ${res.status}`); const linguist = parseYaml(await res.text()) as Record; + const cmNames = new Set(cmLanguages.map((l) => l.name)); const out: CmLanguage[] = []; - const missing: string[] = []; - for (const lang of languages) { - const [linguistName, cmName] = typeof lang === 'string' ? [lang, lang] : lang; - const entry = linguist[linguistName]; - if (!entry) { - missing.push(linguistName); - continue; - } + for (const [linguistName, entry] of Object.entries(linguist)) { + const cmName = renames[linguistName] ?? linguistName; + if (!cmNames.has(cmName)) continue; const exExt = new Set(excludeExt[linguistName]); // CodeMirror's matchFilename uses /\.([^.]+)$/ to extract the suffix, so multi-dot // extensions like ".cmake.in" cannot match as extensions and are dropped here. @@ -75,10 +67,6 @@ async function main() { }); } - if (missing.length) { - console.warn(`linguist entries not found: ${missing.join(', ')}`); - } - out.sort((a, b) => a.name.localeCompare(b.name)); const outPath = new URL('../assets/codemirror-languages.json', import.meta.url); From 1bff063446e9a956b9e10ff6a47ce7fa33aa0763 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 07:21:26 +0200 Subject: [PATCH 6/6] review feedback: scope fetch lint to web_src, add detection tests - Move `no-restricted-globals`/`-properties` rules to the web_src block; the underlying restrictions only apply to browser code, so the eslint-disable on `fetch` in the build script can go away. - Extract `buildLanguageDescriptions` so the assembled list is callable from tests, and add coverage for matchFilename across the extended Linguist + manual rules. Co-Authored-By: Claude (Opus 4.7) --- eslint.config.ts | 6 +- tools/generate-codemirror-languages.ts | 2 +- web_src/js/modules/codeeditor/main.test.ts | 84 ++++++++++++++++++++++ web_src/js/modules/codeeditor/main.ts | 84 +++++++++++----------- 4 files changed, 133 insertions(+), 43 deletions(-) create mode 100644 web_src/js/modules/codeeditor/main.test.ts diff --git a/eslint.config.ts b/eslint.config.ts index 29016ed808..91adc06e19 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -570,8 +570,6 @@ export default defineConfig([ 'no-redeclare': [0], // must be disabled for typescript overloads 'no-regex-spaces': [2], 'no-restricted-exports': [0], - 'no-restricted-globals': [2, ...restrictedGlobals], - 'no-restricted-properties': [2, ...restrictedProperties], 'no-restricted-imports': [2, {paths: [ {name: 'jquery', message: 'Use the global $ instead', allowTypeImports: true}, ]}], @@ -1022,5 +1020,9 @@ export default defineConfig([ { files: ['web_src/**/*'], languageOptions: {globals: {...globals.browser, ...globals.jquery}}, + rules: { + 'no-restricted-globals': [2, ...restrictedGlobals], + 'no-restricted-properties': [2, ...restrictedProperties], + }, }, ]); diff --git a/tools/generate-codemirror-languages.ts b/tools/generate-codemirror-languages.ts index 9461580197..05891298f6 100755 --- a/tools/generate-codemirror-languages.ts +++ b/tools/generate-codemirror-languages.ts @@ -44,7 +44,7 @@ type CmLanguage = { }; async function main() { - const res = await fetch(linguistUrl); // eslint-disable-line no-restricted-globals -- node build script, not browser code + const res = await fetch(linguistUrl); if (!res.ok) throw new Error(`fetch ${linguistUrl} failed: ${res.status}`); const linguist = parseYaml(await res.text()) as Record; diff --git a/web_src/js/modules/codeeditor/main.test.ts b/web_src/js/modules/codeeditor/main.test.ts new file mode 100644 index 0000000000..40a3590d35 --- /dev/null +++ b/web_src/js/modules/codeeditor/main.test.ts @@ -0,0 +1,84 @@ +import {buildLanguageDescriptions, importCodemirror} from './main.ts'; + +test('matchFilename — language detection covers extended rules', async () => { + const cm = await importCodemirror(); + const list = buildLanguageDescriptions(cm); + const match = (filename: string) => + cm.language.LanguageDescription.matchFilename(list, filename)?.name; + + expect(match('.bashrc')).toBe('Shell'); + expect(match('.zshrc')).toBe('Shell'); + expect(match('.envrc')).toBe('Shell'); + expect(match('foo.zsh')).toBe('Shell'); + expect(match('foo.bats')).toBe('Shell'); + expect(match('PKGBUILD')).toBe('Shell'); + + expect(match('.gitconfig')).toBe('Properties files'); + expect(match('.editorconfig')).toBe('Properties files'); + expect(match('.npmrc')).toBe('Properties files'); + expect(match('foo.cfg')).toBe('Properties files'); + expect(match('foo.conf')).toBe('Properties files'); + expect(match('nginx.conf')).toBe('Nginx'); + + expect(match('Cargo.lock')).toBe('TOML'); + expect(match('Pipfile')).toBe('TOML'); + expect(match('poetry.lock')).toBe('TOML'); + + expect(match('Containerfile')).toBe('Dockerfile'); + expect(match('Containerfile.test')).toBe('Dockerfile'); + expect(match('Dockerfile')).toBe('Dockerfile'); + expect(match('Dockerfile.dev')).toBe('Dockerfile'); + + expect(match('Brewfile')).toBe('Ruby'); + expect(match('Vagrantfile')).toBe('Ruby'); + expect(match('Gemfile')).toBe('Ruby'); + expect(match('foo.gemspec')).toBe('Ruby'); + expect(match('foo.rake')).toBe('Ruby'); + expect(match('foo.ru')).toBe('Ruby'); + + expect(match('foo.psgi')).toBe('Perl'); + expect(match('foo.pyi')).toBe('Python'); + expect(match('Snakefile')).toBe('Python'); + + expect(match('foo.webmanifest')).toBe('JSON'); + expect(match('foo.geojson')).toBe('JSON'); + expect(match('composer.lock')).toBe('JSON'); + expect(match('bun.lock')).toBe('JSON'); + + expect(match('foo.tcc')).toBe('C++'); + expect(match('foo.tpp')).toBe('C++'); + expect(match('foo.cppm')).toBe('C++'); + expect(match('foo.ixx')).toBe('C++'); + + expect(match('foo.xhtml')).toBe('HTML'); + expect(match('foo.jsh')).toBe('Java'); + expect(match('.Rprofile')).toBe('R'); + + expect(match('Makefile')).toBe('Makefile'); + expect(match('Makefile.am')).toBe('Makefile'); + expect(match('BSDmakefile')).toBe('Makefile'); + expect(match('GNUmakefile')).toBe('Makefile'); + expect(match('foo.mk')).toBe('Makefile'); + + expect(match('.env')).toBe('Dotenv'); + expect(match('.env.local')).toBe('Dotenv'); + + expect(match('foo.md')).toBe('Markdown'); + expect(match('foo.mdown')).toBe('Markdown'); + expect(match('foo.json5')).toBe('JSON5'); + expect(match('tsconfig.json')).toBe('JSON'); + + // Smoke tests for languages that already worked, to guard against regressions. + expect(match('foo.go')).toBe('Go'); + expect(match('foo.rs')).toBe('Rust'); + expect(match('foo.ts')).toBe('TypeScript'); + expect(match('foo.py')).toBe('Python'); + expect(match('foo.html')).toBe('HTML'); + expect(match('foo.css')).toBe('CSS'); + expect(match('foo.lua')).toBe('Lua'); + + // Genuinely ambiguous extensions left unhighlighted on purpose. + expect(match('foo.cgi')).toBeUndefined(); + expect(match('foo.fcgi')).toBeUndefined(); + expect(match('foo.fish')).toBeUndefined(); +}); diff --git a/web_src/js/modules/codeeditor/main.ts b/web_src/js/modules/codeeditor/main.ts index e703203038..be8992ce1d 100644 --- a/web_src/js/modules/codeeditor/main.ts +++ b/web_src/js/modules/codeeditor/main.ts @@ -44,7 +44,7 @@ export type CodemirrorEditor = { export type CodemirrorModules = Awaited>; -async function importCodemirror() { +export async function importCodemirror() { const [autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap] = await Promise.all([ import('@codemirror/autocomplete'), import('@codemirror/commands'), @@ -73,6 +73,48 @@ const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const filenameUnion = (filenames: string[]) => filenames.length ? new RegExp(`^(${filenames.map(escapeRegex).join('|')})$`) : undefined; +export function buildLanguageDescriptions(cm: CodemirrorModules): LanguageDescription[] { + const markdown = linguistLanguages.find((l) => l.name === 'Markdown'); + const dockerfile = linguistLanguages.find((l) => l.name === 'Dockerfile'); + const list: LanguageDescription[] = [ + ...buildBaseLanguages(cm), + cm.language.LanguageDescription.of({ + name: 'Markdown', extensions: markdown?.extensions ?? ['md', 'markdown', 'mkd'], + load: async () => (await import('@codemirror/lang-markdown')).markdown({codeLanguages: list}), + }), + cm.language.LanguageDescription.of({ + name: 'Dockerfile', extensions: dockerfile?.extensions ?? ['dockerfile', 'containerfile'], + filename: /^(Containerfile|Dockerfile)(\..+)?$/i, + load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/dockerfile')).dockerFile)), + }), + cm.language.LanguageDescription.of({ + name: 'Elixir', extensions: ['ex', 'exs'], + load: async () => (await import('codemirror-lang-elixir')).elixir(), + }), + cm.language.LanguageDescription.of({ + name: 'Nix', extensions: ['nix'], + load: async () => (await import('@replit/codemirror-lang-nix')).nix(), + }), + cm.language.LanguageDescription.of({ + name: 'Svelte', extensions: ['svelte'], + load: async () => (await import('@replit/codemirror-lang-svelte')).svelte(), + }), + cm.language.LanguageDescription.of({ + name: 'Makefile', extensions: ['mk', 'mak', 'make'], filename: /^(GNU|BSD)?[Mm]akefile(\..+)?$/, + load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)), + }), + cm.language.LanguageDescription.of({ + name: 'Dotenv', extensions: ['env'], filename: /^\.env(\..*)?$/, + load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)), + }), + cm.language.LanguageDescription.of({ + name: 'JSON5', extensions: ['json5', 'jsonc'], + load: async () => (await import('@codemirror/lang-json')).json(), + }), + ]; + return list; +} + let baseLanguagesCache: LanguageDescription[] | null = null; function buildBaseLanguages(cm: CodemirrorModules): LanguageDescription[] { if (baseLanguagesCache) return baseLanguagesCache; @@ -119,45 +161,7 @@ export async function createCodeEditor(textarea: HTMLTextAreaElement, filenameIn const previewableExts = new Set(config.previewableExtensions || []); const lineWrapExts = config.lineWrapExtensions || []; const cm = await importCodemirror(); - const markdown = linguistLanguages.find((l) => l.name === 'Markdown'); - const dockerfile = linguistLanguages.find((l) => l.name === 'Dockerfile'); - - const languageDescriptions: LanguageDescription[] = [ - ...buildBaseLanguages(cm), - cm.language.LanguageDescription.of({ - name: 'Markdown', extensions: markdown?.extensions ?? ['md', 'markdown', 'mkd'], - load: async () => (await import('@codemirror/lang-markdown')).markdown({codeLanguages: languageDescriptions}), - }), - cm.language.LanguageDescription.of({ - name: 'Dockerfile', extensions: dockerfile?.extensions ?? ['dockerfile', 'containerfile'], - filename: /^(Containerfile|Dockerfile)(\..+)?$/i, - load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/dockerfile')).dockerFile)), - }), - cm.language.LanguageDescription.of({ - name: 'Elixir', extensions: ['ex', 'exs'], - load: async () => (await import('codemirror-lang-elixir')).elixir(), - }), - cm.language.LanguageDescription.of({ - name: 'Nix', extensions: ['nix'], - load: async () => (await import('@replit/codemirror-lang-nix')).nix(), - }), - cm.language.LanguageDescription.of({ - name: 'Svelte', extensions: ['svelte'], - load: async () => (await import('@replit/codemirror-lang-svelte')).svelte(), - }), - cm.language.LanguageDescription.of({ - name: 'Makefile', extensions: ['mk', 'mak', 'make'], filename: /^(GNU|BSD)?[Mm]akefile(\..+)?$/, - load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)), - }), - cm.language.LanguageDescription.of({ - name: 'Dotenv', extensions: ['env'], filename: /^\.env(\..*)?$/, - load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)), - }), - cm.language.LanguageDescription.of({ - name: 'JSON5', extensions: ['json5', 'jsonc'], - load: async () => (await import('@codemirror/lang-json')).json(), - }), - ]; + const languageDescriptions = buildLanguageDescriptions(cm); const matchedLang = cm.language.LanguageDescription.matchFilename(languageDescriptions, config.filename); const container = document.createElement('div');