Merge 1bff063446e9a956b9e10ff6a47ce7fa33aa0763 into a5d81d9ce230aaa6e1021b6236ca01cb6d2b56c3

Makefile

@@ -661,6 +661,10 @@ generate-gitignore: ## update gitignore files
 generate-images: | node_modules ## generate images
 	cd tools && node generate-images.ts $(TAGS)
 
+.PHONY: generate-codemirror-languages
+generate-codemirror-languages: | node_modules ## generate codemirror-languages.json
+	node tools/generate-codemirror-languages.ts
+
 .PHONY: generate-manpage
 generate-manpage: ## generate manpage
 	@[ -f gitea ] || make backend

eslint.config.ts

@@ -570,8 +570,6 @@ export default defineConfig([
       'no-redeclare': [0], // must be disabled for typescript overloads
       'no-regex-spaces': [2],
       'no-restricted-exports': [0],
-      'no-restricted-globals': [2, ...restrictedGlobals],
-      'no-restricted-properties': [2, ...restrictedProperties],
       'no-restricted-imports': [2, {paths: [
         {name: 'jquery', message: 'Use the global $ instead', allowTypeImports: true},
       ]}],
@@ -1022,5 +1020,9 @@ export default defineConfig([
   {
     files: ['web_src/**/*'],
     languageOptions: {globals: {...globals.browser, ...globals.jquery}},
+    rules: {
+      'no-restricted-globals': [2, ...restrictedGlobals],
+      'no-restricted-properties': [2, ...restrictedProperties],
+    },
   },
 ]);

tools/generate-codemirror-languages.ts

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+import {load as parseYaml} from 'js-yaml';
+import {writeFile} from 'node:fs/promises';
+import {languages as cmLanguages} from '@codemirror/language-data';
+
+const linguistUrl = 'https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml';
+
+// Linguist names that don't match the corresponding @codemirror/language-data name.
+const renames: Record<string, string> = {
+  'COBOL': 'Cobol',
+  'Diff': 'diff',
+  'INI': 'Properties files',
+  'Less': 'LESS',
+  'Protocol Buffer': 'ProtoBuf',
+  'TeX': 'LaTeX',
+};
+
+// Per-language extensions to drop. Use only for extensions that would actively collide
+// with another language (e.g. .inc claimed by both PHP and C++) or where the syntax is
+// genuinely incompatible with the CodeMirror mode (e.g. .csh vs sh).
+const excludeExt: Record<string, string[]> = {
+  'C++': ['inc'],
+  'INI': ['frm'],
+  'JavaScript': ['_js', 'bones', 'es', 'es6', 'frag', 'gs', 'jake', 'javascript', 'jsb', 'jscad', 'jsfl', 'jslib', 'jsm', 'jspre', 'jss', 'njs', 'pac', 'sjs', 'ssjs', 'xsjs', 'xsjslib'],
+  'Lua': ['fcgi'],
+  'PHP': ['fcgi', 'inc'],
+  'Perl': ['cgi', 'fcgi'],
+  'Python': ['cgi', 'fcgi', 'spec'],
+  'Ruby': ['fcgi', 'spec'],
+  'Shell': ['cgi', 'csh', 'fcgi'],
+  'XML': ['inc', 'jsproj', 'tmpl', 'ts', 'tsx'],
+};
+
+type LinguistEntry = {
+  type: string;
+  extensions?: string[];
+  filenames?: string[];
+};
+
+type CmLanguage = {
+  name: string;
+  extensions: string[];
+  filenames: string[];
+};
+
+async function main() {
+  const res = await fetch(linguistUrl);
+  if (!res.ok) throw new Error(`fetch ${linguistUrl} failed: ${res.status}`);
+  const linguist = parseYaml(await res.text()) as Record<string, LinguistEntry>;
+
+  const cmNames = new Set(cmLanguages.map((l) => l.name));
+  const out: CmLanguage[] = [];
+  for (const [linguistName, entry] of Object.entries(linguist)) {
+    const cmName = renames[linguistName] ?? linguistName;
+    if (!cmNames.has(cmName)) continue;
+    const exExt = new Set(excludeExt[linguistName]);
+    // CodeMirror's matchFilename uses /\.([^.]+)$/ to extract the suffix, so multi-dot
+    // extensions like ".cmake.in" cannot match as extensions and are dropped here.
+    const extensions = (entry.extensions ?? [])
+      .map((e) => e.replace(/^\./, ''))
+      .filter((e) => !e.includes('.') && !exExt.has(e));
+    const filenames = entry.filenames ?? [];
+    out.push({
+      name: cmName,
+      extensions: Array.from(new Set(extensions)),
+      filenames: Array.from(new Set(filenames)),
+    });
+  }
+
+  out.sort((a, b) => a.name.localeCompare(b.name));
+
+  const outPath = new URL('../assets/codemirror-languages.json', import.meta.url);
+  await writeFile(outPath, `${JSON.stringify(out, null, 2)}\n`);
+  console.info(`wrote ${out.length} languages to ${outPath.pathname}`);
+}
+
+try {
+  await main();
+} catch (err) {
+  console.error(err);
+  process.exit(1);
+}

web_src/js/modules/codeeditor/main.test.ts

@@ -0,0 +1,84 @@
+import {buildLanguageDescriptions, importCodemirror} from './main.ts';
+
+test('matchFilename — language detection covers extended rules', async () => {
+  const cm = await importCodemirror();
+  const list = buildLanguageDescriptions(cm);
+  const match = (filename: string) =>
+    cm.language.LanguageDescription.matchFilename(list, filename)?.name;
+
+  expect(match('.bashrc')).toBe('Shell');
+  expect(match('.zshrc')).toBe('Shell');
+  expect(match('.envrc')).toBe('Shell');
+  expect(match('foo.zsh')).toBe('Shell');
+  expect(match('foo.bats')).toBe('Shell');
+  expect(match('PKGBUILD')).toBe('Shell');
+
+  expect(match('.gitconfig')).toBe('Properties files');
+  expect(match('.editorconfig')).toBe('Properties files');
+  expect(match('.npmrc')).toBe('Properties files');
+  expect(match('foo.cfg')).toBe('Properties files');
+  expect(match('foo.conf')).toBe('Properties files');
+  expect(match('nginx.conf')).toBe('Nginx');
+
+  expect(match('Cargo.lock')).toBe('TOML');
+  expect(match('Pipfile')).toBe('TOML');
+  expect(match('poetry.lock')).toBe('TOML');
+
+  expect(match('Containerfile')).toBe('Dockerfile');
+  expect(match('Containerfile.test')).toBe('Dockerfile');
+  expect(match('Dockerfile')).toBe('Dockerfile');
+  expect(match('Dockerfile.dev')).toBe('Dockerfile');
+
+  expect(match('Brewfile')).toBe('Ruby');
+  expect(match('Vagrantfile')).toBe('Ruby');
+  expect(match('Gemfile')).toBe('Ruby');
+  expect(match('foo.gemspec')).toBe('Ruby');
+  expect(match('foo.rake')).toBe('Ruby');
+  expect(match('foo.ru')).toBe('Ruby');
+
+  expect(match('foo.psgi')).toBe('Perl');
+  expect(match('foo.pyi')).toBe('Python');
+  expect(match('Snakefile')).toBe('Python');
+
+  expect(match('foo.webmanifest')).toBe('JSON');
+  expect(match('foo.geojson')).toBe('JSON');
+  expect(match('composer.lock')).toBe('JSON');
+  expect(match('bun.lock')).toBe('JSON');
+
+  expect(match('foo.tcc')).toBe('C++');
+  expect(match('foo.tpp')).toBe('C++');
+  expect(match('foo.cppm')).toBe('C++');
+  expect(match('foo.ixx')).toBe('C++');
+
+  expect(match('foo.xhtml')).toBe('HTML');
+  expect(match('foo.jsh')).toBe('Java');
+  expect(match('.Rprofile')).toBe('R');
+
+  expect(match('Makefile')).toBe('Makefile');
+  expect(match('Makefile.am')).toBe('Makefile');
+  expect(match('BSDmakefile')).toBe('Makefile');
+  expect(match('GNUmakefile')).toBe('Makefile');
+  expect(match('foo.mk')).toBe('Makefile');
+
+  expect(match('.env')).toBe('Dotenv');
+  expect(match('.env.local')).toBe('Dotenv');
+
+  expect(match('foo.md')).toBe('Markdown');
+  expect(match('foo.mdown')).toBe('Markdown');
+  expect(match('foo.json5')).toBe('JSON5');
+  expect(match('tsconfig.json')).toBe('JSON');
+
+  // Smoke tests for languages that already worked, to guard against regressions.
+  expect(match('foo.go')).toBe('Go');
+  expect(match('foo.rs')).toBe('Rust');
+  expect(match('foo.ts')).toBe('TypeScript');
+  expect(match('foo.py')).toBe('Python');
+  expect(match('foo.html')).toBe('HTML');
+  expect(match('foo.css')).toBe('CSS');
+  expect(match('foo.lua')).toBe('Lua');
+
+  // Genuinely ambiguous extensions left unhighlighted on purpose.
+  expect(match('foo.cgi')).toBeUndefined();
+  expect(match('foo.fcgi')).toBeUndefined();
+  expect(match('foo.fish')).toBeUndefined();
+});

web_src/js/modules/codeeditor/main.ts

@@ -7,6 +7,7 @@ import type {PaletteCommand} from './command-palette.ts';
 import {contextMenu, collectSymbols, selectAllOccurrences} from './context-menu.ts';
 import {createJsonLinter, createSyntaxErrorLinter} from './linter.ts';
 import {clickableUrls, goToDefinitionAt, trimTrailingWhitespaceFromView} from './utils.ts';
+import linguistLanguages from '../../../../assets/codemirror-languages.json' with {type: 'json'};
 import type {LanguageDescription, LanguageSupport} from '@codemirror/language';
 import type {Compartment, Extension} from '@codemirror/state';
 import type {EditorView, ViewUpdate} from '@codemirror/view';
@@ -43,7 +44,7 @@ export type CodemirrorEditor = {
 
 export type CodemirrorModules = Awaited<ReturnType<typeof importCodemirror>>;
 
-async function importCodemirror() {
+export async function importCodemirror() {
   const [autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap] = await Promise.all([
     import('@codemirror/autocomplete'),
     import('@codemirror/commands'),
@@ -60,6 +61,81 @@ async function importCodemirror() {
   return {autocomplete, commands, language, languageData, lint, search, state, view, highlight, indentMarkers, vscodeKeymap};
 }
 
+const manualFilenames: Record<string, string[]> = {
+  'Properties files': ['.editorconfig', '.gitconfig', '.npmrc'],
+  'Python': ['Snakefile'],
+};
+const manualExtensions: Record<string, string[]> = {
+  'Properties files': ['conf'],
+};
+const handledByCustomEntry = new Set(['Dockerfile', 'Markdown']);
+const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const filenameUnion = (filenames: string[]) =>
+  filenames.length ? new RegExp(`^(${filenames.map(escapeRegex).join('|')})$`) : undefined;
+
+export function buildLanguageDescriptions(cm: CodemirrorModules): LanguageDescription[] {
+  const markdown = linguistLanguages.find((l) => l.name === 'Markdown');
+  const dockerfile = linguistLanguages.find((l) => l.name === 'Dockerfile');
+  const list: LanguageDescription[] = [
+    ...buildBaseLanguages(cm),
+    cm.language.LanguageDescription.of({
+      name: 'Markdown', extensions: markdown?.extensions ?? ['md', 'markdown', 'mkd'],
+      load: async () => (await import('@codemirror/lang-markdown')).markdown({codeLanguages: list}),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'Dockerfile', extensions: dockerfile?.extensions ?? ['dockerfile', 'containerfile'],
+      filename: /^(Containerfile|Dockerfile)(\..+)?$/i,
+      load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/dockerfile')).dockerFile)),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'Elixir', extensions: ['ex', 'exs'],
+      load: async () => (await import('codemirror-lang-elixir')).elixir(),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'Nix', extensions: ['nix'],
+      load: async () => (await import('@replit/codemirror-lang-nix')).nix(),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'Svelte', extensions: ['svelte'],
+      load: async () => (await import('@replit/codemirror-lang-svelte')).svelte(),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'Makefile', extensions: ['mk', 'mak', 'make'], filename: /^(GNU|BSD)?[Mm]akefile(\..+)?$/,
+      load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'Dotenv', extensions: ['env'], filename: /^\.env(\..*)?$/,
+      load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)),
+    }),
+    cm.language.LanguageDescription.of({
+      name: 'JSON5', extensions: ['json5', 'jsonc'],
+      load: async () => (await import('@codemirror/lang-json')).json(),
+    }),
+  ];
+  return list;
+}
+
+let baseLanguagesCache: LanguageDescription[] | null = null;
+function buildBaseLanguages(cm: CodemirrorModules): LanguageDescription[] {
+  if (baseLanguagesCache) return baseLanguagesCache;
+  const loadByName = new Map<string, LanguageDescription['load']>(
+    cm.languageData.languages.map((l: LanguageDescription) => [l.name, l.load.bind(l)]),
+  );
+  const overrides = linguistLanguages
+    .filter((l) => loadByName.has(l.name) && !handledByCustomEntry.has(l.name))
+    .map((l) => cm.language.LanguageDescription.of({
+      name: l.name,
+      extensions: [...l.extensions, ...(manualExtensions[l.name] ?? [])],
+      filename: filenameUnion([...l.filenames, ...(manualFilenames[l.name] ?? [])]),
+      load: loadByName.get(l.name)!,
+    }));
+  const overrideNames = new Set(overrides.map((o) => o.name));
+  const fallback = cm.languageData.languages.filter(
+    (l: LanguageDescription) => !overrideNames.has(l.name) && !handledByCustomEntry.has(l.name),
+  );
+  return baseLanguagesCache = [...overrides, ...fallback];
+}
+
 function togglePreviewDisplay(previewable: boolean): void {
   // FIXME: here and below, the selector is too broad, it should only query in the editor related scope
   const previewTab = document.querySelector<HTMLElement>('a[data-tab="preview"]');
@@ -85,38 +161,7 @@ export async function createCodeEditor(textarea: HTMLTextAreaElement, filenameIn
   const previewableExts = new Set(config.previewableExtensions || []);
   const lineWrapExts = config.lineWrapExtensions || [];
   const cm = await importCodemirror();
-
-  const languageDescriptions: LanguageDescription[] = [
-    ...cm.languageData.languages.filter((l: LanguageDescription) => l.name !== 'Markdown'),
-    cm.language.LanguageDescription.of({
-      name: 'Markdown', extensions: ['md', 'markdown', 'mkd'],
-      load: async () => (await import('@codemirror/lang-markdown')).markdown({codeLanguages: languageDescriptions}),
-    }),
-    cm.language.LanguageDescription.of({
-      name: 'Elixir', extensions: ['ex', 'exs'],
-      load: async () => (await import('codemirror-lang-elixir')).elixir(),
-    }),
-    cm.language.LanguageDescription.of({
-      name: 'Nix', extensions: ['nix'],
-      load: async () => (await import('@replit/codemirror-lang-nix')).nix(),
-    }),
-    cm.language.LanguageDescription.of({
-      name: 'Svelte', extensions: ['svelte'],
-      load: async () => (await import('@replit/codemirror-lang-svelte')).svelte(),
-    }),
-    cm.language.LanguageDescription.of({
-      name: 'Makefile', filename: /^(GNUm|M|m)akefile$/,
-      load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)),
-    }),
-    cm.language.LanguageDescription.of({
-      name: 'Dotenv', extensions: ['env'], filename: /^\.env(\..*)?$/,
-      load: async () => new cm.language.LanguageSupport(cm.language.StreamLanguage.define((await import('@codemirror/legacy-modes/mode/shell')).shell)),
-    }),
-    cm.language.LanguageDescription.of({
-      name: 'JSON5', extensions: ['json5', 'jsonc'],
-      load: async () => (await import('@codemirror/lang-json')).json(),
-    }),
-  ];
+  const languageDescriptions = buildLanguageDescriptions(cm);
   const matchedLang = cm.language.LanguageDescription.matchFilename(languageDescriptions, config.filename);
 
   const container = document.createElement('div');