problem: refactor statement storage

packages/elastic/index.ts

@@ -5,7 +5,12 @@ import {
 } from 'hydrooj';
 
 const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign'];
-const processDocument = (doc: Partial<ProblemDoc>) => {
+const processDocument = async (pdoc: Partial<ProblemDoc>) => {
+    const doc = {
+        ...pdoc,
+        tag: pdoc.tag?.slice(),
+        content: (await ProblemModel.getStatements(pdoc as ProblemDoc)).map((entry) => entry.content).filter(Boolean).join('\n'),
+    };
     doc.content &&= doc.content.replace(/[[\]【】()（）]/g, ' ');
     doc.title &&= doc.title.replace(/[[\]【】()（）]/g, ' ')
         .replace(/([a-zA-Z]{2,})(\d+)/, '$1$2 $1 $2');
@@ -83,7 +88,7 @@ export default class ElasticSearchService extends Service {
             await this.client.index({
                 index: 'problem',
                 id: `${pdoc.domainId}/${pdoc.docId}`,
-                document: processDocument(pdoc),
+                document: await processDocument(pdoc),
             });
         };
         if (domainId) await iterateAllProblemInDomain(domainId, ProblemModel.PROJECTION_PUBLIC, cb);
@@ -94,25 +99,32 @@ export default class ElasticSearchService extends Service {
 
     async [Service.init]() {
         await this.client.nodes.stats();
-        this.ctx.on('problem/add', async (doc, docId) => {
-            await this.client.index({
-                index: 'problem',
-                id: `${doc.domainId}/${docId}`,
-                document: processDocument(doc),
-            });
-        });
-        this.ctx.on('problem/edit', async (pdoc) => {
-            await this.client.index({
-                index: 'problem',
-                id: `${pdoc.domainId}/${pdoc.docId}`,
-                document: processDocument(pdoc),
-            });
+        const indexProblem = async (pdoc: Partial<ProblemDoc>, docId = pdoc.docId) => {
+            try {
+                await this.client.index({
+                    index: 'problem',
+                    id: `${pdoc.domainId}/${docId}`,
+                    document: await processDocument({ ...pdoc, docId }),
+                });
+            } catch (e) {
+                this.ctx.logger.error(e);
+            }
+        };
+        this.ctx.on('problem/add', indexProblem);
+        this.ctx.on('problem/edit', indexProblem);
+        this.ctx.on('problem/setStatements', async (domainId, docId) => {
+            try {
+                const pdoc = await ProblemModel.get(domainId, docId);
+                if (pdoc) await indexProblem(pdoc);
+            } catch (e) {
+                this.ctx.logger.error(e);
+            }
         });
         this.ctx.on('problem/delete', async (domainId, docId) => {
             await this.client.delete({
                 index: 'problem',
                 id: `${domainId}/${docId}`,
-            });
+            }).catch((e) => this.ctx.logger.error(e));
         });
         this.ctx.addScript(
             'ensureElasticSearch', 'Elastic problem search re-index',

packages/fps-importer/index.ts

@@ -39,9 +39,9 @@ class FpsProblemImportHandler extends Handler {
             }
             const title = decodeHTML(p.title.join(' '));
             const tags = _.filter(p.source, (i: string) => i.trim()).flatMap((i) => i.split(' ')).filter((i) => i);
-            const pid = await ProblemModel.add(domainId, null, title, content, this.user._id, tags);
+            const lang = SystemModel.get('server.language') || 'zh';
+            const pid = await ProblemModel.add(domainId, null, title, '', this.user._id, tags);
             const tasks: Promise<any>[] = [ProblemModel.addTestdata(domainId, pid, 'config.yaml', Buffer.from(yaml.dump(config)))];
-            if (!markdown) tasks.push(ProblemModel.edit(domainId, pid, { html: true }));
             const addTestdata = (node: any, index: string, ext: string) => {
                 if (!node || !['object', 'string'].includes(typeof node)) return; // Ignore file not exist
                 const id = node.$?.name || `${index}`;
@@ -69,9 +69,11 @@ class FpsProblemImportHandler extends Handler {
                     tasks.push(ProblemModel.addAdditionalFile(domainId, pid, filename, Buffer.from(img.base64[0], 'base64')));
                     content = content.replace(img.src[0], `file://${filename}`);
                 }
-                await ProblemModel.edit(domainId, pid, { content });
             }
             await Promise.all(tasks);
+            await ProblemModel.edit(domainId, pid, {
+                content: JSON.stringify([{ name: `problem_${lang}.${markdown ? 'md' : 'html'}`, lang, content }]),
+            });
             if (p.solution) {
                 let s = '';
                 for (const sol of p.solution) {

packages/hydrooj/src/handler/problem.ts

@@ -22,6 +22,7 @@ import {
 import {
     ProblemDoc, ProblemSearchOptions, ProblemStatusDoc, RecordDoc, User,
 } from '../interface';
+import { normalizeStatements, parseSubmittedStatements } from '../lib/content';
 import { PERM, PRIV, STATUS } from '../model/builtin';
 import * as contest from '../model/contest';
 import * as discussion from '../model/discussion';
@@ -379,11 +380,13 @@ export class ProblemDetailHandler extends ContestDetailBaseHandler {
     @query('tid', Types.ObjectId, true)
     @query('pjax', Types.Boolean)
     async get(...args: any[]) {
+        const statements = await problem.getStatements(this.pdoc);
         // Navigate to current additional file download
         // e.g. ![img](file://a.jpg) will navigate to ![img](./pid/file/a.jpg)
         if (!this.request.json || args[2]) {
-            this.response.body.pdoc.content = this.response.body.pdoc.content
-                .replace(/file:\/\/([^ \n)\\"]+)/g, (str: string) => {
+            for (const s of statements) {
+                if (!s.content) continue;
+                s.content = s.content.replace(/file:\/\/([^ \n)\\"]+)/g, (str: string) => {
                     const info = str.match(/file:\/\/([^ \n)\\"]+)/);
                     const fileinfo = info[1];
                     let filename = fileinfo.split('?')[0]; // remove querystring
@@ -394,14 +397,34 @@ export class ProblemDetailHandler extends ContestDetailBaseHandler {
                     if (!args[1]) return `./${this.pdoc.docId}/file/${fileinfo}`;
                     return `./${this.pdoc.docId}/file/${fileinfo}${fileinfo.includes('?') ? '&' : '?'}tid=${args[1]}`;
                 });
+            }
+        }
+        const stmtLangs = [...new Set(statements.map((s) => s.lang).filter(Boolean))];
+        if (!stmtLangs.length) {
+            let content: any;
+            try {
+                content = JSON.parse(this.pdoc.content);
+            } catch (e) { }
+            if (content && typeof content === 'object') stmtLangs.push(...Object.keys(content));
         }
+        const preferredLang = this.request.query.lang || this.user.viewLang || this.session.viewLang;
+        const language = preferredLang || stmtLangs[0];
+        let candidates = statements.filter((s) => s.lang === language);
+        if (!candidates.length) candidates = statements.filter((s) => s.lang?.startsWith(language));
+        const activeLang = (candidates[0] || statements[0])?.lang || language;
+        candidates = statements.filter((s) => s.lang === activeLang);
+        if (!candidates.length) candidates = statements.filter((s) => s.lang?.startsWith(activeLang));
+        const stmtFiles = (candidates.length ? candidates : statements).map((s) => s.name);
+        const data = {
+            pdoc: this.pdoc, tdoc: this.tdoc, statements, stmtLangs, preferredLang, activeLang, stmtFiles,
+        };
+        Object.assign(this.response.body, data);
         this.response.body.page_name = this.tdoc
             ? this.tdoc.rule === 'homework'
                 ? 'homework_detail_problem'
                 : 'contest_detail_problem'
             : 'problem_detail';
         if (args[2]) {
-            const data = { pdoc: this.pdoc, tdoc: this.tdoc };
             this.response.body = {
                 title: this.renderTitle(this.response.body.page_name),
                 fragments: [
@@ -621,6 +644,12 @@ export class ProblemManageHandler extends ProblemDetailHandler {
 export class ProblemEditHandler extends ProblemManageHandler {
     async get() {
         this.response.body.additional_file = sortFiles(this.pdoc.additional_file || []);
+        // strip row metadata and compile artifacts: the edit form round-trips these
+        this.response.body.statementsJSON = JSON.stringify(
+            (await problem.getStatements(this.pdoc)).map((
+                { _id, domainId, docType, docId, render, ...entry },
+            ) => entry),
+        );
         this.response.body.statementLangs = this.ctx.i18n.langs(false);
         this.response.template = 'problem_edit.html';
     }
@@ -638,8 +667,9 @@ export class ProblemEditHandler extends ProblemManageHandler {
     ) {
         if (typeof newPid !== 'string') newPid = `P${newPid}`;
         if (newPid !== this.pdoc.pid && await problem.get(domainId, newPid)) throw new ProblemAlreadyExistError(newPid);
+        parseSubmittedStatements(content, (this.pdoc.additional_file || []).map((i) => i.name));
         const $update: Partial<ProblemDoc> = {
-            title, content, pid: newPid, hidden, tag: tag ?? [], difficulty, html: false,
+            title, content, pid: newPid, hidden, tag: tag ?? [], difficulty,
         };
         const pdoc = await problem.edit(domainId, this.pdoc.docId, $update);
         this.response.redirect = this.url('problem_detail', { pid: newPid || pdoc.docId });
@@ -996,12 +1026,13 @@ export class ProblemStatisticsHandler extends ProblemDetailHandler {
 
 export class ProblemCreateHandler extends Handler {
     async get() {
-        this.response.body.statementLangs = this.ctx.i18n.langs(false);
-        this.response.template = 'problem_edit.html';
         this.response.body = {
             page_name: 'problem_create',
             additional_file: [],
+            statementsJSON: '[]',
+            statementLangs: this.ctx.i18n.langs(false),
         };
+        this.response.template = 'problem_edit.html';
     }
 
     @post('title', Types.Title)
@@ -1016,8 +1047,11 @@ export class ProblemCreateHandler extends Handler {
     ) {
         if (typeof pid !== 'string') pid = `P${pid}`;
         if (pid && await problem.get(domainId, pid)) throw new ProblemAlreadyExistError(pid);
-        const docId = await problem.add(domainId, pid, title, content, this.user._id, tag ?? [], { hidden, difficulty });
+        const entries = parseSubmittedStatements(content, this.user._files.map((i) => i.name))
+            ?? normalizeStatements(content, system.get('server.language') || 'zh');
+        const docId = await problem.add(domainId, pid, title, '', this.user._id, tag ?? [], { hidden, difficulty });
         const files = new Set(Array.from(content.matchAll(/file:\/\/([\w-]+\.[a-zA-Z0-9]+)/g)).map((i) => i[1]));
+        for (const entry of entries) if (entry.file) files.add(entry.file);
         const tasks = [];
         for (const file of files) {
             if (this.user._files.find((i) => i.name === file)) {
@@ -1029,6 +1063,7 @@ export class ProblemCreateHandler extends Handler {
             }
         }
         await Promise.all(tasks);
+        await problem.setStatements(domainId, docId, entries);
         this.response.body = { pid: pid || docId };
         this.response.redirect = this.url('problem_files', { pid: pid || docId });
     }
@@ -1044,6 +1079,12 @@ export const ProblemApi = {
             const pdoc = await problem.get(args.domainId, args.id);
             if (!pdoc) return null;
             if (pdoc.hidden) ctx.checkPerm(PERM.PERM_VIEW_PROBLEM_HIDDEN);
+            (pdoc as any).statements = await problem.getStatements(pdoc);
+            if (pdoc.reference) {
+                const source = await problem.get(pdoc.reference.domainId, pdoc.reference.pid, ['additional_file']);
+                if (!source) throw new ProblemNotFoundError(pdoc.reference.domainId, pdoc.reference.pid);
+                pdoc.additional_file = source.additional_file;
+            }
             return pdoc;
         },
     ),

packages/hydrooj/src/interface.ts

@@ -182,6 +182,7 @@ declare module './model/problem' {
         data: FileInfo[];
         additional_file: FileInfo[];
         hidden?: boolean;
+        /** @deprecated 仅兼容读取旧题面，格式由题面文件扩展名决定。 */
         html?: boolean;
         stats?: any;
         difficulty?: number;
@@ -623,6 +624,7 @@ declare module './service/db' {
         'record.stat': RecordStatDoc;
         'record.history': RecordHistoryDoc;
         document: any;
+        'problem.statement': import('./model/problem').ProblemStatementDoc;
         'document.status': StatusDocBase & {
             [K in keyof DocStatusType]: { docType: K } & DocStatusType[K];
         }[keyof DocStatusType];
@@ -700,6 +702,7 @@ export interface UI {
 export interface ModuleInterfaces {
     hash: (password: string, salt: string, user: User) => boolean | string | Promise<string>;
     problemSearch: ProblemSearch;
+    contentRenderer: import('./lib/content').Renderer;
 }
 
 export interface HydroGlobal {

packages/hydrooj/src/lib/content.ts

@@ -1,3 +1,99 @@
+import { ValidationError } from '../error';
+import type { ProblemDoc } from '../interface';
+
+export interface StatementEntry {
+    name: string;
+    lang: string;
+    content?: string;
+    file?: string;
+    render?: any;
+}
+
+export interface RenderCtx {
+    pdoc: ProblemDoc;
+    tid?: string;
+}
+
+export interface CompileCtx {
+    readAdditionalFile: (name: string) => Promise<Buffer>;
+}
+
+export interface Renderer {
+    render(entry: StatementEntry, ctx: RenderCtx): string;
+    compile?(entry: StatementEntry, ctx: CompileCtx): Promise<any>;
+}
+
+// statement name: safe filename with extension, no path separators
+const NAME_RE = /^\w[\w .()-]*\.[A-Za-z0-9]{1,16}$/;
+
+/**
+ * Parse content field into statement entries.
+ * - Statement list `[{name, lang, content|file}]` → entries as-is
+ * - Legacy `{lang: content}` → normalized entries (name `${lang}.${md|html}`)
+ * - Plain string / malformed → null (caller renders legacy)
+ */
+export function parseStatements(content: string, html = false): StatementEntry[] | null {
+    if (!content || typeof content !== 'string') return null;
+    let parsed: any;
+    try {
+        parsed = JSON.parse(content);
+    } catch {
+        return null;
+    }
+    if (Array.isArray(parsed)) {
+        for (const value of parsed) {
+            if (typeof value !== 'object' || value === null
+                || typeof value.name !== 'string' || typeof value.lang !== 'string') return null;
+        }
+        return parsed;
+    }
+    if (typeof parsed !== 'object' || parsed === null) return null;
+    const entries: StatementEntry[] = [];
+    for (const [lang, value] of Object.entries<any>(parsed)) {
+        if (typeof value !== 'string') return null;
+        entries.push({ name: `${lang}.${html ? 'html' : 'md'}`, lang, content: value });
+    }
+    return entries;
+}
+
+export function normalizeStatements(content: string, lang: string, html = false): StatementEntry[] {
+    if (!content) return [];
+    return parseStatements(content, html) ?? [{ name: `${lang}.${html ? 'html' : 'md'}`, lang, content }];
+}
+
+/**
+ * Parse user-submitted content (edit/create handlers).
+ * Plain strings are legacy submissions → null (caller decides whether to allow).
+ * A JSON array must be a valid statement list; malformed entries throw
+ * instead of silently falling back to legacy rendering.
+ */
+export function parseSubmittedStatements(content: string, availableFiles: string[]): StatementEntry[] | null {
+    if (!content || typeof content !== 'string' || !content.trim().startsWith('[')) return null;
+    let parsed: any;
+    try {
+        parsed = JSON.parse(content);
+    } catch {
+        return null;
+    }
+    if (!Array.isArray(parsed)) return null;
+    const names = new Set<string>();
+    for (const entry of parsed) {
+        if (typeof entry !== 'object' || entry === null
+            || typeof entry.name !== 'string' || typeof entry.lang !== 'string') {
+            throw new ValidationError('statement.entry');
+        }
+        if (!NAME_RE.test(entry.name)) throw new ValidationError('statement.name', null, entry.name);
+        if (names.has(entry.name)) throw new ValidationError('statement.duplicate', null, entry.name);
+        names.add(entry.name);
+        if (!entry.lang) throw new ValidationError('statement.lang', null, entry.name);
+        const hasContent = typeof entry.content === 'string' && entry.content.length > 0;
+        const hasFile = typeof entry.file === 'string' && entry.file.length > 0;
+        if (!hasContent && !hasFile) throw new ValidationError('statement.empty', null, entry.name);
+        if (hasFile && !availableFiles.includes(entry.file)) throw new ValidationError('statement.file', null, entry.file);
+    }
+    return parsed;
+}
+
 export interface ProblemSource {
     background?: string;
     description?: string;