feat：增加分析进度接口

app/datafiles/readers.py

@@ -19,7 +19,15 @@ from python_calamine import (
     SheetVisibleEnum,
 )
 
-from app.datafiles.base import MAX_TABLE_ROWS, DataFileError, brief, rows_exceeded
+from app.datafiles.base import (
+    MAX_TABLE_ROWS,
+    DataFileError,
+    Reporter,
+    brief,
+    no_report,
+    rows_exceeded,
+    scaled,
+)
 from app.datafiles.columns import as_text
 from app.schemas.data_file import ParseWarning
 
@@ -32,6 +40,9 @@ JSON_FLATTEN_LEVELS = 2
 # JSONL 坏行超过这个比例时整个文件判为失败
 MAX_BAD_LINE_RATIO = 0.01
 
+# 逐行读取时每隔这么多行报告一次进度
+PROGRESS_ROWS = 20_000
+
 # 单元格最长的字符数；csv 模块默认 128KB，长文本单元格会报错
 csv.field_size_limit(16 * 1024 * 1024)
 
@@ -53,16 +64,21 @@ class Records:
     warnings: list[ParseWarning] = field(default_factory=list)
 
 
-def read_delimited(text: str, delimiter: str) -> Grid:
+def read_delimited(text: str, delimiter: str, report: Reporter = no_report) -> Grid:
     rows = []
-    for row in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter):
+    stream = io.StringIO(text, newline="")
+    for row in csv.reader(stream, delimiter=delimiter):
         rows.append(row)
         if len(rows) > MAX_TABLE_ROWS + ROW_SLACK:
             raise rows_exceeded(None)
+        if len(rows) % PROGRESS_ROWS == 0:
+            report(stream.tell() / len(text))
     return Grid(None, rows)
 
 
-def read_excel(path: Path) -> tuple[list[Grid], list[ParseWarning]]:
+def read_excel(
+    path: Path, report: Reporter = no_report
+) -> tuple[list[Grid], list[ParseWarning]]:
     try:
         book = CalamineWorkbook.from_path(str(path))
     except PasswordError:
@@ -71,7 +87,9 @@ def read_excel(path: Path) -> tuple[list[Grid], list[ParseWarning]]:
         raise DataFileError(422, "无法读取 Excel 文件，文件可能已损坏") from None
     grids, warnings = [], []
     try:
-        for meta in book.sheets_metadata:
+        sheets = book.sheets_metadata
+        for number, meta in enumerate(sheets):
+            report(number / len(sheets))
             # 图表页、宏表等没有单元格数据
             if meta.typ != SheetTypeEnum.WorkSheet:
                 continue
@@ -102,7 +120,7 @@ def read_excel(path: Path) -> tuple[list[Grid], list[ParseWarning]]:
     return grids, warnings
 
 
-def read_json(value: Any) -> Grid | Records:
+def read_json(value: Any, report: Reporter = no_report) -> Grid | Records:
     """对象数组按记录读取；数组的数组当作网格，第一行通常是表头"""
     warnings = []
     if isinstance(value, dict):
@@ -121,7 +139,7 @@ def read_json(value: Any) -> Grid | Records:
     if all(isinstance(item, list) for item in records):
         return Grid(None, [[_scalar(cell) for cell in row] for row in records])
     if all(isinstance(item, dict) for item in records):
-        result = _flatten_records(records)
+        result = _flatten_records(records, report)
         result.warnings = warnings
         return result
     if not any(isinstance(item, (dict, list)) for item in records):
@@ -129,9 +147,14 @@ def read_json(value: Any) -> Grid | Records:
     raise DataFileError(422, "JSON 数组里的元素类型不一致，需要都是对象或都是数组")
 
 
-def read_jsonl(text: str) -> Records:
+def read_jsonl(text: str, report: Reporter = no_report) -> Records:
     records, bad, total = [], [], 0
-    for number, line in enumerate(text.splitlines(), start=1):
+    lines = text.splitlines()
+    # 解析每行 JSON 和展开成列各占一半
+    parsing = scaled(report, 0.0, 0.5)
+    for number, line in enumerate(lines, start=1):
+        if number % PROGRESS_ROWS == 0:
+            parsing(number / len(lines))
         if not line.strip():
             continue
         total += 1
@@ -149,7 +172,7 @@ def read_jsonl(text: str) -> Records:
         raise DataFileError(
             422, f"JSONL 有 {len(bad)} 行无法解析（例如第 {bad[0]} 行），超过 1%"
         )
-    result = _flatten_records(records)
+    result = _flatten_records(records, scaled(report, 0.5, 1.0))
     if bad:
         result.warnings.append(
             ParseWarning(
@@ -203,11 +226,13 @@ def _is_record_list(value: Any) -> bool:
     )
 
 
-def _flatten_records(records: list[dict]) -> Records:
+def _flatten_records(records: list[dict], report: Reporter = no_report) -> Records:
     # 列按首次出现的顺序排列，记录里没有的字段为空
     index: dict[str, int] = {}
     columns: list[list[Any]] = []
     for row_number, record in enumerate(records):
+        if row_number % PROGRESS_ROWS == 0:
+            report(row_number / len(records))
         flat: dict[str, Any] = {}
         _flatten(record, "", 0, flat)
         for name, value in flat.items():