feat：完成大部分数据文件上传解析功能

2026-10-08 08:23:01 -07:00
parent 05c050d2ba
commit bb6a9405db
27 changed files with 2675 additions and 18 deletions
@@ -0,0 +1,290 @@
+"""在原始网格里识别表：按空行切块，找表头，合并两级表头，去掉标题、说明和合计行
+
+只用规则；判断不了的地方照常产出表，同时记一条警告，调用方据此提示用户
+"""
+
+import math
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from app.datafiles.base import MAX_TABLE_ROWS, rows_exceeded, where
+from app.datafiles.columns import classify, label_text
+from app.datafiles.readers import Grid
+from app.schemas.data_file import ParseWarning
+
+# 只在每块的前几行里找表头
+HEADER_SEARCH_ROWS = 10
+
+# 首个或第二个非空单元格是这些词（去掉空白和冒号后）的行是合计行
+TOTAL_LABELS = {
+    "合计",
+    "总计",
+    "小计",
+    "共计",
+    "总和",
+    "平均",
+    "平均值",
+    "total",
+    "subtotal",
+    "grandtotal",
+    "sum",
+    "average",
+}
+# 例如"华东小计""一季度合计"
+TOTAL_SUFFIXES = ("小计", "合计", "总计")
+
+# 警告里引用的单元格文本最长显示这么多字
+QUOTE_CHARS = 30
+
+
+@dataclass
+class RawTable:
+    sheet: str | None
+    # 同一 sheet 切出多张表时的序号，从 1 开始；只有一张表时为空
+    part: int | None
+    header: list[Any]
+    # 数据行，宽度与表头相同
+    rows: list[list[Any]]
+    warnings: list[ParseWarning] = field(default_factory=list)
+
+
+@dataclass
+class _Piece:
+    columns: list[int]
+    # 表头所在行在网格中的下标
+    header_row: int
+    header: list[Any]
+    # 数据行在网格中的下标
+    data: list[int]
+    warnings: list[ParseWarning]
+
+
+def split_tables(grid: Grid) -> tuple[list[RawTable], list[ParseWarning]]:
+    """返回识别出的表和不属于任何表的警告（被丢弃的块）"""
+    rows = _normalize(grid.rows)
+    sheet = grid.sheet
+    pieces: list[_Piece] = []
+    dropped: list[ParseWarning] = []
+    for block in _blocks(rows):
+        columns = [
+            c for c in range(len(rows[0])) if any(rows[r][c] is not None for r in block)
+        ]
+        last = pieces[-1] if pieces else None
+        # 表中间夹着空行时，后面的块以数据行开头、列也落在上一张表内，接在上一张表后面
+        if (
+            last
+            and not _is_label_row([rows[block[0]][c] for c in columns])
+            and set(columns) <= set(last.columns)
+        ):
+            last.data.extend(block)
+            continue
+        if len(block) < 2:
+            dropped.append(
+                ParseWarning(
+                    code="block_dropped",
+                    message=f"{where(sheet)}第 {block[0] + 1} 行的{_quote(rows[block[0]])}不是表格，已跳过",
+                )
+            )
+            continue
+        pieces.append(_read_block(rows, block, columns, sheet))
+
+    tables = []
+    for piece in pieces:
+        table = _finish(rows, piece, sheet)
+        if table is None:
+            dropped.append(
+                ParseWarning(
+                    code="block_dropped",
+                    message=f"{where(sheet)}第 {piece.header_row + 1} 行开始的表只有表头没有数据，已跳过",
+                )
+            )
+        else:
+            tables.append(table)
+    if len(tables) > 1:
+        for part, table in enumerate(tables, start=1):
+            table.part = part
+    return tables, dropped
+
+
+def _normalize(grid: list[list[Any]]) -> list[list[Any]]:
+    """去掉字符串两端空白，空字符串记为 None，补齐成同样宽度"""
+    width = max((len(row) for row in grid), default=0)
+    rows = []
+    for row in grid:
+        cells = [_cell(value) for value in row]
+        cells.extend([None] * (width - len(cells)))
+        rows.append(cells)
+    return rows
+
+
+def _cell(value: Any) -> Any:
+    if isinstance(value, str):
+        value = value.strip()
+        return value or None
+    return value
+
+
+def _blocks(rows: list[list[Any]]) -> list[list[int]]:
+    blocks, current = [], []
+    for index, row in enumerate(rows):
+        if any(value is not None for value in row):
+            current.append(index)
+        elif current:
+            blocks.append(current)
+            current = []
+    if current:
+        blocks.append(current)
+    return blocks
+
+
+def _read_block(
+    rows: list[list[Any]], block: list[int], columns: list[int], sheet: str | None
+) -> _Piece:
+    cells = [[rows[r][c] for c in columns] for r in block]
+    need = max(1, math.ceil(len(columns) / 2))
+    dense = [sum(value is not None for value in row) >= need for row in cells]
+    search = range(min(HEADER_SEARCH_ROWS, len(cells)))
+    warnings = []
+
+    # 表头：前几行里第一个够满、并且全是文字的行；标题行通常只有一个单元格，不够满
+    header_at = next((i for i in search if dense[i] and _is_label_row(cells[i])), None)
+    confident = header_at is not None
+    if header_at is None:
+        header_at = next((i for i in search if dense[i]), 0)
+        warnings.append(
+            ParseWarning(
+                code="header_uncertain",
+                message=f"{where(sheet)}没有找到明确的表头，用第 {block[header_at] + 1} 行作表头，可能识别错误",
+            )
+        )
+    if header_at > 0:
+        skipped = "".join(
+            _quote(cells[i]) for i in range(header_at) if _quote(cells[i])
+        )
+        warnings.append(
+            ParseWarning(
+                code="title_skipped",
+                message=f"{where(sheet)}{_row_range(block[0], block[header_at - 1])}是标题，已跳过：{skipped}",
+            )
+        )
+
+    header = cells[header_at]
+    data_at = header_at + 1
+    if (
+        confident
+        and header_at + 2 < len(cells)
+        and _is_subheader(cells[header_at], cells[header_at + 1], cells[header_at + 2])
+    ):
+        header = _merge_header(cells[header_at], cells[header_at + 1])
+        data_at = header_at + 2
+        warnings.append(
+            ParseWarning(
+                code="header_merged",
+                message=f"{where(sheet)}{_row_range(block[header_at], block[header_at + 1])}合并为表头",
+            )
+        )
+    return _Piece(columns, block[header_at], header, block[data_at:], warnings)
+
+
+def _finish(rows: list[list[Any]], piece: _Piece, sheet: str | None) -> RawTable | None:
+    data = [(index, [rows[index][c] for c in piece.columns]) for index in piece.data]
+    warnings = piece.warnings
+
+    # 表格下方只有一个文字单元格的行是说明，例如"注：数据来源于……"
+    notes = []
+    while data and len(piece.columns) >= 3 and _is_note(data[-1][1]):
+        notes.append(data.pop())
+    if notes:
+        notes.reverse()
+        shown = "".join(_quote(row) for _, row in notes)
+        warnings.append(
+            ParseWarning(
+                code="note_skipped",
+                message=f"{where(sheet)}{_row_range(notes[0][0], notes[-1][0])}是说明，已跳过：{shown}",
+            )
+        )
+
+    # 合计、小计行不是明细数据，留着会让求和重复计算
+    totals = [index for index, row in data if _is_total(row)]
+    if totals:
+        data = [(index, row) for index, row in data if not _is_total(row)]
+        listed = "、".join(str(index + 1) for index in totals[:5]) + (
+            "等" if len(totals) > 5 else ""
+        )
+        warnings.append(
+            ParseWarning(
+                code="total_rows_removed",
+                message=f"{where(sheet)}已排除 {len(totals)} 行合计、小计（第 {listed} 行）",
+            )
+        )
+
+    if not data:
+        return None
+    if len(data) > MAX_TABLE_ROWS:
+        raise rows_exceeded(sheet)
+    return RawTable(sheet, None, piece.header, [row for _, row in data], warnings)
+
+
+def _is_label(value: Any) -> bool:
+    return isinstance(value, str) and classify(value)[0] == "text"
+
+
+def _is_label_row(row: list[Any]) -> bool:
+    values = [value for value in row if value is not None]
+    return bool(values) and all(_is_label(value) for value in values)
+
+
+def _is_subheader(top: list[Any], sub: list[Any], first_data: list[Any]) -> bool:
+    """第二行也全是文字、紧接着是带数值的数据行，并且两行里有合并单元格留下的空格"""
+    return (
+        _is_label_row(sub)
+        and not _is_label_row(first_data)
+        and (None in top or None in sub)
+    )
+
+
+def _merge_header(top: list[Any], sub: list[Any]) -> list[str]:
+    """合并单元格只有左上角有值，第一行横向向前填充后与第二行用 - 连接，例如 销售额-一月"""
+    merged, carry = [], None
+    for upper, lower in zip(top, sub):
+        if upper is not None:
+            carry = upper
+        parts = [label_text(part) for part in (carry, lower) if part is not None]
+        merged.append("-".join(parts))
+    return merged
+
+
+def _is_note(row: list[Any]) -> bool:
+    values = [value for value in row if value is not None]
+    return len(values) == 1 and _is_label(values[0])
+
+
+def _is_total(row: list[Any]) -> bool:
+    checked = 0
+    for value in row:
+        if value is None:
+            continue
+        if isinstance(value, str):
+            key = re.sub(r"[\s:：]", "", value).lower()
+            if key in TOTAL_LABELS or key.endswith(TOTAL_SUFFIXES):
+                return True
+        checked += 1
+        if checked == 2:
+            return False
+    return False
+
+
+def _quote(row: list[Any]) -> str:
+    """一行里第一个非空单元格，截断后加引号"""
+    value = next((value for value in row if value is not None), None)
+    if value is None:
+        return ""
+    text = label_text(value)
+    if len(text) > QUOTE_CHARS:
+        text = text[:QUOTE_CHARS] + "…"
+    return f"「{text}」"
+
+
+def _row_range(first: int, last: int) -> str:
+    return f"第 {first + 1} 行" if first == last else f"第 {first + 1}–{last + 1} 行"