feat：完成大部分数据文件上传解析功能

2026-10-08 08:23:01 -07:00
parent 05c050d2ba
commit bb6a9405db
27 changed files with 2675 additions and 18 deletions
@@ -0,0 +1,126 @@
+"""列概况和预览：上传时在 Parquet 上用 DuckDB 计算，存进文件记录，供 describe_data 和接口返回"""
+
+import math
+from pathlib import Path
+from typing import Any
+
+import duckdb
+
+from app.datafiles.query import connect, json_value, quote_identifier
+from app.schemas.data_file import DataColumn
+
+PREVIEW_ROWS = 20
+SAMPLE_ROWS = 5
+EXAMPLES = 3
+TOP_VALUES = 5
+# 去重值不超过这么多的文本、布尔列才给出现次数最多的值
+TOP_VALUES_MAX_DISTINCT = 50
+# 样例值从开头这么多行里取
+EXAMPLE_SCAN_ROWS = 1000
+PROFILE_CELL_CHARS = 50
+PREVIEW_CELL_CHARS = 200
+
+
+def profile_table(
+    path: Path, columns: list[DataColumn], notes: dict[str, str]
+) -> tuple[dict[str, Any], list[list[Any]]]:
+    """返回列概况和接口用的预览（前 PREVIEW_ROWS 行）"""
+    connection = connect({"t": path})
+    try:
+        rows = connection.execute("SELECT count(*) FROM t").fetchone()[0]
+        connection.execute("SUMMARIZE t")
+        fields = [column[0] for column in connection.description]
+        summary = {row[0]: dict(zip(fields, row)) for row in connection.fetchall()}
+        head = connection.execute(f"FROM t LIMIT {EXAMPLE_SCAN_ROWS}").fetchall()
+        profiled = [
+            _profile_column(
+                connection,
+                column,
+                summary[column.name],
+                head,
+                position,
+                notes.get(column.name),
+            )
+            for position, column in enumerate(columns)
+        ]
+        sample = connection.execute(
+            f"FROM t USING SAMPLE reservoir({SAMPLE_ROWS} ROWS) REPEATABLE (42)"
+        ).fetchall()
+    finally:
+        connection.close()
+    profile = {
+        "rows": rows,
+        "columns": profiled,
+        "sample": [
+            [json_value(value, PROFILE_CELL_CHARS) for value in row] for row in sample
+        ],
+    }
+    preview = [
+        [json_value(value, PREVIEW_CELL_CHARS) for value in row]
+        for row in head[:PREVIEW_ROWS]
+    ]
+    return profile, preview
+
+
+def _profile_column(
+    connection: duckdb.DuckDBPyConnection,
+    column: DataColumn,
+    stats: dict[str, Any],
+    head: list[tuple],
+    position: int,
+    note: str | None,
+) -> dict[str, Any]:
+    item: dict[str, Any] = {"name": column.name, "type": column.type}
+    if column.format:
+        item["format"] = column.format
+    null_ratio = float(stats["null_percentage"]) / 100
+    item["null_ratio"] = round(null_ratio, 4)
+    # approx_unique 是估算值，小表上可能比行数还大
+    item["distinct"] = min(
+        stats["approx_unique"], round(stats["count"] * (1 - null_ratio))
+    )
+    if column.type == "number":
+        for key in ("min", "max", "avg", "q25", "q50", "q75"):
+            item[key] = _number(stats[key])
+    elif column.type in ("date", "datetime"):
+        # SUMMARIZE 的时间用空格分隔，和预览里的 ISO 写法统一
+        item["min"], item["max"] = (
+            None if stats[key] is None else stats[key].replace(" ", "T", 1)
+            for key in ("min", "max")
+        )
+    if (
+        column.type in ("text", "boolean")
+        and stats["approx_unique"] <= TOP_VALUES_MAX_DISTINCT
+    ):
+        name = quote_identifier(column.name)
+        counts = connection.execute(
+            f"SELECT {name}, count(*) AS n FROM t WHERE {name} IS NOT NULL "
+            f"GROUP BY 1 ORDER BY n DESC, 1 LIMIT {TOP_VALUES}"
+        ).fetchall()
+        item["top_values"] = [
+            {"value": json_value(value, PROFILE_CELL_CHARS), "count": count}
+            for value, count in counts
+        ]
+    examples: list[Any] = []
+    for row in head:
+        value = json_value(row[position], PROFILE_CELL_CHARS)
+        if value is not None and value not in examples:
+            examples.append(value)
+            if len(examples) == EXAMPLES:
+                break
+    item["examples"] = examples
+    if note:
+        item["note"] = note
+    return item
+
+
+def _number(text: str | None) -> int | float | None:
+    """SUMMARIZE 的统计值是字符串；整数不带小数点，其余保留 6 位小数；Parquet 里的 inf 返回空"""
+    if text is None:
+        return None
+    value = float(text)
+    if not math.isfinite(value):
+        return None
+    if value.is_integer() and abs(value) < 2**53:
+        return int(value)
+    return round(value, 6)