feat：LLM 分析文件并生成结构化建议

app/datafiles/insights.py

@@ -9,7 +9,6 @@ import json
 from collections.abc import Awaitable, Callable
 from pathlib import Path
 
-import duckdb
 from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import HumanMessage, SystemMessage
 from loguru import logger
@@ -18,9 +17,9 @@ from pydantic import BaseModel, Field, ValidationError
 from app.clients.llm import get_llm
 from app.config import settings
 from app.core.exceptions import BizException
-from app.datafiles.aggregate import build_aggregate_sql
-from app.datafiles.query import QueryError, run_query
-from app.datafiles.storage import file_dir
+from app.datafiles.aggregate import run_aggregate
+from app.datafiles.query import QueryError
+from app.datafiles.storage import table_path
 from app.prompt.insights import INSIGHTS_SYSTEM_PROMPT
 from app.schemas.data_file import (
     MAX_SUGGESTIONS,
@@ -106,21 +105,16 @@ def build_input(record: DataFileRecord) -> dict:
 
 
 def _table_input(table: DataTable) -> dict:
-    columns = table.profile.get("columns", [])[:MAX_COLUMNS]
-    keep = {column["name"] for column in columns}
-    positions = [
-        position for position, column in enumerate(table.columns) if column.name in keep
-    ]
     return {
         "table": table.table,
         "label": table.label,
         "rows": table.rows,
-        "columns": columns,
-        # 样例行只保留上面那些列，顺序与 columns 一致
-        "sample": [
-            [row[position] for position in positions]
-            for row in table.profile.get("sample", [])
+        "columns": [
+            column.model_dump(mode="json", exclude_none=True)
+            for column in table.profile.columns[:MAX_COLUMNS]
         ],
+        # 样例行和 columns 都按表的列顺序排列，截掉同样的列
+        "sample": [row[:MAX_COLUMNS] for row in table.profile.sample],
     }
 
 
@@ -140,7 +134,7 @@ def check_draft(
         if not title or table is None or key in seen:
             continue
         if not _fits_chart(item) or not _runs(
-            item, table, file_dir(root, record.file_id)
+            item, table, table_path(root, record.file_id, table)
         ):
             logger.info("丢弃不可用的图表推荐：{} {}", record.file_id, item.title)
             continue
@@ -162,20 +156,11 @@ def _fits_chart(item: ChartSuggestion) -> bool:
     return len(request.metrics) == 1 or item.chart not in ("pie", "funnel")
 
 
-def _runs(item: ChartSuggestion, table: DataTable, directory: Path) -> bool:
-    """真实跑一遍汇总：能编译、能执行，并且有足够的行数画图"""
-    path = directory / table.parquet
-    if not path.is_file():
-        return False
+def _runs(item: ChartSuggestion, table: DataTable, path: Path) -> bool:
+    """真实跑一遍汇总：能编译、能执行（文件已被删除也算不能），并且有足够的行数画图"""
     try:
-        sql, _ = build_aggregate_sql(item.request, table.columns)
-        result = run_query(
-            sql,
-            {"t": path},
-            timeout=settings.data_query_timeout,
-            max_rows=item.request.limit,
-        )
-    except (BizException, QueryError, duckdb.Error):
+        result = run_aggregate(path, item.request, table.columns)
+    except (BizException, QueryError):
         return False
     return len(result.rows) >= MIN_RESULT_ROWS