feat：LLM 分析文件并生成结构化建议

2026-10-10 04:56:28 -07:00
parent 7caf0d31da
commit b90842237f
11 changed files with 746 additions and 7 deletions
@@ -1,4 +1,4 @@
-"""上传流程：保存原文件 → 子进程解析（超时终止）→ 分配表名 → 保存记录；以及查看、删除和过期清理"""
+"""上传流程：保存原文件 → 子进程解析（超时终止）→ 分配表名 → 保存记录 → 后台用 LLM 分析；以及查看、删除和过期清理"""

 import asyncio
 import hashlib
@@ -14,16 +14,29 @@ from pathlib import Path
 from typing import BinaryIO
 from uuid import uuid4

+import duckdb
 from loguru import logger

 from app.config import PROJECT_ROOT, settings
 from app.core.exceptions import BizException
+from app.datafiles.aggregate import build_aggregate_sql
+from app.datafiles.insights import (
+    INSIGHTS_CONCURRENCY,
+    INSIGHTS_TIMEOUT,
+    InsightsGenerator,
+)
 from app.datafiles.parse import ParsedFile, ParsedTable
+from app.datafiles.profile import read_rows
+from app.datafiles.query import QueryError, run_query
 from app.datafiles.storage import SOURCE_NAME, DataFileStore, file_dir
 from app.schemas.base import JsonObject
 from app.schemas.data_file import (
+    AggregateRequest,
+    AggregateResult,
    DataFileRecord,
    DataTable,
+    DataTableRows,
+    FileInsights,
    RequestDataFile,
    RequestDataTable,
 )
@@ -54,6 +67,7 @@ class DataFileService:
        *,
        max_mb: int | None = None,
        ttl: int | None = None,
+        insights: InsightsGenerator | None = None,
    ):
        self.store = store
        self.root = root or settings.data_file_dir
@@ -63,6 +77,12 @@ class DataFileService:
        self._parsing = asyncio.Semaphore(PARSE_CONCURRENCY)
        # 文件数检查、分配表名和写入记录要一起完成，并发上传时才不会超限或重名；单实例部署，进程内的锁就够了
        self._saving = asyncio.Lock()
+        # 上传后的 LLM 分析；为空表示不分析，文件的 insights 为空
+        self._insights = insights
+        self._insight_slots = asyncio.Semaphore(INSIGHTS_CONCURRENCY)
+        # 正在本进程里分析的文件。记录里是 pending 却不在这里，说明进程重启过、任务已丢失
+        self._analysing: set[str] = set()
+        self._tasks: set[asyncio.Task] = set()

    async def upload(
        self, user_id: int, conversation_id: str, name: str | None, source: BinaryIO
@@ -89,11 +109,20 @@ class DataFileService:
                    parsed,
                    _next_table_number(existing),
                )
+                if self._insights is not None:
+                    # 先登记再写入，别的请求读到 pending 时不会把它当成丢失的任务
+                    self._analysing.add(file_id)
+                    record = record.model_copy(
+                        update={"insights": FileInsights(status="pending")}
+                    )
                await self.store.insert(record)
        except BaseException:
            # 失败的上传不留文件，会话内的文件数不变
+            self._analysing.discard(file_id)
            await asyncio.to_thread(shutil.rmtree, directory, ignore_errors=True)
            raise
+        if self._insights is not None:
+            self._start_insights(record)
        logger.info(
            "数据文件已上传：{} 格式 {}，{} 张表",
            file_id,
@@ -107,15 +136,92 @@ class DataFileService:
    ) -> list[DataFileRecord]:
        await self._check_conversation(conversation_id, user_id)
        records = await self.store.find(conversation_id)
-        return [record for record in records if record.owner_id == user_id]
+        return [
+            await self._settle(record)
+            for record in records
+            if record.owner_id == user_id
+        ]

    async def get_file(self, user_id: int, file_id: str) -> DataFileRecord:
        record = await self.store.get(file_id)
        # 他人的文件与不存在同样返回 404
        if record is None or record.owner_id != user_id:
            raise BizException(404, NOT_FOUND)
+        return await self._settle(record)
+
+    async def regenerate_insights(self, user_id: int, file_id: str) -> DataFileRecord:
+        """重新分析，用于上次失败之后；正在分析时 409"""
+        if self._insights is None:
+            raise BizException(422, "未启用文件分析")
+        record = await self.get_file(user_id, file_id)
+        if file_id in self._analysing:
+            raise BizException(409, "正在分析，请稍后")
+        self._analysing.add(file_id)
+        pending = FileInsights(status="pending")
+        try:
+            await self.store.update_insights(file_id, pending)
+        except BaseException:
+            self._analysing.discard(file_id)
+            raise
+        record = record.model_copy(update={"insights": pending})
+        self._start_insights(record)
        return record

+    async def get_rows(
+        self,
+        user_id: int,
+        file_id: str,
+        table_name: str,
+        offset: int,
+        limit: int,
+        sort_by: str | None = None,
+        descending: bool = False,
+    ) -> DataTableRows:
+        """表的一页数据；表或排序列不存在时 404 / 422，文件已被清理时同样 404"""
+        record = await self.get_file(user_id, file_id)
+        table = next((item for item in record.tables if item.table == table_name), None)
+        if table is None:
+            raise BizException(404, "数据表不存在")
+        if sort_by is not None and sort_by not in {c.name for c in table.columns}:
+            raise BizException(422, f"排序列不存在：{sort_by}")
+        path = file_dir(self.root, file_id) / table.parquet
+        try:
+            rows = await asyncio.to_thread(
+                read_rows, path, offset, limit, sort_by, descending
+            )
+        except duckdb.IOException:
+            # 记录还在但 Parquet 已被删除（过期清理、删除时目录没删净）
+            raise BizException(404, NOT_FOUND) from None
+        return DataTableRows(total=table.rows, offset=offset, rows=rows)
+
+    async def aggregate(
+        self, user_id: int, file_id: str, table_name: str, request: AggregateRequest
+    ) -> AggregateResult:
+        """对一张表分组汇总，供前端做图表数据；和 agent 的 query_data 同一套只读沙箱"""
+        record = await self.get_file(user_id, file_id)
+        table = next((item for item in record.tables if item.table == table_name), None)
+        if table is None:
+            raise BizException(404, "数据表不存在")
+        sql, names = build_aggregate_sql(request, table.columns)
+        path = file_dir(self.root, file_id) / table.parquet
+        if not path.is_file():
+            raise BizException(404, NOT_FOUND)
+        try:
+            result = await asyncio.to_thread(
+                run_query,
+                sql,
+                {"t": path},
+                timeout=settings.data_query_timeout,
+                max_rows=request.limit,
+            )
+        except QueryError as error:
+            raise BizException(422, str(error)) from None
+        return AggregateResult(
+            columns=names,
+            rows=[dict(zip(names, row)) for row in result.rows],
+            truncated=result.truncated,
+        )
+
    async def delete_file(self, user_id: int, file_id: str) -> None:
        await self.get_file(user_id, file_id)
        await self.store.mark_deleted(file_id)
@@ -166,6 +272,12 @@ class DataFileService:
            for record in records
        ]

+    async def aclose(self) -> None:
+        """应用退出时取消还在进行的分析，记录里留下的 pending 下次读取时会记为失败"""
+        for task in list(self._tasks):
+            task.cancel()
+        await asyncio.gather(*self._tasks, return_exceptions=True)
+
    async def sweep(self) -> int:
        """删除没有对应记录（过期、已删除、内存模式重启后丢失）的目录，返回删除的个数"""
        live = await self.store.live_ids()
@@ -180,6 +292,50 @@ class DataFileService:
                logger.opt(exception=error).error("清理数据文件目录失败")
            await asyncio.sleep(SWEEP_INTERVAL)

+    def _start_insights(self, record: DataFileRecord) -> None:
+        task = asyncio.create_task(self._analyse(record))
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+
+    async def _analyse(self, record: DataFileRecord) -> None:
+        """后台分析并写回记录；任何失败都只记为 failed，不影响文件使用"""
+        assert self._insights is not None
+        try:
+            try:
+                async with self._insight_slots:
+                    insights = await asyncio.wait_for(
+                        self._insights(record, self.root), INSIGHTS_TIMEOUT
+                    )
+            except Exception as error:
+                logger.opt(exception=error).warning(
+                    "数据文件分析失败：{}", record.file_id
+                )
+                insights = FileInsights(status="failed")
+            await self.store.update_insights(record.file_id, insights)
+            logger.info(
+                "数据文件分析完成：{} {}，{} 条推荐",
+                record.file_id,
+                insights.status,
+                len(insights.suggestions),
+            )
+        except Exception as error:
+            logger.opt(exception=error).error("保存数据文件分析结果失败")
+        finally:
+            self._analysing.discard(record.file_id)
+
+    async def _settle(self, record: DataFileRecord) -> DataFileRecord:
+        """pending 但本进程没有在分析（重启丢了任务），记为失败，免得前端一直等"""
+        insights = record.insights
+        if (
+            insights is None
+            or insights.status != "pending"
+            or record.file_id in self._analysing
+        ):
+            return record
+        failed = FileInsights(status="failed")
+        await self.store.update_insights(record.file_id, failed)
+        return record.model_copy(update={"insights": failed})
+
    async def _check_conversation(self, conversation_id: str, user_id: int) -> None:
        if not await self._conversations.exists(conversation_id, user_id):
            raise BizException(404, "会话不存在或已过期")