feat：LLM 分析文件并生成结构化建议

app/datafiles/service.py

@@ -14,12 +14,11 @@ from pathlib import Path
 from typing import BinaryIO
 from uuid import uuid4
 
-import duckdb
 from loguru import logger
 
 from app.config import PROJECT_ROOT, settings
 from app.core.exceptions import BizException
-from app.datafiles.aggregate import build_aggregate_sql
+from app.datafiles.aggregate import run_aggregate
 from app.datafiles.insights import (
     INSIGHTS_CONCURRENCY,
     INSIGHTS_TIMEOUT,
@@ -27,9 +26,8 @@ from app.datafiles.insights import (
 )
 from app.datafiles.parse import ParsedFile, ParsedTable
 from app.datafiles.profile import read_rows
-from app.datafiles.query import QueryError, run_query
-from app.datafiles.storage import SOURCE_NAME, DataFileStore, file_dir
-from app.schemas.base import JsonObject
+from app.datafiles.query import QueryError
+from app.datafiles.storage import SOURCE_NAME, DataFileStore, table_path
 from app.schemas.data_file import (
     AggregateRequest,
     AggregateResult,
@@ -37,8 +35,6 @@ from app.schemas.data_file import (
     DataTable,
     DataTableRows,
     FileInsights,
-    RequestDataFile,
-    RequestDataTable,
 )
 from app.services.conversation import ConversationStore
 
@@ -75,14 +71,18 @@ class DataFileService:
         self._conversations = conversations
         self._ttl = timedelta(seconds=ttl or settings.data_file_ttl)
         self._parsing = asyncio.Semaphore(PARSE_CONCURRENCY)
-        # 文件数检查、分配表名和写入记录要一起完成，并发上传时才不会超限或重名；单实例部署，进程内的锁就够了
+        # 文件数检查和写入记录要一起完成，并发上传时才不会超限；单实例部署，进程内的锁就够了
         self._saving = asyncio.Lock()
         # 上传后的 LLM 分析；为空表示不分析，文件的 insights 为空
         self._insights = insights
         self._insight_slots = asyncio.Semaphore(INSIGHTS_CONCURRENCY)
-        # 正在本进程里分析的文件。记录里是 pending 却不在这里，说明进程重启过、任务已丢失
+        # 本进程里正在分析的文件，用来拒绝重复的重新分析
         self._analysing: set[str] = set()
         self._tasks: set[asyncio.Task] = set()
+        # 在这之前开始、记录里仍是 pending 的分析属于上一个进程，已随重启丢失；
+        # MongoDB 里的时间只精确到毫秒，这里也截到毫秒，刚启动就开始的分析读回后才不会早于它
+        now = datetime.now(UTC)
+        self._started_at = now.replace(microsecond=now.microsecond // 1000 * 1000)
 
     async def upload(
         self, user_id: int, conversation_id: str, name: str | None, source: BinaryIO
@@ -91,14 +91,16 @@ class DataFileService:
         # 先粗查一次，满了就不必上传和解析；写入前在锁里再查一次
         _check_count(await self.store.find(conversation_id))
         file_id = f"f_{uuid4().hex}"
-        directory = file_dir(self.root, file_id)
+        directory = self.root / file_id
         try:
             size, sha256 = await asyncio.to_thread(self._save, source, directory)
             async with self._parsing:
                 parsed = await _parse(directory)
             async with self._saving:
-                existing = await self.store.find(conversation_id, include_deleted=True)
-                _check_count([record for record in existing if not record.deleted])
+                _check_count(await self.store.find(conversation_id))
+                first_number = await self.store.reserve_table_numbers(
+                    conversation_id, len(parsed.tables)
+                )
                 record = self._record(
                     file_id,
                     user_id,
@@ -107,18 +109,11 @@ class DataFileService:
                     size,
                     sha256,
                     parsed,
-                    _next_table_number(existing),
+                    first_number,
                 )
-                if self._insights is not None:
-                    # 先登记再写入，别的请求读到 pending 时不会把它当成丢失的任务
-                    self._analysing.add(file_id)
-                    record = record.model_copy(
-                        update={"insights": FileInsights(status="pending")}
-                    )
                 await self.store.insert(record)
         except BaseException:
             # 失败的上传不留文件，会话内的文件数不变
-            self._analysing.discard(file_id)
             await asyncio.to_thread(shutil.rmtree, directory, ignore_errors=True)
             raise
         if self._insights is not None:
@@ -137,9 +132,7 @@ class DataFileService:
         await self._check_conversation(conversation_id, user_id)
         records = await self.store.find(conversation_id)
         return [
-            await self._settle(record)
-            for record in records
-            if record.owner_id == user_id
+            self._settle(record) for record in records if record.owner_id == user_id
         ]
 
     async def get_file(self, user_id: int, file_id: str) -> DataFileRecord:
@@ -147,7 +140,7 @@ class DataFileService:
         # 他人的文件与不存在同样返回 404
         if record is None or record.owner_id != user_id:
             raise BizException(404, NOT_FOUND)
-        return await self._settle(record)
+        return self._settle(record)
 
     async def regenerate_insights(self, user_id: int, file_id: str) -> DataFileRecord:
         """重新分析，用于上次失败之后；正在分析时 409"""
@@ -157,7 +150,7 @@ class DataFileService:
         if file_id in self._analysing:
             raise BizException(409, "正在分析，请稍后")
         self._analysing.add(file_id)
-        pending = FileInsights(status="pending")
+        pending = FileInsights(status="pending", started_at=datetime.now(UTC))
         try:
             await self.store.update_insights(file_id, pending)
         except BaseException:
@@ -179,19 +172,12 @@ class DataFileService:
     ) -> DataTableRows:
         """表的一页数据；表或排序列不存在时 404 / 422，文件已被清理时同样 404"""
         record = await self.get_file(user_id, file_id)
-        table = next((item for item in record.tables if item.table == table_name), None)
-        if table is None:
-            raise BizException(404, "数据表不存在")
+        table, path = self._table(record, table_name)
         if sort_by is not None and sort_by not in {c.name for c in table.columns}:
             raise BizException(422, f"排序列不存在：{sort_by}")
-        path = file_dir(self.root, file_id) / table.parquet
-        try:
-            rows = await asyncio.to_thread(
-                read_rows, path, offset, limit, sort_by, descending
-            )
-        except duckdb.IOException:
-            # 记录还在但 Parquet 已被删除（过期清理、删除时目录没删净）
-            raise BizException(404, NOT_FOUND) from None
+        rows = await asyncio.to_thread(
+            read_rows, path, offset, limit, sort_by, descending
+        )
         return DataTableRows(total=table.rows, offset=offset, rows=rows)
 
     async def aggregate(
@@ -199,81 +185,36 @@ class DataFileService:
     ) -> AggregateResult:
         """对一张表分组汇总，供前端做图表数据；和 agent 的 query_data 同一套只读沙箱"""
         record = await self.get_file(user_id, file_id)
-        table = next((item for item in record.tables if item.table == table_name), None)
-        if table is None:
-            raise BizException(404, "数据表不存在")
-        sql, names = build_aggregate_sql(request, table.columns)
-        path = file_dir(self.root, file_id) / table.parquet
-        if not path.is_file():
-            raise BizException(404, NOT_FOUND)
+        table, path = self._table(record, table_name)
         try:
-            result = await asyncio.to_thread(
-                run_query,
-                sql,
-                {"t": path},
-                timeout=settings.data_query_timeout,
-                max_rows=request.limit,
-            )
+            return await asyncio.to_thread(run_aggregate, path, request, table.columns)
         except QueryError as error:
             raise BizException(422, str(error)) from None
-        return AggregateResult(
-            columns=names,
-            rows=[dict(zip(names, row)) for row in result.rows],
-            truncated=result.truncated,
-        )
 
     async def delete_file(self, user_id: int, file_id: str) -> None:
         await self.get_file(user_id, file_id)
         await self.store.mark_deleted(file_id)
         # Windows 上正在查询的 Parquet 删不掉，留给定时清理
-        await asyncio.to_thread(
-            shutil.rmtree, file_dir(self.root, file_id), ignore_errors=True
-        )
+        await asyncio.to_thread(shutil.rmtree, self.root / file_id, ignore_errors=True)
 
     async def request_files(
-        self, user_id: int, conversation_id: str, attachments: list[JsonObject]
-    ) -> list[RequestDataFile]:
-        """本次请求可用的文件，调用方已校验会话归属
+        self, user_id: int, conversation_id: str, attached: set[str]
+    ) -> list[DataFileRecord]:
+        """本次请求可用的文件：当前用户在本会话里未过期的文件，调用方已校验会话归属
 
-        attachments 里 kind 为 data_file 的引用必须是本会话未过期的文件，否则 404；其他种类不处理
+        attached 是本轮附件引用的文件，必须都在其中，否则 404
         """
         records = [
             record
             for record in await self.store.find(conversation_id)
             if record.owner_id == user_id
         ]
-        attached = set()
-        for item in attachments:
-            if item.get("kind") != "data_file":
-                continue
-            if not isinstance(file_id := item.get("file_id"), str):
-                raise BizException(422, "data_file 附件缺少 file_id")
-            attached.add(file_id)
         if attached - {record.file_id for record in records}:
             raise BizException(404, NOT_FOUND)
-        return [
-            RequestDataFile(
-                file_id=record.file_id,
-                name=record.name,
-                attached_this_turn=record.file_id in attached,
-                tables=[
-                    RequestDataTable.model_validate(
-                        {
-                            **table.model_dump(),
-                            "path": str(
-                                file_dir(self.root, record.file_id) / table.parquet
-                            ),
-                        }
-                    )
-                    for table in record.tables
-                ],
-                warnings=record.warnings,
-            )
-            for record in records
-        ]
+        return records
 
     async def aclose(self) -> None:
-        """应用退出时取消还在进行的分析，记录里留下的 pending 下次读取时会记为失败"""
+        """应用退出时取消还在进行的分析，记录里留下的 pending 下次启动后读取时按失败返回"""
         for task in list(self._tasks):
             task.cancel()
         await asyncio.gather(*self._tasks, return_exceptions=True)
@@ -293,6 +234,7 @@ class DataFileService:
             await asyncio.sleep(SWEEP_INTERVAL)
 
     def _start_insights(self, record: DataFileRecord) -> None:
+        self._analysing.add(record.file_id)
         task = asyncio.create_task(self._analyse(record))
         self._tasks.add(task)
         task.add_done_callback(self._tasks.discard)
@@ -323,18 +265,32 @@ class DataFileService:
         finally:
             self._analysing.discard(record.file_id)
 
-    async def _settle(self, record: DataFileRecord) -> DataFileRecord:
-        """pending 但本进程没有在分析（重启丢了任务），记为失败，免得前端一直等"""
+    def _settle(self, record: DataFileRecord) -> DataFileRecord:
+        """服务启动前开始、仍是 pending 的分析已随重启丢失，按失败返回，免得前端一直等
+
+        只看记录本身，不写回存储：读到的记录可能已经过时，写回会覆盖刚完成的结果
+        """
         insights = record.insights
         if (
-            insights is None
-            or insights.status != "pending"
-            or record.file_id in self._analysing
+            insights is not None
+            and insights.status == "pending"
+            and (insights.started_at is None or insights.started_at < self._started_at)
         ):
-            return record
-        failed = FileInsights(status="failed")
-        await self.store.update_insights(record.file_id, failed)
-        return record.model_copy(update={"insights": failed})
+            return record.model_copy(update={"insights": FileInsights(status="failed")})
+        return record
+
+    def _table(self, record: DataFileRecord, name: str) -> tuple[DataTable, Path]:
+        """文件里的一张表和它的 Parquet
+
+        表不存在，或 Parquet 已被删除（删除时目录没删净、过期清理）时 404
+        """
+        table = next((item for item in record.tables if item.table == name), None)
+        if table is None:
+            raise BizException(404, "数据表不存在")
+        path = table_path(self.root, record.file_id, table)
+        if not path.is_file():
+            raise BizException(404, NOT_FOUND)
+        return table, path
 
     async def _check_conversation(self, conversation_id: str, user_id: int) -> None:
         if not await self._conversations.exists(conversation_id, user_id):
@@ -348,7 +304,7 @@ class DataFileService:
             while chunk := source.read(CHUNK_BYTES):
                 size += len(chunk)
                 if size > self.max_mb * 1024 * 1024:
-                    raise BizException(413, f"文件不能超过 {self.max_mb} MB")
+                    raise file_too_large(self.max_mb)
                 digest.update(chunk)
                 target.write(chunk)
         return size, digest.hexdigest()
@@ -371,12 +327,7 @@ class DataFileService:
                 DataTable(
                     table=table_name,
                     label=_label(name, table),
-                    sheet=table.sheet,
-                    rows=table.rows,
-                    columns=table.columns,
-                    preview=table.preview,
-                    profile=table.profile,
-                    parquet=table.path.name,
+                    **table.model_dump(exclude={"part", "warnings"}),
                 )
             )
             warnings.extend(
@@ -397,6 +348,11 @@ class DataFileService:
             expires_at=now + self._ttl,
             tables=tables,
             warnings=warnings,
+            insights=(
+                None
+                if self._insights is None
+                else FileInsights(status="pending", started_at=now)
+            ),
         )
 
     def _remove_orphans(self, live: set[str]) -> int:
@@ -466,6 +422,10 @@ def _python() -> tuple[str, dict[str, str] | None]:
     return sys.executable, None
 
 
+def file_too_large(max_mb: int) -> BizException:
+    return BizException(413, f"文件不能超过 {max_mb} MB")
+
+
 def _check_count(records: list[DataFileRecord]) -> None:
     if len(records) >= MAX_FILES_PER_CONVERSATION:
         raise BizException(
@@ -473,12 +433,6 @@ def _check_count(records: list[DataFileRecord]) -> None:
         )
 
 
-def _next_table_number(records: list[DataFileRecord]) -> int:
-    """表名在会话内递增，已删除文件的表名也不再使用"""
-    used = (int(table.table[1:]) for record in records for table in record.tables)
-    return max(used, default=0) + 1
-
-
 def _clean_name(name: str | None) -> str:
     # 有的浏览器会带上本地路径，只保留文件名
     base = re.split(r"[\\/]", name or "")[-1].strip()