feat：完成大部分数据文件上传解析功能

app/datafiles/storage.py

@@ -0,0 +1,152 @@
+"""数据文件的存储：文件放本地目录，元数据启用 MongoDB 时进集合 data_files，否则放进程内存
+
+目录结构是 DATA_FILE_DIR/<file_id>/，里面有原文件 source 和每张表的 1.parquet、2.parquet……
+DuckDB 只能读本地文件，所以文件不进 MongoDB；单实例部署，本地磁盘就够了
+"""
+
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Protocol
+
+from bson.codec_options import CodecOptions
+from pymongo import ASCENDING
+
+from app.schemas.data_file import DataFileRecord
+
+# 原文件不带扩展名：calamine 见到 .xls 等扩展名会按扩展名打开，用户的扩展名和内容不符时会失败
+SOURCE_NAME = "source"
+
+
+def file_dir(root: Path, file_id: str) -> Path:
+    return root / file_id
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class DataFileStore(Protocol):
+    """内存和 MongoDB 实现同一组方法；过期的记录一律当作不存在"""
+
+    async def insert(self, record: DataFileRecord) -> None: ...
+
+    async def get(self, file_id: str) -> DataFileRecord | None:
+        """未过期且未删除的记录"""
+        ...
+
+    async def find(
+        self, conversation_id: str, *, include_deleted: bool = False
+    ) -> list[DataFileRecord]:
+        """会话内未过期的记录，按上传时间排列"""
+        ...
+
+    async def mark_deleted(self, file_id: str) -> None: ...
+
+    async def live_ids(self) -> set[str]:
+        """未过期且未删除的 file_id，清理目录时用"""
+        ...
+
+
+class InMemoryDataFileStore:
+    def __init__(self) -> None:
+        self._records: dict[str, DataFileRecord] = {}
+
+    def _purge_expired(self) -> None:
+        now = _now()
+        for file_id in [
+            key for key, record in self._records.items() if record.expires_at <= now
+        ]:
+            del self._records[file_id]
+
+    async def insert(self, record: DataFileRecord) -> None:
+        self._purge_expired()
+        self._records[record.file_id] = record
+
+    async def get(self, file_id: str) -> DataFileRecord | None:
+        self._purge_expired()
+        record = self._records.get(file_id)
+        return None if record is None or record.deleted else record
+
+    async def find(
+        self, conversation_id: str, *, include_deleted: bool = False
+    ) -> list[DataFileRecord]:
+        self._purge_expired()
+        records = [
+            record
+            for record in self._records.values()
+            if record.conversation_id == conversation_id
+            and (include_deleted or not record.deleted)
+        ]
+        return sorted(records, key=lambda record: record.created_at)
+
+    async def mark_deleted(self, file_id: str) -> None:
+        if (record := self._records.get(file_id)) is not None:
+            self._records[file_id] = record.model_copy(update={"deleted": True})
+
+    async def live_ids(self) -> set[str]:
+        self._purge_expired()
+        return {key for key, record in self._records.items() if not record.deleted}
+
+
+class MongoDataFileStore:
+    def __init__(self, db: Any, prefix: str = ""):
+        # 读回的时间统一带 UTC 时区；集合名前缀用于测试隔离
+        options = CodecOptions(tz_aware=True, tzinfo=UTC)
+        self._files = db.get_collection(f"{prefix}data_files", codec_options=options)
+
+    async def ensure_indexes(self) -> None:
+        # 过期记录由 TTL 索引删除；TTL 每分钟才扫一次，查询时仍按 expires_at 过滤
+        await self._files.create_index("expires_at", expireAfterSeconds=0)
+        await self._files.create_index(
+            [("conversation_id", ASCENDING), ("created_at", ASCENDING)]
+        )
+
+    async def insert(self, record: DataFileRecord) -> None:
+        await self._files.insert_one({"_id": record.file_id, **record.model_dump()})
+
+    async def get(self, file_id: str) -> DataFileRecord | None:
+        document = await self._files.find_one(
+            {"_id": file_id, "deleted": False, "expires_at": {"$gt": _now()}}
+        )
+        return None if document is None else _record(document)
+
+    async def find(
+        self, conversation_id: str, *, include_deleted: bool = False
+    ) -> list[DataFileRecord]:
+        query: dict[str, Any] = {
+            "conversation_id": conversation_id,
+            "expires_at": {"$gt": _now()},
+        }
+        if not include_deleted:
+            query["deleted"] = False
+        documents = await (
+            self._files.find(query)
+            .sort([("created_at", ASCENDING), ("_id", ASCENDING)])
+            .to_list()
+        )
+        return [_record(document) for document in documents]
+
+    async def mark_deleted(self, file_id: str) -> None:
+        await self._files.update_one({"_id": file_id}, {"$set": {"deleted": True}})
+
+    async def live_ids(self) -> set[str]:
+        return set(
+            await self._files.distinct(
+                "_id", {"deleted": False, "expires_at": {"$gt": _now()}}
+            )
+        )
+
+
+def _record(document: dict[str, Any]) -> DataFileRecord:
+    return DataFileRecord.model_validate(
+        {key: value for key, value in document.items() if key != "_id"}
+    )
+
+
+async def open_data_file_store(db: Any) -> DataFileStore:
+    """db 为空（未启用 MongoDB）时用内存实现，和会话记录一致"""
+    if db is None:
+        return InMemoryDataFileStore()
+    store = MongoDataFileStore(db)
+    await store.ensure_indexes()
+    return store