153 lines
5.3 KiB
Python
153 lines
5.3 KiB
Python
"""数据文件的存储：文件放本地目录，元数据启用 MongoDB 时进集合 data_files，否则放进程内存
|
|
|
|
目录结构是 DATA_FILE_DIR/<file_id>/，里面有原文件 source 和每张表的 1.parquet、2.parquet……
|
|
DuckDB 只能读本地文件，所以文件不进 MongoDB；单实例部署，本地磁盘就够了
|
|
"""
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any, Protocol
|
|
|
|
from bson.codec_options import CodecOptions
|
|
from pymongo import ASCENDING
|
|
|
|
from app.schemas.data_file import DataFileRecord
|
|
|
|
# 原文件不带扩展名：calamine 见到 .xls 等扩展名会按扩展名打开，用户的扩展名和内容不符时会失败
|
|
SOURCE_NAME = "source"
|
|
|
|
|
|
def file_dir(root: Path, file_id: str) -> Path:
|
|
return root / file_id
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
class DataFileStore(Protocol):
|
|
"""内存和 MongoDB 实现同一组方法；过期的记录一律当作不存在"""
|
|
|
|
async def insert(self, record: DataFileRecord) -> None: ...
|
|
|
|
async def get(self, file_id: str) -> DataFileRecord | None:
|
|
"""未过期且未删除的记录"""
|
|
...
|
|
|
|
async def find(
|
|
self, conversation_id: str, *, include_deleted: bool = False
|
|
) -> list[DataFileRecord]:
|
|
"""会话内未过期的记录，按上传时间排列"""
|
|
...
|
|
|
|
async def mark_deleted(self, file_id: str) -> None: ...
|
|
|
|
async def live_ids(self) -> set[str]:
|
|
"""未过期且未删除的 file_id，清理目录时用"""
|
|
...
|
|
|
|
|
|
class InMemoryDataFileStore:
|
|
def __init__(self) -> None:
|
|
self._records: dict[str, DataFileRecord] = {}
|
|
|
|
def _purge_expired(self) -> None:
|
|
now = _now()
|
|
for file_id in [
|
|
key for key, record in self._records.items() if record.expires_at <= now
|
|
]:
|
|
del self._records[file_id]
|
|
|
|
async def insert(self, record: DataFileRecord) -> None:
|
|
self._purge_expired()
|
|
self._records[record.file_id] = record
|
|
|
|
async def get(self, file_id: str) -> DataFileRecord | None:
|
|
self._purge_expired()
|
|
record = self._records.get(file_id)
|
|
return None if record is None or record.deleted else record
|
|
|
|
async def find(
|
|
self, conversation_id: str, *, include_deleted: bool = False
|
|
) -> list[DataFileRecord]:
|
|
self._purge_expired()
|
|
records = [
|
|
record
|
|
for record in self._records.values()
|
|
if record.conversation_id == conversation_id
|
|
and (include_deleted or not record.deleted)
|
|
]
|
|
return sorted(records, key=lambda record: record.created_at)
|
|
|
|
async def mark_deleted(self, file_id: str) -> None:
|
|
if (record := self._records.get(file_id)) is not None:
|
|
self._records[file_id] = record.model_copy(update={"deleted": True})
|
|
|
|
async def live_ids(self) -> set[str]:
|
|
self._purge_expired()
|
|
return {key for key, record in self._records.items() if not record.deleted}
|
|
|
|
|
|
class MongoDataFileStore:
|
|
def __init__(self, db: Any, prefix: str = ""):
|
|
# 读回的时间统一带 UTC 时区；集合名前缀用于测试隔离
|
|
options = CodecOptions(tz_aware=True, tzinfo=UTC)
|
|
self._files = db.get_collection(f"{prefix}data_files", codec_options=options)
|
|
|
|
async def ensure_indexes(self) -> None:
|
|
# 过期记录由 TTL 索引删除；TTL 每分钟才扫一次，查询时仍按 expires_at 过滤
|
|
await self._files.create_index("expires_at", expireAfterSeconds=0)
|
|
await self._files.create_index(
|
|
[("conversation_id", ASCENDING), ("created_at", ASCENDING)]
|
|
)
|
|
|
|
async def insert(self, record: DataFileRecord) -> None:
|
|
await self._files.insert_one({"_id": record.file_id, **record.model_dump()})
|
|
|
|
async def get(self, file_id: str) -> DataFileRecord | None:
|
|
document = await self._files.find_one(
|
|
{"_id": file_id, "deleted": False, "expires_at": {"$gt": _now()}}
|
|
)
|
|
return None if document is None else _record(document)
|
|
|
|
async def find(
|
|
self, conversation_id: str, *, include_deleted: bool = False
|
|
) -> list[DataFileRecord]:
|
|
query: dict[str, Any] = {
|
|
"conversation_id": conversation_id,
|
|
"expires_at": {"$gt": _now()},
|
|
}
|
|
if not include_deleted:
|
|
query["deleted"] = False
|
|
documents = await (
|
|
self._files.find(query)
|
|
.sort([("created_at", ASCENDING), ("_id", ASCENDING)])
|
|
.to_list()
|
|
)
|
|
return [_record(document) for document in documents]
|
|
|
|
async def mark_deleted(self, file_id: str) -> None:
|
|
await self._files.update_one({"_id": file_id}, {"$set": {"deleted": True}})
|
|
|
|
async def live_ids(self) -> set[str]:
|
|
return set(
|
|
await self._files.distinct(
|
|
"_id", {"deleted": False, "expires_at": {"$gt": _now()}}
|
|
)
|
|
)
|
|
|
|
|
|
def _record(document: dict[str, Any]) -> DataFileRecord:
|
|
return DataFileRecord.model_validate(
|
|
{key: value for key, value in document.items() if key != "_id"}
|
|
)
|
|
|
|
|
|
async def open_data_file_store(db: Any) -> DataFileStore:
|
|
"""db 为空（未启用 MongoDB）时用内存实现，和会话记录一致"""
|
|
if db is None:
|
|
return InMemoryDataFileStore()
|
|
store = MongoDataFileStore(db)
|
|
await store.ensure_indexes()
|
|
return store
|