"""数据文件接口:上传限制、上传解析及进度、列出、查看、删除,见 docs/screen-agent/data-files.md 第 5 节""" from typing import Annotated, Literal, get_args from fastapi import APIRouter, Depends, File, Form, Query, Request, UploadFile from fastapi.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send from app.common.response import ApiResponse, error_content, success_response from app.core.auth import get_current_user from app.datafiles.base import MAX_TABLE_ROWS from app.datafiles.service import ( MAX_FILES_PER_CONVERSATION, PARSE_CONCURRENCY, DataFileService, file_too_large, ) from app.schemas.base import NonBlankString from app.schemas.data_file import ( MAX_AGGREGATE_ROWS, MAX_METRICS, AggregateRequest, AggregateResult, DataFileDetail, DataFileInfo, DataFileLimits, DataFileList, DataFileRecord, DataFormat, DataTableRows, UploadProgress, ) from app.schemas.user import CurrentUser # 上传限制不需要登录,单独一个路由;要先于 /files/{file_id} 注册 limits_router = APIRouter(prefix="/files", tags=["DataFile"]) router = APIRouter(prefix="/files", tags=["DataFile"]) # 翻页取表数据:单页上限,offset 不超过一张表最多的行数 MAX_PAGE_ROWS = 500 MAX_OFFSET = MAX_TABLE_ROWS # 文件选择框的过滤条件;后端按文件内容识别格式,不看扩展名 ACCEPT = ".csv,.tsv,.txt,.xlsx,.xls,.xlsb,.json,.jsonl,.ndjson,.parquet" # 上传的请求体比文件本身多出 multipart 的分隔行、字段名和会话 ID UPLOAD_OVERHEAD_BYTES = 64 * 1024 def _service(request: Request) -> DataFileService: return request.app.state.data_file_service class UploadSizeLimit: """上传接口在读取请求体之前按 Content-Length 拒绝过大的请求 FastAPI 会先把整个 multipart 请求体收完、写进临时文件,再调用接口函数; 只在 DataFileService 里边写边查大小的话,超大的上传已经占满了带宽和磁盘 """ def __init__(self, app: ASGIApp, path: str): self.app = app self.path = path async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if ( scope["type"] == "http" and scope["method"] == "POST" and scope["path"] == self.path and (rejection := self._reject(scope)) is not None ): await rejection(scope, receive, send) return await self.app(scope, receive, send) @staticmethod def _reject(scope: Scope) -> JSONResponse | None: max_mb = scope["app"].state.data_file_service.max_mb length = dict(scope["headers"]).get(b"content-length") if length is None or not length.isdigit(): return JSONResponse(error_content(411, "缺少 Content-Length"), 411) if int(length) > max_mb * 1024 * 1024 + UPLOAD_OVERHEAD_BYTES: error = file_too_large(max_mb) return JSONResponse(error_content(error.code, error.msg), error.code) return None def _public(record: DataFileRecord) -> dict: return {**record.model_dump(), "pinned": bool(record.pinned_by)} def _detail(record: DataFileRecord) -> DataFileDetail: # 按接口模型重新校验,去掉 owner_id、sha256、列概况等只在后端用的字段 return DataFileDetail.model_validate(_public(record)) @limits_router.get("/limits") async def get_limits(request: Request) -> ApiResponse[DataFileLimits]: return success_response( DataFileLimits( max_size_mb=_service(request).max_mb, max_files_per_conversation=MAX_FILES_PER_CONVERSATION, formats=list(get_args(DataFormat)), accept=ACCEPT, parse_concurrency=PARSE_CONCURRENCY, rows_max_page=MAX_PAGE_ROWS, aggregate_max_metrics=MAX_METRICS, aggregate_max_rows=MAX_AGGREGATE_ROWS, ) ) @router.post("", status_code=202) async def upload_file( request: Request, conversation_id: Annotated[NonBlankString, Form()], file: Annotated[UploadFile, File()], user: Annotated[CurrentUser, Depends(get_current_user)], ) -> ApiResponse[UploadProgress]: """原文件保存后立即返回,解析在后台进行,前端用返回的 file_id 轮询进度""" progress = await _service(request).upload( user.id, conversation_id, file.filename, file.file ) return success_response(progress) @router.get("") async def list_files( request: Request, conversation_id: Annotated[NonBlankString, Query()], user: Annotated[CurrentUser, Depends(get_current_user)], ) -> ApiResponse[DataFileList]: service = _service(request) records = await service.list_files(user.id, conversation_id) return success_response( DataFileList( files=[DataFileInfo.model_validate(_public(record)) for record in records], uploads=service.list_uploads(user.id, conversation_id), ) ) @router.get("/{file_id}") async def get_file( file_id: str, request: Request, user: Annotated[CurrentUser, Depends(get_current_user)], ) -> ApiResponse[DataFileDetail]: return success_response(_detail(await _service(request).get_file(user.id, file_id))) @router.get("/{file_id}/progress") async def get_progress( file_id: str, request: Request, user: Annotated[CurrentUser, Depends(get_current_user)], ) -> ApiResponse[UploadProgress]: return success_response(await _service(request).get_progress(user.id, file_id)) @router.get("/{file_id}/tables/{table}/rows") async def get_table_rows( file_id: str, table: str, request: Request, user: Annotated[CurrentUser, Depends(get_current_user)], offset: Annotated[int, Query(ge=0, le=MAX_OFFSET)] = 0, limit: Annotated[int, Query(ge=1, le=MAX_PAGE_ROWS)] = 100, sort_by: Annotated[str | None, Query(min_length=1)] = None, order: Annotated[Literal["asc", "desc"], Query()] = "asc", ) -> ApiResponse[DataTableRows]: return success_response( await _service(request).get_rows( user.id, file_id, table, offset, limit, sort_by, order == "desc" ) ) @router.post("/{file_id}/tables/{table}/aggregate") async def aggregate_table( file_id: str, table: str, body: AggregateRequest, request: Request, user: Annotated[CurrentUser, Depends(get_current_user)], ) -> ApiResponse[AggregateResult]: return success_response( await _service(request).aggregate(user.id, file_id, table, body) ) @router.delete("/{file_id}") async def delete_file( file_id: str, request: Request, user: Annotated[CurrentUser, Depends(get_current_user)], ) -> ApiResponse[dict]: await _service(request).delete_file(user.id, file_id) return success_response()