197 lines
6.6 KiB
Python
197 lines
6.6 KiB
Python
"""数据文件接口：上传限制、上传解析、列出、查看、删除，见 docs/screen-agent/data-files.md 第 5 节"""
|
|
|
|
from typing import Annotated, Literal, get_args
|
|
|
|
from fastapi import APIRouter, Depends, File, Form, Query, Request, UploadFile
|
|
from fastapi.responses import JSONResponse
|
|
from starlette.types import ASGIApp, Receive, Scope, Send
|
|
|
|
from app.common.response import ApiResponse, error_content, success_response
|
|
from app.core.auth import get_current_user
|
|
from app.datafiles.base import MAX_TABLE_ROWS
|
|
from app.datafiles.service import (
|
|
MAX_FILES_PER_CONVERSATION,
|
|
PARSE_CONCURRENCY,
|
|
DataFileService,
|
|
file_too_large,
|
|
)
|
|
from app.schemas.base import NonBlankString
|
|
from app.schemas.data_file import (
|
|
MAX_AGGREGATE_ROWS,
|
|
MAX_METRICS,
|
|
AggregateRequest,
|
|
AggregateResult,
|
|
DataFileDetail,
|
|
DataFileInfo,
|
|
DataFileLimits,
|
|
DataFileList,
|
|
DataFileRecord,
|
|
DataFormat,
|
|
DataTableRows,
|
|
)
|
|
from app.schemas.user import CurrentUser
|
|
|
|
# 上传限制不需要登录，单独一个路由；要先于 /files/{file_id} 注册
|
|
limits_router = APIRouter(prefix="/files", tags=["DataFile"])
|
|
router = APIRouter(prefix="/files", tags=["DataFile"])
|
|
|
|
# 翻页取表数据：单页上限，offset 不超过一张表最多的行数
|
|
MAX_PAGE_ROWS = 500
|
|
MAX_OFFSET = MAX_TABLE_ROWS
|
|
|
|
# 文件选择框的过滤条件；后端按文件内容识别格式，不看扩展名
|
|
ACCEPT = ".csv,.tsv,.txt,.xlsx,.xls,.xlsb,.json,.jsonl,.ndjson,.parquet"
|
|
|
|
# 上传的请求体比文件本身多出 multipart 的分隔行、字段名和会话 ID
|
|
UPLOAD_OVERHEAD_BYTES = 64 * 1024
|
|
|
|
|
|
def _service(request: Request) -> DataFileService:
|
|
return request.app.state.data_file_service
|
|
|
|
|
|
class UploadSizeLimit:
|
|
"""上传接口在读取请求体之前按 Content-Length 拒绝过大的请求
|
|
|
|
FastAPI 会先把整个 multipart 请求体收完、写进临时文件，再调用接口函数；
|
|
只在 DataFileService 里边写边查大小的话，超大的上传已经占满了带宽和磁盘
|
|
"""
|
|
|
|
def __init__(self, app: ASGIApp, path: str):
|
|
self.app = app
|
|
self.path = path
|
|
|
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
if (
|
|
scope["type"] == "http"
|
|
and scope["method"] == "POST"
|
|
and scope["path"] == self.path
|
|
and (rejection := self._reject(scope)) is not None
|
|
):
|
|
await rejection(scope, receive, send)
|
|
return
|
|
await self.app(scope, receive, send)
|
|
|
|
@staticmethod
|
|
def _reject(scope: Scope) -> JSONResponse | None:
|
|
max_mb = scope["app"].state.data_file_service.max_mb
|
|
length = dict(scope["headers"]).get(b"content-length")
|
|
if length is None or not length.isdigit():
|
|
return JSONResponse(error_content(411, "缺少 Content-Length"), 411)
|
|
if int(length) > max_mb * 1024 * 1024 + UPLOAD_OVERHEAD_BYTES:
|
|
error = file_too_large(max_mb)
|
|
return JSONResponse(error_content(error.code, error.msg), error.code)
|
|
return None
|
|
|
|
|
|
def _detail(record: DataFileRecord) -> DataFileDetail:
|
|
# 按接口模型重新校验，去掉 owner_id、sha256、列概况等只在后端用的字段
|
|
return DataFileDetail.model_validate(record.model_dump())
|
|
|
|
|
|
@limits_router.get("/limits")
|
|
async def get_limits(request: Request) -> ApiResponse[DataFileLimits]:
|
|
return success_response(
|
|
DataFileLimits(
|
|
max_size_mb=_service(request).max_mb,
|
|
max_files_per_conversation=MAX_FILES_PER_CONVERSATION,
|
|
formats=list(get_args(DataFormat)),
|
|
accept=ACCEPT,
|
|
parse_concurrency=PARSE_CONCURRENCY,
|
|
rows_max_page=MAX_PAGE_ROWS,
|
|
aggregate_max_metrics=MAX_METRICS,
|
|
aggregate_max_rows=MAX_AGGREGATE_ROWS,
|
|
)
|
|
)
|
|
|
|
|
|
@router.post("")
|
|
async def upload_file(
|
|
request: Request,
|
|
conversation_id: Annotated[NonBlankString, Form()],
|
|
file: Annotated[UploadFile, File()],
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
) -> ApiResponse[DataFileDetail]:
|
|
record = await _service(request).upload(
|
|
user.id, conversation_id, file.filename, file.file
|
|
)
|
|
return success_response(_detail(record))
|
|
|
|
|
|
@router.get("")
|
|
async def list_files(
|
|
request: Request,
|
|
conversation_id: Annotated[NonBlankString, Query()],
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
) -> ApiResponse[DataFileList]:
|
|
records = await _service(request).list_files(user.id, conversation_id)
|
|
return success_response(
|
|
DataFileList(
|
|
files=[
|
|
DataFileInfo.model_validate(record.model_dump()) for record in records
|
|
]
|
|
)
|
|
)
|
|
|
|
|
|
@router.get("/{file_id}")
|
|
async def get_file(
|
|
file_id: str,
|
|
request: Request,
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
) -> ApiResponse[DataFileDetail]:
|
|
return success_response(_detail(await _service(request).get_file(user.id, file_id)))
|
|
|
|
|
|
@router.post("/{file_id}/insights")
|
|
async def regenerate_insights(
|
|
file_id: str,
|
|
request: Request,
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
) -> ApiResponse[DataFileDetail]:
|
|
"""重新分析文件，上次分析失败后用；分析在后台进行，返回的 insights 为 pending"""
|
|
return success_response(
|
|
_detail(await _service(request).regenerate_insights(user.id, file_id))
|
|
)
|
|
|
|
|
|
@router.get("/{file_id}/tables/{table}/rows")
|
|
async def get_table_rows(
|
|
file_id: str,
|
|
table: str,
|
|
request: Request,
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
offset: Annotated[int, Query(ge=0, le=MAX_OFFSET)] = 0,
|
|
limit: Annotated[int, Query(ge=1, le=MAX_PAGE_ROWS)] = 100,
|
|
sort_by: Annotated[str | None, Query(min_length=1)] = None,
|
|
order: Annotated[Literal["asc", "desc"], Query()] = "asc",
|
|
) -> ApiResponse[DataTableRows]:
|
|
return success_response(
|
|
await _service(request).get_rows(
|
|
user.id, file_id, table, offset, limit, sort_by, order == "desc"
|
|
)
|
|
)
|
|
|
|
|
|
@router.post("/{file_id}/tables/{table}/aggregate")
|
|
async def aggregate_table(
|
|
file_id: str,
|
|
table: str,
|
|
body: AggregateRequest,
|
|
request: Request,
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
) -> ApiResponse[AggregateResult]:
|
|
return success_response(
|
|
await _service(request).aggregate(user.id, file_id, table, body)
|
|
)
|
|
|
|
|
|
@router.delete("/{file_id}")
|
|
async def delete_file(
|
|
file_id: str,
|
|
request: Request,
|
|
user: Annotated[CurrentUser, Depends(get_current_user)],
|
|
) -> ApiResponse[dict]:
|
|
await _service(request).delete_file(user.id, file_id)
|
|
return success_response()
|