123 lines
3.3 KiB
Python
123 lines
3.3 KiB
Python
"""数据文件的接口模型：文件记录、表、列和解析警告，见 docs/screen-agent/data-files.md 5.4、6.6、7
|
|
|
|
文件和表各有三层，后一层在前一层上加字段：列表接口返回的 Info、单个文件返回的 Detail、存储用的完整记录
|
|
接口返回时用对应的模型重新校验存储记录，多出的字段自然丢掉
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, JsonValue
|
|
|
|
DataFormat = Literal["csv", "tsv", "xlsx", "xls", "xlsb", "json", "jsonl", "parquet"]
|
|
|
|
ColumnType = Literal["text", "number", "date", "datetime", "boolean"]
|
|
|
|
WarningCode = Literal[
|
|
"title_skipped",
|
|
"note_skipped",
|
|
"block_dropped",
|
|
"header_merged",
|
|
"header_uncertain",
|
|
"total_rows_removed",
|
|
"column_renamed",
|
|
"kept_as_text",
|
|
"bad_lines_skipped",
|
|
"json_path_selected",
|
|
"empty_sheet_skipped",
|
|
"hidden_sheet_skipped",
|
|
]
|
|
|
|
|
|
class DataColumn(BaseModel):
|
|
name: str
|
|
type: ColumnType
|
|
# 原文是百分比时为 percent，值已存为小数
|
|
format: Literal["percent"] | None = None
|
|
|
|
|
|
class ParseWarning(BaseModel):
|
|
code: WarningCode
|
|
# 表名；为空表示针对整个文件。解析时还没有分配表名，由上传流程填写
|
|
table: str | None = None
|
|
# 可以直接展示的中文说明
|
|
message: str
|
|
|
|
|
|
class DataTableInfo(BaseModel):
|
|
# 会话内的表名 t1、t2……，模型写 SQL 时用
|
|
table: str
|
|
# 文件名 / sheet，同一 sheet 切出多张表时再加序号
|
|
label: str
|
|
sheet: str | None
|
|
rows: int
|
|
columns: list[DataColumn]
|
|
|
|
|
|
class DataTableDetail(DataTableInfo):
|
|
# 前 20 行，顺序与 columns 一致
|
|
preview: list[list[JsonValue]]
|
|
|
|
|
|
class DataTable(DataTableDetail):
|
|
# 列概况，给 describe_data 用
|
|
profile: dict[str, JsonValue]
|
|
# 文件目录下的 Parquet 文件名
|
|
parquet: str
|
|
|
|
|
|
class DataFileInfo(BaseModel):
|
|
file_id: str
|
|
conversation_id: str
|
|
name: str
|
|
format: DataFormat
|
|
# 文本文件识别出的编码，二进制格式为空
|
|
encoding: str | None
|
|
size: int
|
|
created_at: datetime
|
|
expires_at: datetime
|
|
tables: list[DataTableInfo]
|
|
warnings: list[ParseWarning]
|
|
|
|
|
|
class DataFileDetail(DataFileInfo):
|
|
tables: list[DataTableDetail]
|
|
|
|
|
|
class DataFileRecord(DataFileDetail):
|
|
owner_id: int
|
|
sha256: str
|
|
tables: list[DataTable]
|
|
# 删除只做标记，记录留到过期，会话内的表名不会被新文件重复使用
|
|
deleted: bool = False
|
|
|
|
|
|
class DataFileList(BaseModel):
|
|
files: list[DataFileInfo]
|
|
|
|
|
|
class DataFileLimits(BaseModel):
|
|
max_size_mb: int
|
|
max_files_per_conversation: int
|
|
formats: list[DataFormat]
|
|
# 文件选择框的 accept，只用于前端过滤；后端按文件内容识别格式
|
|
accept: str
|
|
|
|
|
|
class RequestDataTable(DataTableInfo):
|
|
"""本次请求可用的表：多出工具要用的列概况和 Parquet 路径，不带预览"""
|
|
|
|
profile: dict[str, JsonValue]
|
|
path: str
|
|
|
|
|
|
class RequestDataFile(BaseModel):
|
|
"""本次请求可用的文件：服务端按会话读取后放进请求，前端传的同名字段会被覆盖"""
|
|
|
|
file_id: str
|
|
name: str
|
|
# 本轮消息的 attachments 里带了这个文件，通常是刚上传的
|
|
attached_this_turn: bool
|
|
tables: list[RequestDataTable]
|
|
warnings: list[ParseWarning]
|