307 lines
9.8 KiB
Python
307 lines
9.8 KiB
Python
"""数据文件的接口模型：文件记录、表、列和解析警告，见 docs/screen-agent/data-files.md 5.4、6.6、7
|
|
|
|
文件和表各有三层，后一层在前一层上加字段：列表接口返回的 Info、单个文件返回的 Detail、存储用的完整记录
|
|
接口返回时用对应的模型重新校验存储记录，多出的字段自然丢掉
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, JsonValue
|
|
|
|
DataFormat = Literal["csv", "tsv", "xlsx", "xls", "xlsb", "json", "jsonl", "parquet"]
|
|
|
|
ColumnType = Literal["text", "number", "date", "datetime", "boolean"]
|
|
|
|
WarningCode = Literal[
|
|
"title_skipped",
|
|
"note_skipped",
|
|
"block_dropped",
|
|
"header_merged",
|
|
"header_uncertain",
|
|
"total_rows_removed",
|
|
"column_renamed",
|
|
"kept_as_text",
|
|
"bad_lines_skipped",
|
|
"json_path_selected",
|
|
"empty_sheet_skipped",
|
|
"hidden_sheet_skipped",
|
|
]
|
|
|
|
|
|
class DataColumn(BaseModel):
|
|
name: str
|
|
type: ColumnType
|
|
# 原文是百分比时为 percent，值已存为小数
|
|
format: Literal["percent"] | None = None
|
|
|
|
|
|
class ParseWarning(BaseModel):
|
|
code: WarningCode
|
|
# 表名；为空表示针对整个文件。解析时还没有分配表名，由上传流程填写
|
|
table: str | None = None
|
|
# 可以直接展示的中文说明
|
|
message: str
|
|
|
|
|
|
class TopValue(BaseModel):
|
|
value: JsonValue
|
|
count: int
|
|
|
|
|
|
class ColumnProfile(DataColumn):
|
|
"""一列的概况，上传时在全表上算出；不适用的统计为空，交给模型时省略"""
|
|
|
|
null_ratio: float
|
|
# 去重值个数，是估算值
|
|
distinct: int
|
|
# 数值列是数字，日期、日期时间列是 ISO 写法的字符串
|
|
min: int | float | str | None = None
|
|
max: int | float | str | None = None
|
|
# 只有数值列有
|
|
avg: int | float | None = None
|
|
q25: int | float | None = None
|
|
q50: int | float | None = None
|
|
q75: int | float | None = None
|
|
# 去重值不多的文本、布尔列才有，按出现次数从多到少
|
|
top_values: list[TopValue] | None = None
|
|
# 开头几行里不同的取值
|
|
examples: list[JsonValue]
|
|
# 保留为文本的原因等说明
|
|
note: str | None = None
|
|
|
|
|
|
class TableProfile(BaseModel):
|
|
"""表的列概况，给 describe_data 和上传后的分析用"""
|
|
|
|
columns: list[ColumnProfile]
|
|
# 随机抽的几行，顺序与 columns 一致
|
|
sample: list[list[JsonValue]]
|
|
|
|
|
|
class DataTableInfo(BaseModel):
|
|
# 会话内的表名 t1、t2……，模型写 SQL 时用
|
|
table: str
|
|
# 文件名 / sheet，同一 sheet 切出多张表时再加序号
|
|
label: str
|
|
sheet: str | None
|
|
rows: int
|
|
columns: list[DataColumn]
|
|
|
|
|
|
class DataTableDetail(DataTableInfo):
|
|
# 前 20 行，顺序与 columns 一致
|
|
preview: list[list[JsonValue]]
|
|
|
|
|
|
class DataTable(DataTableDetail):
|
|
profile: TableProfile
|
|
# 文件目录下的 Parquet 文件名
|
|
parquet: str
|
|
|
|
|
|
Aggregation = Literal["sum", "avg", "min", "max", "count", "count_distinct"]
|
|
|
|
# 日期列按粒度分组，结果是字符串：年 2025，季度 2025-Q1，月 2025-01，日 2025-01-31
|
|
Grain = Literal["year", "quarter", "month", "day"]
|
|
|
|
MAX_GROUP_BY = 2
|
|
MAX_METRICS = 5
|
|
MAX_AGGREGATE_ROWS = 1000
|
|
|
|
|
|
class GroupBy(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
column: str
|
|
# 只有日期、日期时间列可以指定
|
|
grain: Grain | None = None
|
|
|
|
|
|
class Metric(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
agg: Aggregation
|
|
# count 可以不写，数整张表的行数；其他聚合必填。sum、avg 只能用于数值列
|
|
column: str | None = None
|
|
# 结果列名，默认沿用列名（计数类加后缀），按需指定以避免重名
|
|
alias: str | None = Field(default=None, min_length=1, max_length=100)
|
|
|
|
|
|
class OrderBy(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
# 必须是结果里的列名（分组列或指标的别名）
|
|
by: str
|
|
direction: Literal["asc", "desc"] = "asc"
|
|
|
|
|
|
class AggregateRequest(BaseModel):
|
|
"""对一张表分组汇总：group_by 为空时整表汇总成一行"""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
group_by: list[GroupBy] = Field(default_factory=list, max_length=MAX_GROUP_BY)
|
|
metrics: list[Metric] = Field(min_length=1, max_length=MAX_METRICS)
|
|
# 不写时有分组按第一个分组列升序
|
|
order_by: OrderBy | None = None
|
|
limit: int = Field(default=100, ge=1, le=MAX_AGGREGATE_ROWS)
|
|
|
|
|
|
class AggregateResult(BaseModel):
|
|
# 结果列名，按分组列、指标的顺序
|
|
columns: list[str]
|
|
# 对象数组，键是列名，可直接作为图表物料的 dataset.source
|
|
rows: list[dict[str, JsonValue]]
|
|
# 结果超过 limit 行，只返回了前 limit 行
|
|
truncated: bool
|
|
|
|
|
|
# 前端图表物料支持绑定汇总数据的系列类型，推荐只会给出这几种
|
|
ChartKind = Literal["bar", "line", "pie", "funnel", "pictorialBar"]
|
|
|
|
ColumnRole = Literal["dimension", "measure", "time", "id", "other"]
|
|
|
|
MAX_SUGGESTIONS = 3
|
|
|
|
|
|
class ChartSuggestion(BaseModel):
|
|
"""一条图表推荐：一个可以直接回答的问题，加上能直接跑的汇总请求"""
|
|
|
|
# 给用户看的问题，如“各区域销售额对比”，也作为图表标题
|
|
title: str
|
|
# 针对哪张表，是文件里的表名（t1、t2……）
|
|
table: str
|
|
chart: ChartKind
|
|
request: AggregateRequest
|
|
|
|
|
|
class ColumnInsight(BaseModel):
|
|
table: str
|
|
# 原始列名，与 columns[].name 对应
|
|
name: str
|
|
# 中文含义，列名已经够清楚时与列名相同
|
|
label: str
|
|
role: ColumnRole
|
|
# 单位，如“元”“%”“件”；没有或不确定时为空
|
|
unit: str | None = None
|
|
|
|
|
|
class FileInsights(BaseModel):
|
|
"""上传后由 LLM 一次分析出的文件洞察：一句话说明、图表推荐、列的含义
|
|
|
|
后续要加的分析结果都往这里加带默认值的字段，version 随字段含义变化递增
|
|
"""
|
|
|
|
version: int = 1
|
|
# pending：正在分析；ready：已完成（推荐可能为空）；failed：分析失败，文件本身可用
|
|
status: Literal["pending", "ready", "failed"]
|
|
# pending 时是开始分析的时间；服务启动前开始、至今仍是 pending 的分析已随重启丢失，读取时按失败返回
|
|
started_at: datetime | None = None
|
|
summary: str = ""
|
|
suggestions: list[ChartSuggestion] = Field(default_factory=list)
|
|
columns: list[ColumnInsight] = Field(default_factory=list)
|
|
|
|
|
|
class DataFileInfo(BaseModel):
|
|
file_id: str
|
|
conversation_id: str
|
|
name: str
|
|
format: DataFormat
|
|
# 文本文件识别出的编码，二进制格式为空
|
|
encoding: str | None
|
|
size: int
|
|
created_at: datetime
|
|
expires_at: datetime
|
|
tables: list[DataTableInfo]
|
|
warnings: list[ParseWarning]
|
|
# 未启用智能分析，或这是功能上线前上传的文件时为空
|
|
insights: FileInsights | None = None
|
|
|
|
|
|
class DataFileDetail(DataFileInfo):
|
|
tables: list[DataTableDetail]
|
|
|
|
|
|
class DataFileRecord(DataFileDetail):
|
|
owner_id: int
|
|
sha256: str
|
|
tables: list[DataTable]
|
|
# 删除只做标记，记录留到过期，会话内的表名不会被新文件重复使用
|
|
deleted: bool = False
|
|
|
|
|
|
UploadStatus = Literal["queued", "parsing", "ready", "failed"]
|
|
|
|
UploadStage = Literal["detecting", "reading", "building"]
|
|
|
|
|
|
class UploadError(BaseModel):
|
|
# 与同步接口失败时的 HTTP 状态码和 msg 一致，如 422 + 解析失败的原因
|
|
code: int
|
|
msg: str
|
|
|
|
|
|
class UploadProgress(BaseModel):
|
|
"""一次上传的解析进度；上传接口、进度接口和文件列表里的 uploads 都是这个结构"""
|
|
|
|
file_id: str
|
|
name: str
|
|
# queued 排队中；parsing 解析中；ready 已完成，文件记录已可查；failed 失败，原因在 error
|
|
status: UploadStatus
|
|
# 解析完成的比例，0 到 1，只增不减；ready 时为 1，解析中不会到 1
|
|
progress: float
|
|
# 解析中所处的环节；其他状态为空
|
|
stage: UploadStage | None
|
|
# 预计还要多少秒；排队中、刚开始解析、已结束时为空
|
|
eta_seconds: float | None
|
|
# 排队中时，前面还有几个文件在排队；其他状态为 0
|
|
queued_ahead: int
|
|
error: UploadError | None
|
|
|
|
|
|
class DataFileList(BaseModel):
|
|
files: list[DataFileInfo]
|
|
# 本会话里排队、解析中和刚失败的上传，刷新页面后据此恢复轮询；完成的已在 files 里
|
|
uploads: list[UploadProgress] = Field(default_factory=list)
|
|
|
|
|
|
class DataTableRows(BaseModel):
|
|
"""一张表的一页数据，用于预览时翻页看全表"""
|
|
|
|
# 表的总行数（与文件记录里的 rows 一致），前端据此决定滚动长度
|
|
total: int
|
|
offset: int
|
|
# 这一页的行，顺序与 columns 一致；到末尾时可能少于请求的 limit
|
|
rows: list[list[JsonValue]]
|
|
|
|
|
|
class DataFileLimits(BaseModel):
|
|
max_size_mb: int
|
|
max_files_per_conversation: int
|
|
formats: list[DataFormat]
|
|
# 文件选择框的 accept，只用于前端过滤；后端按文件内容识别格式
|
|
accept: str
|
|
# 同时解析的文件数，其余排队；前端据此决定并发上传几个
|
|
parse_concurrency: int
|
|
# 翻页取表数据时单页行数上限
|
|
rows_max_page: int
|
|
# 分组汇总的指标个数上限、返回行数上限
|
|
aggregate_max_metrics: int
|
|
aggregate_max_rows: int
|
|
|
|
|
|
class RequestDataFile(BaseModel):
|
|
"""本次请求可用的文件：服务端按会话读取后放进请求，前端传的同名字段会被覆盖
|
|
|
|
请求会进检查点，所以只放表名、行数和列；列概况和文件路径在图的运行上下文里，见 processor/state.py
|
|
"""
|
|
|
|
file_id: str
|
|
name: str
|
|
# 本轮消息的 attachments 里带了这个文件，通常是刚上传的
|
|
attached_this_turn: bool
|
|
tables: list[DataTableInfo]
|
|
# 上传后的分析结果，只有 ready 的才有；给模型一句说明和列的含义，没有时和以前完全一样
|
|
insights: FileInsights | None = None
|