From bb6a9405db49d2245eda1f559acd0f20d7a50a90 Mon Sep 17 00:00:00 2001 From: Marcus <1922576605@qq.com> Date: Thu, 8 Oct 2026 08:23:01 -0700 Subject: [PATCH] =?UTF-8?q?feat=EF=BC=9A=E5=AE=8C=E6=88=90=E5=A4=A7?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=95=B0=E6=8D=AE=E6=96=87=E4=BB=B6=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E8=A7=A3=E6=9E=90=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 7 + app/agent/data_tools.py | 126 ++++++++ app/agent/screen_agent.py | 26 +- app/agent/tools.py | 4 +- app/api/files.py | 95 ++++++ app/config.py | 30 +- app/datafiles/__init__.py | 0 app/datafiles/base.py | 32 +++ app/datafiles/columns.py | 349 +++++++++++++++++++++++ app/datafiles/detect.py | 143 ++++++++++ app/datafiles/parse.py | 141 +++++++++ app/datafiles/profile.py | 126 ++++++++ app/datafiles/query.py | 168 +++++++++++ app/datafiles/readers.py | 240 ++++++++++++++++ app/datafiles/service.py | 334 ++++++++++++++++++++++ app/datafiles/storage.py | 152 ++++++++++ app/datafiles/tables.py | 290 +++++++++++++++++++ app/datafiles/worker.py | 27 ++ app/main.py | 18 +- app/processor/nodes/screen_agent_node.py | 9 +- app/processor/summary.py | 30 +- app/schemas/data_file.py | 122 ++++++++ app/schemas/input_context.py | 3 + app/schemas/request.py | 3 +- app/services/agent.py | 10 +- pyproject.toml | 8 + uv.lock | 200 +++++++++++++ 27 files changed, 2675 insertions(+), 18 deletions(-) create mode 100644 app/agent/data_tools.py create mode 100644 app/api/files.py create mode 100644 app/datafiles/__init__.py create mode 100644 app/datafiles/base.py create mode 100644 app/datafiles/columns.py create mode 100644 app/datafiles/detect.py create mode 100644 app/datafiles/parse.py create mode 100644 app/datafiles/profile.py create mode 100644 app/datafiles/query.py create mode 100644 app/datafiles/readers.py create mode 100644 app/datafiles/service.py create mode 100644 app/datafiles/storage.py create mode 100644 app/datafiles/tables.py create mode 100644 app/datafiles/worker.py create mode 100644 app/schemas/data_file.py diff --git a/.env.example b/.env.example index 0940792..a8da072 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,13 @@ MONGODB_DB=screen_agent # 检查点保留时长（秒），默认 7 天，0 表示永久保留 MONGODB_CHECKPOINT_TTL=604800 +# 上传的数据文件：存放目录（相对路径相对项目根目录，容器部署时挂载出来）、单个文件大小上限（MB）、保留时长（秒，默认 7 天） +DATA_FILE_DIR=data/files +DATA_FILE_MAX_MB=50 +DATA_FILE_TTL=604800 +# 模型写的单条数据查询的超时（秒） +DATA_QUERY_TIMEOUT=10 + # 用户中心 USER_CENTER_URL=http://127.0.0.1:8100 USER_CENTER_TIMEOUT_SECONDS=5 diff --git a/app/agent/data_tools.py b/app/agent/data_tools.py new file mode 100644 index 0000000..dfd1eec --- /dev/null +++ b/app/agent/data_tools.py @@ -0,0 +1,126 @@ +"""数据文件的本地工具：describe_data 查看列概况，query_data 执行只读 SQL；问答和 screen agent 共用 + +工具只能看到 DataContext 里的表，也就是当前用户在本会话里未过期的文件，查询引擎也只为这些表建视图 +工具通过 runtime.context.data 读取 DataContext，用到它们的 agent 上下文都要有 data 字段； +上下文类型因 agent 而异，所以写 ToolRuntime[Any]（不写类型参数时按 None 处理，序列化会告警） +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated, Any + +from langchain.tools import ToolRuntime, tool +from langchain_core.messages import ToolMessage +from pydantic import Field + +from app.config import settings +from app.datafiles.query import QueryError, run_query +from app.schemas.data_file import RequestDataFile, RequestDataTable + +# 列超过这么多时，不指定 columns 只返回列名和类型，避免一次塞进太多概况 +BRIEF_COLUMNS = 50 + + +@dataclass(frozen=True) +class DataContext: + files: tuple[RequestDataFile, ...] = () + + def find(self, name: str) -> tuple[RequestDataFile, RequestDataTable] | None: + for file in self.files: + for table in file.tables: + if table.table == name: + return file, table + return None + + def paths(self) -> dict[str, Path]: + return { + table.table: Path(table.path) + for file in self.files + for table in file.tables + } + + +def _error(runtime: ToolRuntime, message: str) -> ToolMessage: + # 返回给模型修正，不中断 agent + return ToolMessage(message, tool_call_id=runtime.tool_call_id, status="error") + + +def _table_list(data: DataContext) -> str: + return "、".join(data.paths()) or "无" + + +@tool +def describe_data( + table: str, + runtime: ToolRuntime[Any], + columns: Annotated[ + list[str] | None, + Field(description="只看这些列；不传时看全部列"), + ] = None, +) -> dict[str, Any] | ToolMessage: + """查看一张数据表的列概况（类型、空值率、去重值个数、数值范围和四分位、常见值、样例）、随机样例行，以及解析时做的调整 + + 分析或取数前先看要用的表；表名见请求 data_files 里的 table。统计值来自全表，样例只是其中几行 + 列很多时只返回列名和类型，再用 columns 指定要看的列 + """ + data: DataContext = runtime.context.data + found = data.find(table) + if found is None: + return _error(runtime, f"没有表 {table}，可用的表：{_table_list(data)}") + file, item = found + profiled: list[dict[str, Any]] = item.profile["columns"] + result: dict[str, Any] = { + "table": item.table, + "label": item.label, + "rows": item.rows, + } + if columns is None and len(profiled) > BRIEF_COLUMNS: + result["columns"] = [ + {key: column[key] for key in ("name", "type", "format") if key in column} + for column in profiled + ] + result["note"] = ( + f"共 {len(profiled)} 列，只列出列名和类型；用 columns 指定列查看详细概况和样例" + ) + else: + index = {column["name"]: position for position, column in enumerate(profiled)} + names = list(index) if columns is None else list(dict.fromkeys(columns)) + positions = [index[name] for name in names if name in index] + result["columns"] = [profiled[position] for position in positions] + # 样例行的值和 columns 一一对应 + result["sample_rows"] = [ + [row[position] for position in positions] for row in item.profile["sample"] + ] + if missing := [name for name in names if name not in index]: + result["missing_columns"] = missing + # 针对这张表的警告，以及针对整个文件的警告（例如跳过的坏行） + result["warnings"] = [ + warning.message + for warning in file.warnings + if warning.table in (item.table, None) + ] + return result + + +@tool +def query_data(sql: str, runtime: ToolRuntime[Any]) -> dict[str, Any] | ToolMessage: + """对上传的数据表执行一条只读 SQL（DuckDB 语法），表名是请求 data_files 里的 table（t1、t2……） + + 中文或带符号的列名用双引号括起来；宽表转长表用 UNPIVOT，PIVOT 要用 IN 写明取值 + 最多返回 200 行，truncated 为 true 时结果不完整，改用聚合或 LIMIT + 回答和图表里的数字以查询结果为准，不要自己心算 + """ + data: DataContext = runtime.context.data + paths = data.paths() + # 任务执行期间被删除或清理的文件跳过，不影响查询其他表 + existing = {name: path for name, path in paths.items() if path.exists()} + try: + return run_query(sql, existing, timeout=settings.data_query_timeout).to_dict() + except QueryError as error: + message = str(error) + if gone := [name for name in paths if name not in existing]: + message += f"；{'、'.join(gone)} 的文件已删除或过期，不能再查询" + return _error(runtime, message) + + +DATA_TOOLS = [describe_data, query_data] diff --git a/app/agent/screen_agent.py b/app/agent/screen_agent.py index 34b58d5..aa5d6b1 100644 --- a/app/agent/screen_agent.py +++ b/app/agent/screen_agent.py @@ -5,7 +5,7 @@ 组件目录和物料在物料 MCP，各类任务的做法在 skill；系统提示只放每次都要用、别处放不下的原则 """ -from typing import Literal, Self +from typing import Any, Literal, Self from langchain.agents import create_agent from langchain.agents.middleware import ModelCallLimitMiddleware, wrap_model_call @@ -14,6 +14,7 @@ from langchain_core.messages import AIMessage, HumanMessage from loguru import logger from pydantic import BaseModel, Field, model_validator +from app.agent.data_tools import DATA_TOOLS from app.agent.skills import skills_prompt from app.agent.tools import ( ScreenAgentContext, @@ -98,16 +99,22 @@ async def require_tool_call(request, handler): return response -_agent = None +# 是否挂数据工具 → agent +_agents: dict[bool, Any] = {} -async def get_screen_agent(): - """第一次使用时组装，之后复用；系统提示是常量，不随物料目录变化""" - global _agent - if _agent is None: - _agent = create_agent( +async def get_screen_agent(with_data: bool = False): + """第一次使用时组装，之后复用；系统提示是常量，不随物料目录变化 + + 会话里有数据文件时用挂了 describe_data、query_data 的实例，没有时不让模型看到用不了的工具 + """ + if with_data not in _agents: + tools = [get_nodes, apply_actions, *await get_mcp_tools()] + if with_data: + tools.extend(DATA_TOOLS) + _agents[with_data] = create_agent( model=get_llm(), - tools=[get_nodes, apply_actions, *await get_mcp_tools()], + tools=tools, system_prompt=f"{SYSTEM_PROMPT}\n\n{skills_prompt()}", # 用工具调用给出结论：试过原生结构化输出（ProviderStrategy），歧义时模型更常直接执行而不追问，评测多删了组件 response_format=ToolStrategy(AgentReply), @@ -124,5 +131,4 @@ async def get_screen_agent(): checkpointer=False, name="screen_agent", ) - - return _agent + return _agents[with_data] diff --git a/app/agent/tools.py b/app/agent/tools.py index 96f3e3a..8226376 100644 --- a/app/agent/tools.py +++ b/app/agent/tools.py @@ -3,7 +3,7 @@ 它们读写的是本次请求的大屏和 agent 状态，所以放在本地；物料等能力由 MCP 提供 """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Annotated, NotRequired from langchain.agents import AgentState @@ -13,6 +13,7 @@ from langgraph.types import Command from pydantic import Field from app.actions.batch import BatchError, run_batch +from app.agent.data_tools import DataContext from app.schemas.actions import Action, AddAction, AgentAction from app.schemas.component_resource import ComponentCatalog from app.schemas.screen import Screen @@ -22,6 +23,7 @@ from app.schemas.screen import Screen class ScreenAgentContext: screen: Screen # 本次请求的原始大屏，只读 catalog: ComponentCatalog # 当前物料目录，只用于补全和校验 + data: DataContext = field(default_factory=DataContext) # 本会话可用的数据表 def _latest(_, new): diff --git a/app/api/files.py b/app/api/files.py new file mode 100644 index 0000000..4538938 --- /dev/null +++ b/app/api/files.py @@ -0,0 +1,95 @@ +"""数据文件接口：上传限制、上传解析、列出、查看、删除，见 docs/screen-agent/data-files.md 第 5 节""" + +from typing import Annotated, get_args + +from fastapi import APIRouter, Depends, File, Form, Query, Request, UploadFile + +from app.common.response import ApiResponse, success_response +from app.core.auth import get_current_user +from app.datafiles.service import MAX_FILES_PER_CONVERSATION, DataFileService +from app.schemas.base import NonBlankString +from app.schemas.data_file import ( + DataFileDetail, + DataFileInfo, + DataFileLimits, + DataFileList, + DataFileRecord, + DataFormat, +) +from app.schemas.user import CurrentUser + +# 上传限制不需要登录，单独一个路由；要先于 /files/{file_id} 注册 +limits_router = APIRouter(prefix="/files", tags=["DataFile"]) +router = APIRouter(prefix="/files", tags=["DataFile"]) + +# 文件选择框的过滤条件；后端按文件内容识别格式，不看扩展名 +ACCEPT = ".csv,.tsv,.txt,.xlsx,.xls,.xlsb,.json,.jsonl,.ndjson,.parquet" + + +def _service(request: Request) -> DataFileService: + return request.app.state.data_file_service + + +def _detail(record: DataFileRecord) -> DataFileDetail: + # 按接口模型重新校验，去掉 owner_id、sha256、列概况等只在后端用的字段 + return DataFileDetail.model_validate(record.model_dump()) + + +@limits_router.get("/limits") +async def get_limits(request: Request) -> ApiResponse[DataFileLimits]: + return success_response( + DataFileLimits( + max_size_mb=_service(request).max_mb, + max_files_per_conversation=MAX_FILES_PER_CONVERSATION, + formats=list(get_args(DataFormat)), + accept=ACCEPT, + ) + ) + + +@router.post("") +async def upload_file( + request: Request, + conversation_id: Annotated[NonBlankString, Form()], + file: Annotated[UploadFile, File()], + user: Annotated[CurrentUser, Depends(get_current_user)], +) -> ApiResponse[DataFileDetail]: + record = await _service(request).upload( + user.id, conversation_id, file.filename, file.file + ) + return success_response(_detail(record)) + + +@router.get("") +async def list_files( + request: Request, + conversation_id: Annotated[NonBlankString, Query()], + user: Annotated[CurrentUser, Depends(get_current_user)], +) -> ApiResponse[DataFileList]: + records = await _service(request).list_files(user.id, conversation_id) + return success_response( + DataFileList( + files=[ + DataFileInfo.model_validate(record.model_dump()) for record in records + ] + ) + ) + + +@router.get("/{file_id}") +async def get_file( + file_id: str, + request: Request, + user: Annotated[CurrentUser, Depends(get_current_user)], +) -> ApiResponse[DataFileDetail]: + return success_response(_detail(await _service(request).get_file(user.id, file_id))) + + +@router.delete("/{file_id}") +async def delete_file( + file_id: str, + request: Request, + user: Annotated[CurrentUser, Depends(get_current_user)], +) -> ApiResponse[dict]: + await _service(request).delete_file(user.id, file_id) + return success_response() diff --git a/app/config.py b/app/config.py index 8ac2ee0..6a630e9 100644 --- a/app/config.py +++ b/app/config.py @@ -4,7 +4,9 @@ from pathlib import Path from dotenv import load_dotenv -load_dotenv(Path(__file__).resolve().parent.parent / ".env") +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +load_dotenv(PROJECT_ROOT / ".env") @dataclass(frozen=True) @@ -87,6 +89,26 @@ class Settings: default_factory=lambda: int(os.getenv("MONGODB_CHECKPOINT_TTL", "604800")) ) + # 上传的数据文件：原文件和解析出的 Parquet 放这个目录，相对路径相对项目根目录；容器部署时挂载出来 + data_file_dir: Path = field( + default_factory=lambda: ( + PROJECT_ROOT / (os.getenv("DATA_FILE_DIR") or "data/files") + ) + ) + # 单个数据文件的大小上限（MB） + data_file_max_mb: int = field( + default_factory=lambda: int(os.getenv("DATA_FILE_MAX_MB", "50")) + ) + # 数据文件保留时长（秒），从上传时算起，默认 7 天 + data_file_ttl: int = field( + default_factory=lambda: int(os.getenv("DATA_FILE_TTL", "604800")) + ) + + # 模型写的单条数据查询的超时（秒） + data_query_timeout: float = field( + default_factory=lambda: float(os.getenv("DATA_QUERY_TIMEOUT", "10")) + ) + # 用户中心：请求携带的 token 交给它校验，换回 user_id user_center_url: str = field( default_factory=lambda: ( @@ -125,6 +147,12 @@ class Settings: raise ValueError("MAX_AGENT_STEPS 必须是正整数") if self.mcp_tool_timeout <= 0: raise ValueError("MCP_TOOL_TIMEOUT 必须是正数") + if self.data_file_max_mb < 1: + raise ValueError("DATA_FILE_MAX_MB 必须是正整数") + if self.data_file_ttl < 1: + raise ValueError("DATA_FILE_TTL 必须是正整数") + if self.data_query_timeout <= 0: + raise ValueError("DATA_QUERY_TIMEOUT 必须是正数") if not 0 <= self.intent_qa_min_probability <= 1: raise ValueError("INTENT_QA_MIN_PROBABILITY 必须在 0 到 1 之间") diff --git a/app/datafiles/__init__.py b/app/datafiles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/datafiles/base.py b/app/datafiles/base.py new file mode 100644 index 0000000..4896947 --- /dev/null +++ b/app/datafiles/base.py @@ -0,0 +1,32 @@ +"""数据文件各环节共用的异常和上限""" + +# 每张表的数据行数上限 +MAX_TABLE_ROWS = 1_000_000 + +UNSUPPORTED_FORMAT = "不支持的文件格式，支持 CSV、TSV、Excel、JSON、JSONL、Parquet" + + +class DataFileError(Exception): + """文件无法接收或解析；status 是返回给调用方的 HTTP 状态码，message 可以直接展示 + + 解析在子进程里执行，异常要能跨进程传递，所以参数原样放进 args + """ + + def __init__(self, status: int, message: str): + super().__init__(status, message) + self.status = status + self.message = message + + def __str__(self) -> str: + return self.message + + +def where(sheet: str | None) -> str: + """警告和错误信息的位置前缀：Excel 写 sheet 名，其他格式不写""" + return f"{sheet}：" if sheet else "" + + +def rows_exceeded(sheet: str | None) -> DataFileError: + return DataFileError( + 422, f"{where(sheet)}超过 {MAX_TABLE_ROWS // 10_000} 万行，请拆分后上传" + ) diff --git a/app/datafiles/columns.py b/app/datafiles/columns.py new file mode 100644 index 0000000..7077cb9 --- /dev/null +++ b/app/datafiles/columns.py @@ -0,0 +1,349 @@ +"""列名清理和类型推断，输出 Arrow 列 + +CSV 的单元格全是字符串；Excel 的数值、日期单元格直接采用，文本单元格再按规则推断 +一列的非空值必须全部符合同一类型才转换，否则保留文本：宁可让模型在 SQL 里 TRY_CAST，也不悄悄丢掉值 +""" + +import json +import math +import re +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, date, datetime, time +from typing import Any + +import pyarrow as pa + +from app.datafiles.base import where +from app.schemas.data_file import DataColumn, ParseWarning + +NULL_TOKENS = {"", "-", "--", "—", "——", "N/A", "n/a", "NULL", "null", "#N/A"} +EXCEL_ERROR = re.compile(r"^#(?:NULL!|DIV/0!|VALUE!|REF!|NAME\?|NUM!|N/A)$") + +CURRENCY_SYMBOLS = "¥￥$€" +NUMBER = re.compile(r"^[+-]?(?:\d+|\d{1,3}(?:,\d{3})+)(?:\.\d+)?(?:[eE][+-]?\d+)?$") +INTEGER = re.compile(r"^[+-]?\d+$") +# 16 位及以上的纯数字是证件号、卡号一类的编号，转成数值会丢失精度 +MIN_ID_DIGITS = 16 + +# 按年份后的第一个字符选格式；%m、%d 也接受一位数 +DATE_FORMATS = { + "-": ("%Y-%m-%d", "%Y-%m"), + "/": ("%Y/%m/%d", "%Y/%m"), + ".": ("%Y.%m.%d",), + "年": ("%Y年%m月%d日", "%Y年%m月"), +} +DATETIME_FORMATS = ( + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y/%m/%d %H:%M:%S", + "%Y/%m/%d %H:%M", +) + +INT64_RANGE = range(-(2**63), 2**63) +# 超过这个范围的整数存成 float 会丢精度 +EXACT_FLOAT_INT = 2**53 + +NUMERIC_KINDS = ("int", "float", "percent") +DATE_KINDS = ("date", "datetime") + + +@dataclass +class TypedColumns: + columns: list[DataColumn] + table: pa.Table + # 列名 → 写进列概况的说明，例如保留为文本的原因 + notes: dict[str, str] + + +def clean_names( + raw: Sequence[Any], sheet: str | None +) -> tuple[list[str], ParseWarning | None]: + """去掉空白和换行；空列名按位置命名，重复列名加后缀""" + names, changes, seen = [], [], set() + for position, value in enumerate(raw, start=1): + name = re.sub(r"\s*[\r\n]+\s*", " ", label_text(value)).strip() + if not name: + name = f"列{position}" + changes.append(f"第 {position} 列没有列名，命名为「{name}」") + if name in seen: + suffix = 2 + while f"{name}_{suffix}" in seen: + suffix += 1 + changes.append(f"「{name}」重名，第 {position} 列改为「{name}_{suffix}」") + name = f"{name}_{suffix}" + seen.add(name) + names.append(name) + if not changes: + return names, None + shown = "；".join(changes[:5]) + ("等" if len(changes) > 5 else "") + return names, ParseWarning(code="column_renamed", message=f"{where(sheet)}{shown}") + + +def label_text(value: Any) -> str: + """表头单元格转成列名；Excel 里年份等数字表头读出来是 2025.0""" + if value is None: + return "" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + if isinstance(value, (date, time)): + return value.isoformat() + return str(value) + + +def build_columns( + names: list[str], values: list[Sequence[Any]], sheet: str | None +) -> tuple[TypedColumns, ParseWarning | None]: + """按列推断类型；有列因少数无法解析的值保留为文本时，返回一条 kept_as_text 警告""" + columns, arrays, notes = [], [], {} + for name, column_values in zip(names, values): + column, array, note = infer_column(name, column_values) + columns.append(column) + arrays.append(array) + if note: + notes[name] = note + warning = None + if notes: + listed = "、".join(f"「{name}」" for name in notes) + warning = ParseWarning( + code="kept_as_text", + message=f"{where(sheet)}{listed}有无法解析的值，整列按文本处理", + ) + return TypedColumns( + columns, pa.Table.from_arrays(arrays, names=names), notes + ), warning + + +def arrow_columns(table: pa.Table) -> tuple[TypedColumns, ParseWarning | None]: + """Parquet 自带类型，只映射到接口类型；列表、嵌套结构等存成 JSON 字符串""" + names, warning = clean_names(table.column_names, None) + columns, arrays = [], [] + for name, array in zip(names, table.columns): + kind = _arrow_kind(array.type) + if kind is None: + values = array.to_pylist() + array = pa.array( + [None if v is None else as_text(v) for v in values], pa.string() + ) + kind = "text" + elif kind == "text" and pa.types.is_dictionary(array.type): + array = array.cast(pa.string()) + columns.append(DataColumn(name=name, type=kind)) + arrays.append(array) + return TypedColumns(columns, pa.Table.from_arrays(arrays, names=names), {}), warning + + +def _arrow_kind(arrow_type: pa.DataType) -> str | None: + types = pa.types + if ( + types.is_integer(arrow_type) + or types.is_floating(arrow_type) + or types.is_decimal(arrow_type) + ): + return "number" + if types.is_boolean(arrow_type): + return "boolean" + if types.is_date(arrow_type): + return "date" + if types.is_timestamp(arrow_type): + return "datetime" + if ( + types.is_string(arrow_type) + or types.is_large_string(arrow_type) + or types.is_string_view(arrow_type) + ): + return "text" + if types.is_dictionary(arrow_type) and _arrow_kind(arrow_type.value_type) == "text": + return "text" + return None + + +def infer_column( + name: str, values: Sequence[Any] +) -> tuple[DataColumn, pa.Array, str | None]: + value_kinds, parsed = [], [] + for value in values: + kind, result = classify(value) + value_kinds.append(kind) + parsed.append(result) + kinds = Counter(kind for kind in value_kinds if kind != "null") + present = set(kinds) + + if present and present <= {"bool"}: + return DataColumn(name=name, type="boolean"), pa.array(parsed, pa.bool_()), None + if present and present <= {"int", "float"}: + if all( + value is None + or isinstance(value, int) + or (value.is_integer() and abs(value) < EXACT_FLOAT_INT) + for value in parsed + ): + ints = [None if value is None else int(value) for value in parsed] + return ( + DataColumn(name=name, type="number"), + pa.array(ints, pa.int64()), + None, + ) + return ( + DataColumn(name=name, type="number"), + pa.array(parsed, pa.float64()), + None, + ) + if present == {"percent"}: + column = DataColumn(name=name, type="number", format="percent") + return column, pa.array(parsed, pa.float64()), None + if present and present <= {"date"}: + return DataColumn(name=name, type="date"), pa.array(parsed, pa.date32()), None + if present and present <= set(DATE_KINDS): + stamps = [ + datetime.combine(value, time()) if type(value) is date else value + for value in parsed + ] + return ( + DataColumn(name=name, type="datetime"), + pa.array(stamps, pa.timestamp("us")), + None, + ) + + texts = [ + None if result is None else as_text(value) + for value, result in zip(values, parsed) + ] + note = _text_note(texts, value_kinds, kinds) + return DataColumn(name=name, type="text"), pa.array(texts, pa.string()), note + + +def classify(value: Any) -> tuple[str, Any]: + """单个值的类型和转换后的值；类型是 null、bool、int、float、percent、date、datetime、text 之一""" + if value is None: + return "null", None + if isinstance(value, bool): + return "bool", value + if isinstance(value, int): + return ("int", value) if value in INT64_RANGE else ("text", str(value)) + if isinstance(value, float): + return ( + ("null", None) + if math.isnan(value) or math.isinf(value) + else ("float", value) + ) + if isinstance(value, datetime): + # Excel 只设了日期格式的单元格读出来是零点的 datetime + if value.time() == time() and value.tzinfo is None: + return "date", value.date() + return "datetime", _naive(value) + if isinstance(value, date): + return "date", value + if not isinstance(value, str): + return "text", as_text(value) + + text = value.strip() + if text in NULL_TOKENS or EXCEL_ERROR.match(text): + return "null", None + if text.lower() in ("true", "false"): + return "bool", text.lower() == "true" + number = parse_number(text) + if number is not None: + return ("int" if isinstance(number, int) else "float"), number + if text.endswith("%"): + number = parse_number(text[:-1].rstrip()) + if number is not None: + return "percent", number / 100 + moment = parse_datetime(text) + if moment is not None: + return ("datetime" if isinstance(moment, datetime) else "date"), moment + return "text", text + + +def parse_number(text: str) -> int | float | None: + """千分位和货币符号可以有；有前导零或 16 位以上的数字串是编号，不算数值""" + # 最常见的纯数字走快速路径，不用正则 + if text.isdigit(): + if len(text) >= MIN_ID_DIGITS or (len(text) > 1 and text[0] == "0"): + return None + return int(text) + if text and text[0] in CURRENCY_SYMBOLS: + text = text[1:].lstrip() + if not NUMBER.match(text): + return None + digits = text.lstrip("+-") + if len(digits) > 1 and digits[0] == "0" and digits[1].isdigit(): + return None + if digits.isdigit() and len(digits) >= MIN_ID_DIGITS: + return None + text = text.replace(",", "") + if INTEGER.match(text): + return int(text) + return float(text) + + +def parse_datetime(text: str) -> date | datetime | None: + if len(text) < 6 or not text[:4].isdigit(): + return None + # 标准 ISO 写法先用 fromisoformat，比 strptime 快一个数量级 + if ":" in text: + try: + return _naive(datetime.fromisoformat(text)) + except ValueError: + pass + for pattern in DATETIME_FORMATS: + try: + return datetime.strptime(text, pattern) + except ValueError: + continue + return None + if len(text) == 10 and text[4] == "-": + try: + return date.fromisoformat(text) + except ValueError: + pass + for pattern in DATE_FORMATS.get(text[4], ()): + try: + return datetime.strptime(text, pattern).date() + except ValueError: + continue + return None + + +def as_text(value: Any) -> str: + if isinstance(value, str): + return value.strip() + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + if isinstance(value, datetime): + return value.isoformat(sep=" ") + if isinstance(value, (date, time)): + return value.isoformat() + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def _naive(value: datetime) -> datetime: + # 带时区的时间统一换成 UTC 再去掉时区，同一列里的值才能比较 + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + +def _text_note( + texts: list[str | None], value_kinds: list[str], kinds: Counter[str] +) -> str | None: + """大部分值是数值或日期、少数无法解析时，说明原因，模型可以在 SQL 里用 TRY_CAST""" + failed = kinds["text"] + numeric = sum(kinds[kind] for kind in NUMERIC_KINDS) + dates = sum(kinds[kind] for kind in DATE_KINDS) + if not failed or max(numeric, dates) < failed: + return None + examples: list[str] = [] + for text, kind in zip(texts, value_kinds): + if kind == "text" and text not in examples: + examples.append(text) + if len(examples) == 2: + break + shown = "、".join(f"「{example}」" for example in examples) + label = "数值" if numeric >= dates else "日期" + return f"看起来是{label}，有 {failed} 个值无法解析，例如{shown}" diff --git a/app/datafiles/detect.py b/app/datafiles/detect.py new file mode 100644 index 0000000..163825d --- /dev/null +++ b/app/datafiles/detect.py @@ -0,0 +1,143 @@ +"""按文件内容识别格式和文本编码，不看扩展名""" + +import codecs +import csv +import json +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from app.datafiles.base import UNSUPPORTED_FORMAT, DataFileError +from app.schemas.data_file import DataFormat + +# xlsx/xlsb 解压后的总大小上限，防止解压炸弹 +MAX_UNZIPPED_BYTES = 1024 * 1024 * 1024 + +# 带 BOM 的文本直接按 BOM 解码；UTF-32 的 BOM 以 UTF-16 的开头，解出来含 NUL，按二进制拒绝 +BOMS = ( + (codecs.BOM_UTF8, "utf-8", "utf-8"), + (codecs.BOM_UTF16_LE, "utf-16-le", "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16-be", "utf-16"), +) + +# 没有 BOM 时依次尝试；GB18030 兼容 GBK、GB2312，中文 Windows 上 Excel 另存的 CSV 多是这种 +FALLBACK_ENCODINGS = ("utf-8", "gb18030") + +DELIMITERS = (",", "\t", ";", "|") + +# 判断分隔符只看开头这么多行 +SNIFF_LINES = 50 + + +@dataclass(frozen=True) +class Detected: + format: DataFormat + encoding: str | None = None + # 文本格式解码后的内容 + text: str | None = None + # CSV/TSV 的分隔符 + delimiter: str | None = None + # JSON 已解析的内容，避免读取时再解析一遍 + value: Any = None + + +def detect(path: Path) -> Detected: + with path.open("rb") as file: + head = file.read(8) + if head.startswith(b"PK\x03\x04"): + return Detected(_zip_format(path)) + if head.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): + return Detected("xls") + if head.startswith(b"PAR1"): + return Detected("parquet") + + text, encoding = decode_text(path.read_bytes()) + stripped = text.lstrip() + if not stripped: + raise DataFileError(422, "文件是空的") + if stripped[0] in "[{": + return _json_format(text, encoding) + delimiter = sniff_delimiter(text) + return Detected("tsv" if delimiter == "\t" else "csv", encoding, text, delimiter) + + +def _zip_format(path: Path) -> DataFormat: + try: + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + except zipfile.BadZipFile: + raise DataFileError(415, UNSUPPORTED_FORMAT) from None + if sum(info.file_size for info in infos) > MAX_UNZIPPED_BYTES: + raise DataFileError(413, "Excel 文件解压后过大") + names = {info.filename for info in infos} + if "xl/workbook.xml" in names: + return "xlsx" + if "xl/workbook.bin" in names: + return "xlsb" + raise DataFileError(415, UNSUPPORTED_FORMAT) + + +def decode_text(data: bytes) -> tuple[str, str]: + """返回解码后的文本和编码名""" + for bom, codec, name in BOMS: + if data.startswith(bom): + try: + text = data[len(bom) :].decode(codec) + except UnicodeDecodeError: + raise DataFileError( + 422, "无法识别文件编码，请另存为 UTF-8 后重新上传" + ) from None + return _reject_binary(text), name + for encoding in FALLBACK_ENCODINGS: + try: + return _reject_binary(data.decode(encoding)), encoding + except UnicodeDecodeError: + continue + raise DataFileError(422, "无法识别文件编码，请另存为 UTF-8 后重新上传") + + +def _reject_binary(text: str) -> str: + # 文本文件不会有 NUL；图片、PDF 等二进制文件偶尔能按 GB18030 解码成功，在这里拦下 + if "\x00" in text: + raise DataFileError(415, UNSUPPORTED_FORMAT) + return text + + +def _json_format(text: str, encoding: str) -> Detected: + try: + return Detected("json", encoding, text, value=json.loads(text)) + except ValueError as error: + json_error = error + first = next(line for line in text.splitlines() if line.strip()) + try: + if isinstance(json.loads(first), dict): + return Detected("jsonl", encoding, text) + except ValueError: + pass + # 以 [ 或 { 开头却不是合法 JSON，按 JSON 报错比当成 CSV 解析出一堆乱码更有用 + line = getattr(json_error, "lineno", None) + position = f"第 {line} 行" if line else "" + raise DataFileError( + 422, f"JSON 格式错误：{position}{getattr(json_error, 'msg', json_error)}" + ) + + +def sniff_delimiter(text: str) -> str: + """选出让开头各行列数最一致、列数最多的分隔符；都只有一列时按逗号处理""" + lines = [] + for line in text.splitlines(): + if line.strip(): + lines.append(line) + if len(lines) == SNIFF_LINES: + break + best, best_score = ",", (0.0, 1) + for delimiter in DELIMITERS: + widths = [len(row) for row in csv.reader(lines, delimiter=delimiter)] + width = max(set(widths), key=widths.count) + if width < 2: + continue + score = (widths.count(width) / len(widths), width) + if score > best_score: + best, best_score = delimiter, score + return best diff --git a/app/datafiles/parse.py b/app/datafiles/parse.py new file mode 100644 index 0000000..25f583f --- /dev/null +++ b/app/datafiles/parse.py @@ -0,0 +1,141 @@ +"""解析入口：识别格式 → 读取 → 拆表 → 推断类型 → 每张表写一个 Parquet → 列概况 + +只读写本地文件，不碰数据库；上传流程在子进程里调用（worker.py），超时直接终止子进程 +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pyarrow.parquet as pq +from pydantic import BaseModel + +from app.datafiles.base import DataFileError +from app.datafiles.columns import ( + TypedColumns, + arrow_columns, + build_columns, + clean_names, +) +from app.datafiles.detect import detect +from app.datafiles.profile import profile_table +from app.datafiles.readers import ( + Grid, + Records, + read_delimited, + read_excel, + read_json, + read_jsonl, + read_parquet, +) +from app.datafiles.tables import split_tables +from app.schemas.data_file import DataColumn, DataFormat, ParseWarning + + +# 子进程把解析结果转成 JSON 交回上传流程，所以用 pydantic 模型 +class ParsedTable(BaseModel): + sheet: str | None + # 同一 sheet 切出多张表时的序号，从 1 开始 + part: int | None + rows: int + columns: list[DataColumn] + path: Path + profile: dict[str, Any] + preview: list[list[Any]] + warnings: list[ParseWarning] + + +class ParsedFile(BaseModel): + format: DataFormat + encoding: str | None + tables: list[ParsedTable] + # 不属于某张表的警告，例如跳过的 sheet + warnings: list[ParseWarning] + + +@dataclass +class _Built: + sheet: str | None + part: int | None + typed: TypedColumns + warnings: list[ParseWarning] + + +def parse_file(path: Path, out_dir: Path) -> ParsedFile: + """解析 path，Parquet 写到 out_dir/1.parquet、2.parquet……""" + detected = detect(path) + warnings: list[ParseWarning] = [] + built: list[_Built] = [] + + def add_grid(grid: Grid) -> None: + tables, dropped = split_tables(grid) + warnings.extend(dropped) + for table in tables: + names, renamed = clean_names(table.header, table.sheet) + typed, kept = build_columns(names, list(zip(*table.rows)), table.sheet) + built.append( + _Built( + table.sheet, + table.part, + typed, + [*table.warnings, *_present(renamed, kept)], + ) + ) + + def add_records(records: Records) -> None: + warnings.extend(records.warnings) + names, renamed = clean_names(records.names, None) + typed, kept = build_columns(names, records.columns, None) + built.append(_Built(None, None, typed, _present(renamed, kept))) + + match detected.format: + case "csv" | "tsv": + add_grid(read_delimited(detected.text, detected.delimiter)) + case "xlsx" | "xls" | "xlsb": + grids, sheet_warnings = read_excel(path) + warnings.extend(sheet_warnings) + for grid in grids: + add_grid(grid) + case "json": + result = read_json(detected.value) + if isinstance(result, Grid): + add_grid(result) + else: + add_records(result) + case "jsonl": + add_records(read_jsonl(detected.text)) + case "parquet": + typed, renamed = arrow_columns(read_parquet(path)) + if typed.table.num_rows: + built.append(_Built(None, None, typed, _present(renamed))) + + if not built: + raise DataFileError(422, "文件里没有找到数据") + out_dir.mkdir(parents=True, exist_ok=True) + tables = [] + for index, item in enumerate(built, start=1): + target = out_dir / f"{index}.parquet" + pq.write_table(item.typed.table, target) + profile, preview = profile_table(target, item.typed.columns, item.typed.notes) + tables.append( + ParsedTable( + sheet=item.sheet, + part=item.part, + rows=item.typed.table.num_rows, + columns=item.typed.columns, + path=target, + profile=profile, + preview=preview, + warnings=item.warnings, + ) + ) + return ParsedFile( + format=detected.format, + encoding=detected.encoding, + tables=tables, + warnings=warnings, + ) + + +def _present(*warnings: ParseWarning | None) -> list[ParseWarning]: + return [warning for warning in warnings if warning is not None] diff --git a/app/datafiles/profile.py b/app/datafiles/profile.py new file mode 100644 index 0000000..95f985d --- /dev/null +++ b/app/datafiles/profile.py @@ -0,0 +1,126 @@ +"""列概况和预览：上传时在 Parquet 上用 DuckDB 计算，存进文件记录，供 describe_data 和接口返回""" + +import math +from pathlib import Path +from typing import Any + +import duckdb + +from app.datafiles.query import connect, json_value, quote_identifier +from app.schemas.data_file import DataColumn + +PREVIEW_ROWS = 20 +SAMPLE_ROWS = 5 +EXAMPLES = 3 +TOP_VALUES = 5 +# 去重值不超过这么多的文本、布尔列才给出现次数最多的值 +TOP_VALUES_MAX_DISTINCT = 50 +# 样例值从开头这么多行里取 +EXAMPLE_SCAN_ROWS = 1000 +PROFILE_CELL_CHARS = 50 +PREVIEW_CELL_CHARS = 200 + + +def profile_table( + path: Path, columns: list[DataColumn], notes: dict[str, str] +) -> tuple[dict[str, Any], list[list[Any]]]: + """返回列概况和接口用的预览（前 PREVIEW_ROWS 行）""" + connection = connect({"t": path}) + try: + rows = connection.execute("SELECT count(*) FROM t").fetchone()[0] + connection.execute("SUMMARIZE t") + fields = [column[0] for column in connection.description] + summary = {row[0]: dict(zip(fields, row)) for row in connection.fetchall()} + head = connection.execute(f"FROM t LIMIT {EXAMPLE_SCAN_ROWS}").fetchall() + profiled = [ + _profile_column( + connection, + column, + summary[column.name], + head, + position, + notes.get(column.name), + ) + for position, column in enumerate(columns) + ] + sample = connection.execute( + f"FROM t USING SAMPLE reservoir({SAMPLE_ROWS} ROWS) REPEATABLE (42)" + ).fetchall() + finally: + connection.close() + profile = { + "rows": rows, + "columns": profiled, + "sample": [ + [json_value(value, PROFILE_CELL_CHARS) for value in row] for row in sample + ], + } + preview = [ + [json_value(value, PREVIEW_CELL_CHARS) for value in row] + for row in head[:PREVIEW_ROWS] + ] + return profile, preview + + +def _profile_column( + connection: duckdb.DuckDBPyConnection, + column: DataColumn, + stats: dict[str, Any], + head: list[tuple], + position: int, + note: str | None, +) -> dict[str, Any]: + item: dict[str, Any] = {"name": column.name, "type": column.type} + if column.format: + item["format"] = column.format + null_ratio = float(stats["null_percentage"]) / 100 + item["null_ratio"] = round(null_ratio, 4) + # approx_unique 是估算值，小表上可能比行数还大 + item["distinct"] = min( + stats["approx_unique"], round(stats["count"] * (1 - null_ratio)) + ) + if column.type == "number": + for key in ("min", "max", "avg", "q25", "q50", "q75"): + item[key] = _number(stats[key]) + elif column.type in ("date", "datetime"): + # SUMMARIZE 的时间用空格分隔，和预览里的 ISO 写法统一 + item["min"], item["max"] = ( + None if stats[key] is None else stats[key].replace(" ", "T", 1) + for key in ("min", "max") + ) + if ( + column.type in ("text", "boolean") + and stats["approx_unique"] <= TOP_VALUES_MAX_DISTINCT + ): + name = quote_identifier(column.name) + counts = connection.execute( + f"SELECT {name}, count(*) AS n FROM t WHERE {name} IS NOT NULL " + f"GROUP BY 1 ORDER BY n DESC, 1 LIMIT {TOP_VALUES}" + ).fetchall() + item["top_values"] = [ + {"value": json_value(value, PROFILE_CELL_CHARS), "count": count} + for value, count in counts + ] + examples: list[Any] = [] + for row in head: + value = json_value(row[position], PROFILE_CELL_CHARS) + if value is not None and value not in examples: + examples.append(value) + if len(examples) == EXAMPLES: + break + item["examples"] = examples + if note: + item["note"] = note + return item + + +def _number(text: str | None) -> int | float | None: + """SUMMARIZE 的统计值是字符串；整数不带小数点，其余保留 6 位小数；Parquet 里的 inf 返回空""" + if text is None: + return None + value = float(text) + if not math.isfinite(value): + return None + if value.is_integer() and abs(value) < 2**53: + return int(value) + return round(value, 6) diff --git a/app/datafiles/query.py b/app/datafiles/query.py new file mode 100644 index 0000000..1e55302 --- /dev/null +++ b/app/datafiles/query.py @@ -0,0 +1,168 @@ +"""执行模型写的 SQL：连接只能读本次可见的 Parquet 文件，只允许单条 SELECT + +每次查询新建一个内存连接： +- allowed_paths 只放这些 Parquet 文件；不用 allowed_directories，否则同目录下的其他文件能被 glob 读到， + COPY、ATTACH 也能往目录里写 +- 关闭外部访问、扩展加载和 Python 变量替换扫描，最后锁定配置，SQL 里的 SET 一律失败 +""" + +import math +import threading +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from pathlib import Path +from typing import Any + +import duckdb + +MEMORY_LIMIT = "512MB" +THREADS = 2 +MAX_ROWS = 200 +MAX_CELL_CHARS = 200 + + +class QueryError(ValueError): + """查询失败的原因，原样返回给模型修正""" + + +@dataclass(frozen=True) +class QueryResult: + columns: list[str] + rows: list[list[Any]] + # 结果超过 max_rows 行，只返回了前 max_rows 行 + truncated: bool + + def to_dict(self) -> dict[str, Any]: + return { + "columns": self.columns, + "rows": self.rows, + "row_count": len(self.rows), + "truncated": self.truncated, + } + + +def run_query( + sql: str, + tables: Mapping[str, Path], + *, + timeout: float, + max_rows: int = MAX_ROWS, +) -> QueryResult: + """tables 是表名到 Parquet 文件的映射，SQL 里只能用这些表名""" + try: + connection = connect(tables) + except duckdb.Error as error: + # 任务执行期间文件被删除或过期清理 + raise QueryError( + f"数据表无法读取，文件可能已被删除：{_first_line(error)}" + ) from None + try: + _check_statement(connection, sql, tables) + timer = threading.Timer(timeout, connection.interrupt) + timer.start() + try: + connection.execute(sql) + columns = [column[0] for column in connection.description] + rows = connection.fetchmany(max_rows + 1) + except duckdb.InterruptException: + raise QueryError( + f"查询超过 {timeout:g} 秒被中止，请缩小范围或先聚合" + ) from None + except duckdb.PermissionException: + raise QueryError(f"只能查询上传的数据表：{_table_list(tables)}") from None + except duckdb.Error as error: + raise QueryError( + f"{_first_line(error)}；可用的表：{_table_list(tables)}" + ) from None + finally: + timer.cancel() + finally: + connection.close() + return QueryResult( + columns, + [[json_value(value) for value in row] for row in rows[:max_rows]], + len(rows) > max_rows, + ) + + +def connect(tables: Mapping[str, Path]) -> duckdb.DuckDBPyConnection: + """建好视图并锁定的连接；列概况也用它，和模型查询看到的数据一致""" + connection = duckdb.connect( + ":memory:", + config={ + "autoinstall_known_extensions": False, + "autoload_known_extensions": False, + "python_enable_replacements": False, + "memory_limit": MEMORY_LIMIT, + "threads": THREADS, + }, + ) + try: + paths = [str(Path(path).resolve()) for path in tables.values()] + for name, path in zip(tables, paths): + connection.execute( + f"CREATE VIEW {quote_identifier(name)} AS FROM read_parquet({_literal(path)})" + ) + if paths: + connection.execute("SET allowed_paths = ?", [paths]) + connection.execute("SET enable_external_access = false") + connection.execute("SET lock_configuration = true") + except BaseException: + connection.close() + raise + return connection + + +def quote_identifier(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def json_value(value: Any, max_chars: int = MAX_CELL_CHARS) -> Any: + """查询结果转成 JSON 能表示的值；长文本截断""" + if value is None or isinstance(value, (bool, int)): + return value + if isinstance(value, float): + return None if math.isnan(value) or math.isinf(value) else value + if isinstance(value, Decimal): + return float(value) + if isinstance(value, str): + return value if len(value) <= max_chars else value[:max_chars] + "…" + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, (list, tuple)): + return [json_value(item, max_chars) for item in value] + if isinstance(value, dict): + return {str(key): json_value(item, max_chars) for key, item in value.items()} + if isinstance(value, timedelta): + return str(value) + return json_value(str(value), max_chars) + + +def _check_statement( + connection: duckdb.DuckDBPyConnection, sql: str, tables: Mapping[str, Path] +) -> None: + try: + statements = connection.extract_statements(sql) + except duckdb.Error as error: + raise QueryError( + f"{_first_line(error)}；可用的表：{_table_list(tables)}" + ) from None + # PIVOT 不写 IN 时会先建一个枚举类型，解析出来是两条语句 + if len(statements) != 1 or statements[0].type != duckdb.StatementType.SELECT: + raise QueryError( + "只能执行一条 SELECT 查询（可以用 WITH、UNPIVOT；PIVOT 要用 IN 写明取值）" + ) + + +def _literal(text: str) -> str: + return "'" + text.replace("'", "''") + "'" + + +def _first_line(error: Exception) -> str: + return str(error).strip().splitlines()[0] + + +def _table_list(tables: Mapping[str, Path]) -> str: + return "、".join(tables) if tables else "（无）" diff --git a/app/datafiles/readers.py b/app/datafiles/readers.py new file mode 100644 index 0000000..cb8bc22 --- /dev/null +++ b/app/datafiles/readers.py @@ -0,0 +1,240 @@ +"""把各种格式读成原始网格（CSV、Excel）或按列的记录（JSON、JSONL），Parquet 直接读成 Arrow 表 + +CSV 和 Excel 可能有标题行、多行表头、合计行，所以先读成不带表头的网格，再由 tables.py 识别表区域 +""" + +import csv +import io +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq +from python_calamine import ( + CalamineWorkbook, + PasswordError, + SheetTypeEnum, + SheetVisibleEnum, +) + +from app.datafiles.base import MAX_TABLE_ROWS, DataFileError, rows_exceeded +from app.schemas.data_file import ParseWarning + +# 原始网格比数据多出的标题、表头、空行，读取时先粗略拦住明显超限的文件，精确的行数在拆表后检查 +ROW_SLACK = 1000 + +# JSON 嵌套对象展开的层数，更深的部分存成 JSON 字符串 +JSON_FLATTEN_LEVELS = 2 + +# JSONL 坏行超过这个比例时整个文件判为失败 +MAX_BAD_LINE_RATIO = 0.01 + +# 单元格最长的字符数；csv 模块默认 128KB，长文本单元格会报错 +csv.field_size_limit(16 * 1024 * 1024) + + +@dataclass +class Grid: + """不带表头的原始网格；rows[i] 是文件或 sheet 中的第 i + 1 行""" + + sheet: str | None + rows: list[list[Any]] + + +@dataclass +class Records: + """已有列结构的数据，按列存放""" + + names: list[str] + columns: list[list[Any]] + warnings: list[ParseWarning] = field(default_factory=list) + + +def read_delimited(text: str, delimiter: str) -> Grid: + rows = [] + for row in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter): + rows.append(row) + if len(rows) > MAX_TABLE_ROWS + ROW_SLACK: + raise rows_exceeded(None) + return Grid(None, rows) + + +def read_excel(path: Path) -> tuple[list[Grid], list[ParseWarning]]: + try: + book = CalamineWorkbook.from_path(str(path)) + except PasswordError: + raise DataFileError(422, "Excel 文件设置了密码，请去掉密码后重新上传") from None + except Exception: + raise DataFileError(422, "无法读取 Excel 文件，文件可能已损坏") from None + grids, warnings = [], [] + try: + for meta in book.sheets_metadata: + # 图表页、宏表等没有单元格数据 + if meta.typ != SheetTypeEnum.WorkSheet: + continue + if meta.visible != SheetVisibleEnum.Visible: + warnings.append( + ParseWarning( + code="hidden_sheet_skipped", + message=f"跳过了隐藏的 sheet「{meta.name}」", + ) + ) + continue + sheet = book.get_sheet_by_name(meta.name) + if sheet.total_height > MAX_TABLE_ROWS + ROW_SLACK: + raise rows_exceeded(meta.name) + # 保留左上角的空白区域，行号和 Excel 里看到的一致 + rows = sheet.to_python(skip_empty_area=False) + if not any(cell != "" for row in rows for cell in row): + warnings.append( + ParseWarning( + code="empty_sheet_skipped", + message=f"sheet「{meta.name}」是空的，已跳过", + ) + ) + continue + grids.append(Grid(meta.name, rows)) + finally: + book.close() + return grids, warnings + + +def read_json(value: Any) -> Grid | Records: + """对象数组按记录读取；数组的数组当作网格，第一行通常是表头""" + warnings = [] + if isinstance(value, dict): + records, path = _find_records(value) + if path: + warnings.append( + ParseWarning( + code="json_path_selected", + message=f"使用了 JSON 中「{path}」下的 {len(records)} 条记录", + ) + ) + else: + records = value + if not records: + raise DataFileError(422, "JSON 里没有找到数据") + if all(isinstance(item, list) for item in records): + return Grid(None, [[_scalar(cell) for cell in row] for row in records]) + if all(isinstance(item, dict) for item in records): + result = _flatten_records(records) + result.warnings = warnings + return result + if not any(isinstance(item, (dict, list)) for item in records): + return Records(["值"], [list(records)], warnings) + raise DataFileError(422, "JSON 数组里的元素类型不一致，需要都是对象或都是数组") + + +def read_jsonl(text: str) -> Records: + records, bad, total = [], [], 0 + for number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + total += 1 + try: + record = json.loads(line) + except ValueError: + record = None + if isinstance(record, dict): + records.append(record) + else: + bad.append(number) + if not records: + raise DataFileError(422, "JSONL 里没有找到数据") + if len(bad) > total * MAX_BAD_LINE_RATIO: + raise DataFileError( + 422, f"JSONL 有 {len(bad)} 行无法解析（例如第 {bad[0]} 行），超过 1%" + ) + result = _flatten_records(records) + if bad: + result.warnings.append( + ParseWarning( + code="bad_lines_skipped", + message=f"跳过了 {len(bad)} 行无法解析的内容：{_line_list(bad)}", + ) + ) + return result + + +def read_parquet(path: Path) -> pa.Table: + try: + table = pq.read_table(path) + except Exception: + raise DataFileError(422, "无法读取 Parquet 文件，文件可能已损坏") from None + if table.num_rows > MAX_TABLE_ROWS: + raise rows_exceeded(None) + # struct 展开一层，列名写成 a.b；更深的嵌套由 columns.py 存成 JSON 字符串 + return table.flatten() + + +def _find_records(value: dict) -> tuple[list, str | None]: + """在对象的前两层里找最长的对象数组，例如 {"data": [...]}；找不到时按列式或单条记录处理""" + candidates = [] + for key, item in value.items(): + if _is_record_list(item): + candidates.append((len(item), str(key), item)) + elif isinstance(item, dict): + for sub_key, sub in item.items(): + if _is_record_list(sub): + candidates.append((len(sub), f"{key}.{sub_key}", sub)) + if candidates: + _, path, records = max(candidates, key=lambda candidate: candidate[0]) + return records, path + lists = list(value.values()) + # 列式：{"地区": [...], "销售额": [...]} + if ( + lists + and all(isinstance(item, list) for item in lists) + and len({len(item) for item in lists}) == 1 + ): + return [dict(zip(value, row)) for row in zip(*lists)], None + return [value], None + + +def _is_record_list(value: Any) -> bool: + return ( + isinstance(value, list) + and bool(value) + and all(isinstance(item, dict) for item in value) + ) + + +def _flatten_records(records: list[dict]) -> Records: + # 列按首次出现的顺序排列，记录里没有的字段为空 + index: dict[str, int] = {} + columns: list[list[Any]] = [] + for row_number, record in enumerate(records): + flat: dict[str, Any] = {} + _flatten(record, "", 0, flat) + for name, value in flat.items(): + if name not in index: + index[name] = len(columns) + columns.append([None] * row_number) + columns[index[name]].append(value) + for column in columns: + if len(column) == row_number: + column.append(None) + return Records(list(index), columns) + + +def _flatten(record: dict, prefix: str, level: int, out: dict[str, Any]) -> None: + for key, value in record.items(): + name = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, dict) and value and level < JSON_FLATTEN_LEVELS: + _flatten(value, name, level + 1, out) + else: + out[name] = _scalar(value) + + +def _scalar(value: Any) -> Any: + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False) + return value + + +def _line_list(numbers: list[int]) -> str: + shown = "、".join(f"第 {number} 行" for number in numbers[:5]) + return shown + ("等" if len(numbers) > 5 else "") diff --git a/app/datafiles/service.py b/app/datafiles/service.py new file mode 100644 index 0000000..48c8014 --- /dev/null +++ b/app/datafiles/service.py @@ -0,0 +1,334 @@ +"""上传流程：保存原文件 → 子进程解析（超时终止）→ 分配表名 → 保存记录；以及查看、删除和过期清理""" + +import asyncio +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import BinaryIO +from uuid import uuid4 + +from loguru import logger + +from app.config import PROJECT_ROOT, settings +from app.core.exceptions import BizException +from app.datafiles.parse import ParsedFile, ParsedTable +from app.datafiles.storage import SOURCE_NAME, DataFileStore, file_dir +from app.schemas.base import JsonObject +from app.schemas.data_file import ( + DataFileRecord, + DataTable, + RequestDataFile, + RequestDataTable, +) +from app.services.conversation import ConversationStore + +MAX_FILES_PER_CONVERSATION = 10 +# 解析超时（秒），超时终止子进程；同时最多解析的文件数，其余排队 +PARSE_TIMEOUT = 60 +PARSE_CONCURRENCY = 2 +# 每小时清理一次目录；一小时内创建的目录不删，避免删掉正在上传的文件 +SWEEP_INTERVAL = 3600 +SWEEP_GRACE = 3600 +# 只清理这个格式的目录，DATA_FILE_DIR 配错时也不会删到别的东西 +FILE_ID = re.compile(r"^f_[0-9a-f]{32}$") +MAX_NAME_CHARS = 200 +CHUNK_BYTES = 1024 * 1024 + +NOT_FOUND = "文件不存在或已过期" +PARSE_CRASHED = "解析文件时出错，请检查文件内容，或另存为 CSV 后重新上传" + + +class DataFileService: + def __init__( + self, + store: DataFileStore, + conversations: ConversationStore, + root: Path | None = None, + *, + max_mb: int | None = None, + ttl: int | None = None, + ): + self.store = store + self.root = root or settings.data_file_dir + self.max_mb = max_mb or settings.data_file_max_mb + self._conversations = conversations + self._ttl = timedelta(seconds=ttl or settings.data_file_ttl) + self._parsing = asyncio.Semaphore(PARSE_CONCURRENCY) + # 文件数检查、分配表名和写入记录要一起完成，并发上传时才不会超限或重名；单实例部署，进程内的锁就够了 + self._saving = asyncio.Lock() + + async def upload( + self, user_id: int, conversation_id: str, name: str | None, source: BinaryIO + ) -> DataFileRecord: + await self._check_conversation(conversation_id, user_id) + # 先粗查一次，满了就不必上传和解析；写入前在锁里再查一次 + _check_count(await self.store.find(conversation_id)) + file_id = f"f_{uuid4().hex}" + directory = file_dir(self.root, file_id) + try: + size, sha256 = await asyncio.to_thread(self._save, source, directory) + async with self._parsing: + parsed = await _parse(directory) + async with self._saving: + existing = await self.store.find(conversation_id, include_deleted=True) + _check_count([record for record in existing if not record.deleted]) + record = self._record( + file_id, + user_id, + conversation_id, + _clean_name(name), + size, + sha256, + parsed, + _next_table_number(existing), + ) + await self.store.insert(record) + except BaseException: + # 失败的上传不留文件，会话内的文件数不变 + await asyncio.to_thread(shutil.rmtree, directory, ignore_errors=True) + raise + logger.info( + "数据文件已上传：{} 格式 {}，{} 张表", + file_id, + record.format, + len(record.tables), + ) + return record + + async def list_files( + self, user_id: int, conversation_id: str + ) -> list[DataFileRecord]: + await self._check_conversation(conversation_id, user_id) + records = await self.store.find(conversation_id) + return [record for record in records if record.owner_id == user_id] + + async def get_file(self, user_id: int, file_id: str) -> DataFileRecord: + record = await self.store.get(file_id) + # 他人的文件与不存在同样返回 404 + if record is None or record.owner_id != user_id: + raise BizException(404, NOT_FOUND) + return record + + async def delete_file(self, user_id: int, file_id: str) -> None: + await self.get_file(user_id, file_id) + await self.store.mark_deleted(file_id) + # Windows 上正在查询的 Parquet 删不掉，留给定时清理 + await asyncio.to_thread( + shutil.rmtree, file_dir(self.root, file_id), ignore_errors=True + ) + + async def request_files( + self, user_id: int, conversation_id: str, attachments: list[JsonObject] + ) -> list[RequestDataFile]: + """本次请求可用的文件，调用方已校验会话归属 + + attachments 里 kind 为 data_file 的引用必须是本会话未过期的文件，否则 404；其他种类不处理 + """ + records = [ + record + for record in await self.store.find(conversation_id) + if record.owner_id == user_id + ] + attached = set() + for item in attachments: + if item.get("kind") != "data_file": + continue + if not isinstance(file_id := item.get("file_id"), str): + raise BizException(422, "data_file 附件缺少 file_id") + attached.add(file_id) + if attached - {record.file_id for record in records}: + raise BizException(404, NOT_FOUND) + return [ + RequestDataFile( + file_id=record.file_id, + name=record.name, + attached_this_turn=record.file_id in attached, + tables=[ + RequestDataTable.model_validate( + { + **table.model_dump(), + "path": str( + file_dir(self.root, record.file_id) / table.parquet + ), + } + ) + for table in record.tables + ], + warnings=record.warnings, + ) + for record in records + ] + + async def sweep(self) -> int: + """删除没有对应记录（过期、已删除、内存模式重启后丢失）的目录，返回删除的个数""" + live = await self.store.live_ids() + return await asyncio.to_thread(self._remove_orphans, live) + + async def run_sweeper(self) -> None: + while True: + try: + if removed := await self.sweep(): + logger.info("已清理 {} 个过期或删除的数据文件目录", removed) + except Exception as error: + logger.opt(exception=error).error("清理数据文件目录失败") + await asyncio.sleep(SWEEP_INTERVAL) + + async def _check_conversation(self, conversation_id: str, user_id: int) -> None: + if not await self._conversations.exists(conversation_id, user_id): + raise BizException(404, "会话不存在或已过期") + + def _save(self, source: BinaryIO, directory: Path) -> tuple[int, str]: + """边写边算大小和 sha256，超过上限立即停止""" + directory.mkdir(parents=True) + digest, size = hashlib.sha256(), 0 + with (directory / SOURCE_NAME).open("wb") as target: + while chunk := source.read(CHUNK_BYTES): + size += len(chunk) + if size > self.max_mb * 1024 * 1024: + raise BizException(413, f"文件不能超过 {self.max_mb} MB") + digest.update(chunk) + target.write(chunk) + return size, digest.hexdigest() + + def _record( + self, + file_id: str, + user_id: int, + conversation_id: str, + name: str, + size: int, + sha256: str, + parsed: ParsedFile, + first_number: int, + ) -> DataFileRecord: + tables, warnings = [], list(parsed.warnings) + for number, table in enumerate(parsed.tables, start=first_number): + table_name = f"t{number}" + tables.append( + DataTable( + table=table_name, + label=_label(name, table), + sheet=table.sheet, + rows=table.rows, + columns=table.columns, + preview=table.preview, + profile=table.profile, + parquet=table.path.name, + ) + ) + warnings.extend( + warning.model_copy(update={"table": table_name}) + for warning in table.warnings + ) + now = datetime.now(UTC) + return DataFileRecord( + file_id=file_id, + owner_id=user_id, + conversation_id=conversation_id, + name=name, + format=parsed.format, + encoding=parsed.encoding, + size=size, + sha256=sha256, + created_at=now, + expires_at=now + self._ttl, + tables=tables, + warnings=warnings, + ) + + def _remove_orphans(self, live: set[str]) -> int: + if not self.root.is_dir(): + return 0 + cutoff = time.time() - SWEEP_GRACE + removed = 0 + for entry in self.root.iterdir(): + if ( + FILE_ID.match(entry.name) + and entry.name not in live + and entry.is_dir() + and entry.stat().st_mtime < cutoff + ): + shutil.rmtree(entry, ignore_errors=True) + removed += 1 + return removed + + +async def _parse(directory: Path) -> ParsedFile: + python, env = _python() + command = [ + python, + "-m", + "app.datafiles.worker", + str(directory / SOURCE_NAME), + str(directory), + ] + try: + # 超时后 subprocess.run 会终止子进程 + completed = await asyncio.to_thread( + subprocess.run, + command, + cwd=PROJECT_ROOT, + env=env, + stdin=subprocess.DEVNULL, + capture_output=True, + timeout=PARSE_TIMEOUT, + ) + except subprocess.TimeoutExpired: + raise BizException( + 422, f"解析超过 {PARSE_TIMEOUT} 秒，文件可能太大或太复杂，请拆分后上传" + ) from None + if completed.returncode != 0: + logger.error( + "数据文件解析进程异常退出（{}）：{}", + completed.returncode, + completed.stderr.decode(errors="replace")[-2000:], + ) + raise BizException(422, PARSE_CRASHED) + result = json.loads(completed.stdout) + if error := result.get("error"): + raise BizException(error["status"], error["message"]) + return ParsedFile.model_validate(result["file"]) + + +def _python() -> tuple[str, dict[str, str] | None]: + """Windows 的 venv 里 sys.executable 是启动器，由它再拉起真正的解释器；超时时只杀得掉启动器， + 刚启动就被杀时解释器会成为孤儿，一直占着输出管道。和 multiprocessing 一样直接启动基础解释器， + 用 __PYVENV_LAUNCHER__ 告诉它所在的 venv + """ + if sys.platform == "win32" and sys._base_executable != sys.executable: + return sys._base_executable, { + **os.environ, + "__PYVENV_LAUNCHER__": sys.executable, + } + return sys.executable, None + + +def _check_count(records: list[DataFileRecord]) -> None: + if len(records) >= MAX_FILES_PER_CONVERSATION: + raise BizException( + 409, f"每个会话最多 {MAX_FILES_PER_CONVERSATION} 个文件，请先删除不用的文件" + ) + + +def _next_table_number(records: list[DataFileRecord]) -> int: + """表名在会话内递增，已删除文件的表名也不再使用""" + used = (int(table.table[1:]) for record in records for table in record.tables) + return max(used, default=0) + 1 + + +def _clean_name(name: str | None) -> str: + # 有的浏览器会带上本地路径，只保留文件名 + base = re.split(r"[\\/]", name or "")[-1].strip() + return base[:MAX_NAME_CHARS] or "未命名文件" + + +def _label(name: str, table: ParsedTable) -> str: + label = f"{name} / {table.sheet}" if table.sheet else name + return f"{label}（{table.part}）" if table.part else label diff --git a/app/datafiles/storage.py b/app/datafiles/storage.py new file mode 100644 index 0000000..64d7d7d --- /dev/null +++ b/app/datafiles/storage.py @@ -0,0 +1,152 @@ +"""数据文件的存储：文件放本地目录，元数据启用 MongoDB 时进集合 data_files，否则放进程内存 + +目录结构是 DATA_FILE_DIR//，里面有原文件 source 和每张表的 1.parquet、2.parquet…… +DuckDB 只能读本地文件，所以文件不进 MongoDB；单实例部署，本地磁盘就够了 +""" + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Protocol + +from bson.codec_options import CodecOptions +from pymongo import ASCENDING + +from app.schemas.data_file import DataFileRecord + +# 原文件不带扩展名：calamine 见到 .xls 等扩展名会按扩展名打开，用户的扩展名和内容不符时会失败 +SOURCE_NAME = "source" + + +def file_dir(root: Path, file_id: str) -> Path: + return root / file_id + + +def _now() -> datetime: + return datetime.now(UTC) + + +class DataFileStore(Protocol): + """内存和 MongoDB 实现同一组方法；过期的记录一律当作不存在""" + + async def insert(self, record: DataFileRecord) -> None: ... + + async def get(self, file_id: str) -> DataFileRecord | None: + """未过期且未删除的记录""" + ... + + async def find( + self, conversation_id: str, *, include_deleted: bool = False + ) -> list[DataFileRecord]: + """会话内未过期的记录，按上传时间排列""" + ... + + async def mark_deleted(self, file_id: str) -> None: ... + + async def live_ids(self) -> set[str]: + """未过期且未删除的 file_id，清理目录时用""" + ... + + +class InMemoryDataFileStore: + def __init__(self) -> None: + self._records: dict[str, DataFileRecord] = {} + + def _purge_expired(self) -> None: + now = _now() + for file_id in [ + key for key, record in self._records.items() if record.expires_at <= now + ]: + del self._records[file_id] + + async def insert(self, record: DataFileRecord) -> None: + self._purge_expired() + self._records[record.file_id] = record + + async def get(self, file_id: str) -> DataFileRecord | None: + self._purge_expired() + record = self._records.get(file_id) + return None if record is None or record.deleted else record + + async def find( + self, conversation_id: str, *, include_deleted: bool = False + ) -> list[DataFileRecord]: + self._purge_expired() + records = [ + record + for record in self._records.values() + if record.conversation_id == conversation_id + and (include_deleted or not record.deleted) + ] + return sorted(records, key=lambda record: record.created_at) + + async def mark_deleted(self, file_id: str) -> None: + if (record := self._records.get(file_id)) is not None: + self._records[file_id] = record.model_copy(update={"deleted": True}) + + async def live_ids(self) -> set[str]: + self._purge_expired() + return {key for key, record in self._records.items() if not record.deleted} + + +class MongoDataFileStore: + def __init__(self, db: Any, prefix: str = ""): + # 读回的时间统一带 UTC 时区；集合名前缀用于测试隔离 + options = CodecOptions(tz_aware=True, tzinfo=UTC) + self._files = db.get_collection(f"{prefix}data_files", codec_options=options) + + async def ensure_indexes(self) -> None: + # 过期记录由 TTL 索引删除；TTL 每分钟才扫一次，查询时仍按 expires_at 过滤 + await self._files.create_index("expires_at", expireAfterSeconds=0) + await self._files.create_index( + [("conversation_id", ASCENDING), ("created_at", ASCENDING)] + ) + + async def insert(self, record: DataFileRecord) -> None: + await self._files.insert_one({"_id": record.file_id, **record.model_dump()}) + + async def get(self, file_id: str) -> DataFileRecord | None: + document = await self._files.find_one( + {"_id": file_id, "deleted": False, "expires_at": {"$gt": _now()}} + ) + return None if document is None else _record(document) + + async def find( + self, conversation_id: str, *, include_deleted: bool = False + ) -> list[DataFileRecord]: + query: dict[str, Any] = { + "conversation_id": conversation_id, + "expires_at": {"$gt": _now()}, + } + if not include_deleted: + query["deleted"] = False + documents = await ( + self._files.find(query) + .sort([("created_at", ASCENDING), ("_id", ASCENDING)]) + .to_list() + ) + return [_record(document) for document in documents] + + async def mark_deleted(self, file_id: str) -> None: + await self._files.update_one({"_id": file_id}, {"$set": {"deleted": True}}) + + async def live_ids(self) -> set[str]: + return set( + await self._files.distinct( + "_id", {"deleted": False, "expires_at": {"$gt": _now()}} + ) + ) + + +def _record(document: dict[str, Any]) -> DataFileRecord: + return DataFileRecord.model_validate( + {key: value for key, value in document.items() if key != "_id"} + ) + + +async def open_data_file_store(db: Any) -> DataFileStore: + """db 为空（未启用 MongoDB）时用内存实现，和会话记录一致""" + if db is None: + return InMemoryDataFileStore() + store = MongoDataFileStore(db) + await store.ensure_indexes() + return store diff --git a/app/datafiles/tables.py b/app/datafiles/tables.py new file mode 100644 index 0000000..25caf5b --- /dev/null +++ b/app/datafiles/tables.py @@ -0,0 +1,290 @@ +"""在原始网格里识别表：按空行切块，找表头，合并两级表头，去掉标题、说明和合计行 + +只用规则；判断不了的地方照常产出表，同时记一条警告，调用方据此提示用户 +""" + +import math +import re +from dataclasses import dataclass, field +from typing import Any + +from app.datafiles.base import MAX_TABLE_ROWS, rows_exceeded, where +from app.datafiles.columns import classify, label_text +from app.datafiles.readers import Grid +from app.schemas.data_file import ParseWarning + +# 只在每块的前几行里找表头 +HEADER_SEARCH_ROWS = 10 + +# 首个或第二个非空单元格是这些词（去掉空白和冒号后）的行是合计行 +TOTAL_LABELS = { + "合计", + "总计", + "小计", + "共计", + "总和", + "平均", + "平均值", + "total", + "subtotal", + "grandtotal", + "sum", + "average", +} +# 例如"华东小计""一季度合计" +TOTAL_SUFFIXES = ("小计", "合计", "总计") + +# 警告里引用的单元格文本最长显示这么多字 +QUOTE_CHARS = 30 + + +@dataclass +class RawTable: + sheet: str | None + # 同一 sheet 切出多张表时的序号，从 1 开始；只有一张表时为空 + part: int | None + header: list[Any] + # 数据行，宽度与表头相同 + rows: list[list[Any]] + warnings: list[ParseWarning] = field(default_factory=list) + + +@dataclass +class _Piece: + columns: list[int] + # 表头所在行在网格中的下标 + header_row: int + header: list[Any] + # 数据行在网格中的下标 + data: list[int] + warnings: list[ParseWarning] + + +def split_tables(grid: Grid) -> tuple[list[RawTable], list[ParseWarning]]: + """返回识别出的表和不属于任何表的警告（被丢弃的块）""" + rows = _normalize(grid.rows) + sheet = grid.sheet + pieces: list[_Piece] = [] + dropped: list[ParseWarning] = [] + for block in _blocks(rows): + columns = [ + c for c in range(len(rows[0])) if any(rows[r][c] is not None for r in block) + ] + last = pieces[-1] if pieces else None + # 表中间夹着空行时，后面的块以数据行开头、列也落在上一张表内，接在上一张表后面 + if ( + last + and not _is_label_row([rows[block[0]][c] for c in columns]) + and set(columns) <= set(last.columns) + ): + last.data.extend(block) + continue + if len(block) < 2: + dropped.append( + ParseWarning( + code="block_dropped", + message=f"{where(sheet)}第 {block[0] + 1} 行的{_quote(rows[block[0]])}不是表格，已跳过", + ) + ) + continue + pieces.append(_read_block(rows, block, columns, sheet)) + + tables = [] + for piece in pieces: + table = _finish(rows, piece, sheet) + if table is None: + dropped.append( + ParseWarning( + code="block_dropped", + message=f"{where(sheet)}第 {piece.header_row + 1} 行开始的表只有表头没有数据，已跳过", + ) + ) + else: + tables.append(table) + if len(tables) > 1: + for part, table in enumerate(tables, start=1): + table.part = part + return tables, dropped + + +def _normalize(grid: list[list[Any]]) -> list[list[Any]]: + """去掉字符串两端空白，空字符串记为 None，补齐成同样宽度""" + width = max((len(row) for row in grid), default=0) + rows = [] + for row in grid: + cells = [_cell(value) for value in row] + cells.extend([None] * (width - len(cells))) + rows.append(cells) + return rows + + +def _cell(value: Any) -> Any: + if isinstance(value, str): + value = value.strip() + return value or None + return value + + +def _blocks(rows: list[list[Any]]) -> list[list[int]]: + blocks, current = [], [] + for index, row in enumerate(rows): + if any(value is not None for value in row): + current.append(index) + elif current: + blocks.append(current) + current = [] + if current: + blocks.append(current) + return blocks + + +def _read_block( + rows: list[list[Any]], block: list[int], columns: list[int], sheet: str | None +) -> _Piece: + cells = [[rows[r][c] for c in columns] for r in block] + need = max(1, math.ceil(len(columns) / 2)) + dense = [sum(value is not None for value in row) >= need for row in cells] + search = range(min(HEADER_SEARCH_ROWS, len(cells))) + warnings = [] + + # 表头：前几行里第一个够满、并且全是文字的行；标题行通常只有一个单元格，不够满 + header_at = next((i for i in search if dense[i] and _is_label_row(cells[i])), None) + confident = header_at is not None + if header_at is None: + header_at = next((i for i in search if dense[i]), 0) + warnings.append( + ParseWarning( + code="header_uncertain", + message=f"{where(sheet)}没有找到明确的表头，用第 {block[header_at] + 1} 行作表头，可能识别错误", + ) + ) + if header_at > 0: + skipped = "".join( + _quote(cells[i]) for i in range(header_at) if _quote(cells[i]) + ) + warnings.append( + ParseWarning( + code="title_skipped", + message=f"{where(sheet)}{_row_range(block[0], block[header_at - 1])}是标题，已跳过：{skipped}", + ) + ) + + header = cells[header_at] + data_at = header_at + 1 + if ( + confident + and header_at + 2 < len(cells) + and _is_subheader(cells[header_at], cells[header_at + 1], cells[header_at + 2]) + ): + header = _merge_header(cells[header_at], cells[header_at + 1]) + data_at = header_at + 2 + warnings.append( + ParseWarning( + code="header_merged", + message=f"{where(sheet)}{_row_range(block[header_at], block[header_at + 1])}合并为表头", + ) + ) + return _Piece(columns, block[header_at], header, block[data_at:], warnings) + + +def _finish(rows: list[list[Any]], piece: _Piece, sheet: str | None) -> RawTable | None: + data = [(index, [rows[index][c] for c in piece.columns]) for index in piece.data] + warnings = piece.warnings + + # 表格下方只有一个文字单元格的行是说明，例如"注：数据来源于……" + notes = [] + while data and len(piece.columns) >= 3 and _is_note(data[-1][1]): + notes.append(data.pop()) + if notes: + notes.reverse() + shown = "".join(_quote(row) for _, row in notes) + warnings.append( + ParseWarning( + code="note_skipped", + message=f"{where(sheet)}{_row_range(notes[0][0], notes[-1][0])}是说明，已跳过：{shown}", + ) + ) + + # 合计、小计行不是明细数据，留着会让求和重复计算 + totals = [index for index, row in data if _is_total(row)] + if totals: + data = [(index, row) for index, row in data if not _is_total(row)] + listed = "、".join(str(index + 1) for index in totals[:5]) + ( + "等" if len(totals) > 5 else "" + ) + warnings.append( + ParseWarning( + code="total_rows_removed", + message=f"{where(sheet)}已排除 {len(totals)} 行合计、小计（第 {listed} 行）", + ) + ) + + if not data: + return None + if len(data) > MAX_TABLE_ROWS: + raise rows_exceeded(sheet) + return RawTable(sheet, None, piece.header, [row for _, row in data], warnings) + + +def _is_label(value: Any) -> bool: + return isinstance(value, str) and classify(value)[0] == "text" + + +def _is_label_row(row: list[Any]) -> bool: + values = [value for value in row if value is not None] + return bool(values) and all(_is_label(value) for value in values) + + +def _is_subheader(top: list[Any], sub: list[Any], first_data: list[Any]) -> bool: + """第二行也全是文字、紧接着是带数值的数据行，并且两行里有合并单元格留下的空格""" + return ( + _is_label_row(sub) + and not _is_label_row(first_data) + and (None in top or None in sub) + ) + + +def _merge_header(top: list[Any], sub: list[Any]) -> list[str]: + """合并单元格只有左上角有值，第一行横向向前填充后与第二行用 - 连接，例如 销售额-一月""" + merged, carry = [], None + for upper, lower in zip(top, sub): + if upper is not None: + carry = upper + parts = [label_text(part) for part in (carry, lower) if part is not None] + merged.append("-".join(parts)) + return merged + + +def _is_note(row: list[Any]) -> bool: + values = [value for value in row if value is not None] + return len(values) == 1 and _is_label(values[0]) + + +def _is_total(row: list[Any]) -> bool: + checked = 0 + for value in row: + if value is None: + continue + if isinstance(value, str): + key = re.sub(r"[\s:：]", "", value).lower() + if key in TOTAL_LABELS or key.endswith(TOTAL_SUFFIXES): + return True + checked += 1 + if checked == 2: + return False + return False + + +def _quote(row: list[Any]) -> str: + """一行里第一个非空单元格，截断后加引号""" + value = next((value for value in row if value is not None), None) + if value is None: + return "" + text = label_text(value) + if len(text) > QUOTE_CHARS: + text = text[:QUOTE_CHARS] + "…" + return f"「{text}」" + + +def _row_range(first: int, last: int) -> str: + return f"第 {first + 1} 行" if first == last else f"第 {first + 1}–{last + 1} 行" diff --git a/app/datafiles/worker.py b/app/datafiles/worker.py new file mode 100644 index 0000000..eb1a663 --- /dev/null +++ b/app/datafiles/worker.py @@ -0,0 +1,27 @@ +"""解析子进程：python -m app.datafiles.worker <原文件> <输出目录> + +结果以一行 JSON 写到 stdout：成功是 {"file": ParsedFile}，文件本身的问题是 {"error": {"status", "message"}} +其他异常让进程以非零状态退出，堆栈留在 stderr，由上传流程记日志 +""" + +import json +import sys +from pathlib import Path + +from app.datafiles.base import DataFileError +from app.datafiles.parse import parse_file + + +def main(source: str, out_dir: str) -> None: + try: + result = { + "file": parse_file(Path(source), Path(out_dir)).model_dump(mode="json") + } + except DataFileError as error: + result = {"error": {"status": error.status, "message": error.message}} + # 只输出 ASCII，不受控制台编码影响 + sys.stdout.write(json.dumps(result)) + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/app/main.py b/app/main.py index 99e5124..7424159 100644 --- a/app/main.py +++ b/app/main.py @@ -8,6 +8,8 @@ from loguru import logger from app.api.agent import router from app.api.conversations import router as conversation_router +from app.api.files import limits_router as file_limits_router +from app.api.files import router as file_router from app.api.materials import router as material_router from app.core.auth import get_current_user from app.core.exceptions import register_exception_handlers @@ -21,6 +23,8 @@ async def lifespan(application: FastAPI): from app.clients.decision import close_decision_engine from app.clients.mcp import start_mcp_servers, stop_mcp_servers from app.clients.user_center import close_user_center + from app.datafiles.service import DataFileService + from app.datafiles.storage import open_data_file_store from app.observability.tracing import flush_tracing from app.processor.graph import build_graph from app.services.agent import AgentService @@ -31,7 +35,14 @@ async def lifespan(application: FastAPI): persistence = await open_persistence() application.state.conversation_store = persistence.store mcp_processes = [] + sweeper = None try: + # 数据文件的元数据和会话记录用同一种存储；启动时先清理一遍过期目录，之后每小时一次 + data_files = DataFileService( + await open_data_file_store(persistence.db), persistence.store + ) + application.state.data_file_service = data_files + sweeper = asyncio.create_task(data_files.run_sweeper()) # 物料目录启动时载入内存，动作校验与前端接口共用，避免重复查询数据库 await warm_material_cache(persistence.db) # 拉起 mcp_servers.json 中由应用管理的服务；单个服务失败只记录，用到它的请求执行时报错 @@ -42,10 +53,12 @@ async def lifespan(application: FastAPI): except Exception as error: logger.opt(exception=error).error("screen agent 预先组装失败") application.state.agent_service = AgentService( - build_graph(persistence.checkpointer), persistence.store + build_graph(persistence.checkpointer), persistence.store, data_files ) yield finally: + if sweeper is not None: + sweeper.cancel() # 共享客户端只在应用退出时关闭，追踪刷新放到线程避免阻塞事件循环 try: await stop_mcp_servers(mcp_processes) @@ -76,6 +89,9 @@ def create_app() -> FastAPI: application.include_router( conversation_router, prefix="/api", dependencies=authenticated ) + # 上传限制无需登录；要先于 /files/{file_id} 注册，否则会被当成文件 ID + application.include_router(file_limits_router, prefix="/api") + application.include_router(file_router, prefix="/api", dependencies=authenticated) application.include_router(material_router, prefix="/api") return application diff --git a/app/processor/nodes/screen_agent_node.py b/app/processor/nodes/screen_agent_node.py index 2536e50..b5b0461 100644 --- a/app/processor/nodes/screen_agent_node.py +++ b/app/processor/nodes/screen_agent_node.py @@ -7,6 +7,7 @@ import json from langchain_core.messages import HumanMessage +from app.agent.data_tools import DataContext from app.agent.screen_agent import AgentReply, get_screen_agent from app.agent.tools import ScreenAgentContext from app.processor.base import BaseNode @@ -53,7 +54,7 @@ class ScreenAgentNode(BaseNode): return updated_state context = state["input_context"] - agent = await get_screen_agent() + agent = await get_screen_agent(with_data=bool(context.data_files)) result = await agent.ainvoke( { # 会话历史作为此前的对话消息，本次请求是最后一条用户消息 @@ -64,7 +65,11 @@ class ScreenAgentNode(BaseNode): ), ] }, - context=ScreenAgentContext(screen=context.screen, catalog=catalog), + context=ScreenAgentContext( + screen=context.screen, + catalog=catalog, + data=DataContext(tuple(context.data_files)), + ), ) if result.get("structured_response") is None: last = result["messages"][-1] diff --git a/app/processor/summary.py b/app/processor/summary.py index 5cb5079..ca8b6d3 100644 --- a/app/processor/summary.py +++ b/app/processor/summary.py @@ -4,11 +4,15 @@ """ from app.schemas.base import JsonObject +from app.schemas.data_file import RequestDataFile from app.schemas.input_context import InputContext +# 每张表在摘要里最多列出的列名，完整的列用 describe_data 查看 +SUMMARY_COLUMNS = 30 + def request_summary(context: InputContext) -> JsonObject: - """用户输入、澄清记录、编辑上下文、附件和大屏摘要""" + """用户输入、澄清记录、编辑上下文、附件、数据文件和大屏摘要""" nodes: list[JsonObject] = [] for node in context.screen.nodes: # 图表标题帮助判断用户指的是哪个组件；没有 option 或标题格式不符时按无标题处理 @@ -45,8 +49,32 @@ def request_summary(context: InputContext) -> JsonObject: for source in context.screen.dataSource ], }, + # 数据文件的附件由 data_files 的 attached_this_turn 表示 "attachments": [ {key: item[key] for key in ("id", "kind", "name") if key in item} for item in context.attachments + if item.get("kind") != "data_file" ], + "data_files": [_data_file(item) for item in context.data_files], + } + + +def _data_file(item: RequestDataFile) -> JsonObject: + """每个文件只给表名、label、行数和列名；列概况由 describe_data 按需查看""" + tables: list[JsonObject] = [] + for table in item.tables: + summary: JsonObject = { + "table": table.table, + "label": table.label, + "rows": table.rows, + "columns": [column.name for column in table.columns[:SUMMARY_COLUMNS]], + } + if len(table.columns) > SUMMARY_COLUMNS: + summary["column_count"] = len(table.columns) + tables.append(summary) + return { + "file_id": item.file_id, + "name": item.name, + "attached_this_turn": item.attached_this_turn, + "tables": tables, } diff --git a/app/schemas/data_file.py b/app/schemas/data_file.py new file mode 100644 index 0000000..196959c --- /dev/null +++ b/app/schemas/data_file.py @@ -0,0 +1,122 @@ +"""数据文件的接口模型：文件记录、表、列和解析警告，见 docs/screen-agent/data-files.md 5.4、6.6、7 + +文件和表各有三层，后一层在前一层上加字段：列表接口返回的 Info、单个文件返回的 Detail、存储用的完整记录 +接口返回时用对应的模型重新校验存储记录，多出的字段自然丢掉 +""" + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, JsonValue + +DataFormat = Literal["csv", "tsv", "xlsx", "xls", "xlsb", "json", "jsonl", "parquet"] + +ColumnType = Literal["text", "number", "date", "datetime", "boolean"] + +WarningCode = Literal[ + "title_skipped", + "note_skipped", + "block_dropped", + "header_merged", + "header_uncertain", + "total_rows_removed", + "column_renamed", + "kept_as_text", + "bad_lines_skipped", + "json_path_selected", + "empty_sheet_skipped", + "hidden_sheet_skipped", +] + + +class DataColumn(BaseModel): + name: str + type: ColumnType + # 原文是百分比时为 percent，值已存为小数 + format: Literal["percent"] | None = None + + +class ParseWarning(BaseModel): + code: WarningCode + # 表名；为空表示针对整个文件。解析时还没有分配表名，由上传流程填写 + table: str | None = None + # 可以直接展示的中文说明 + message: str + + +class DataTableInfo(BaseModel): + # 会话内的表名 t1、t2……，模型写 SQL 时用 + table: str + # 文件名 / sheet，同一 sheet 切出多张表时再加序号 + label: str + sheet: str | None + rows: int + columns: list[DataColumn] + + +class DataTableDetail(DataTableInfo): + # 前 20 行，顺序与 columns 一致 + preview: list[list[JsonValue]] + + +class DataTable(DataTableDetail): + # 列概况，给 describe_data 用 + profile: dict[str, JsonValue] + # 文件目录下的 Parquet 文件名 + parquet: str + + +class DataFileInfo(BaseModel): + file_id: str + conversation_id: str + name: str + format: DataFormat + # 文本文件识别出的编码，二进制格式为空 + encoding: str | None + size: int + created_at: datetime + expires_at: datetime + tables: list[DataTableInfo] + warnings: list[ParseWarning] + + +class DataFileDetail(DataFileInfo): + tables: list[DataTableDetail] + + +class DataFileRecord(DataFileDetail): + owner_id: int + sha256: str + tables: list[DataTable] + # 删除只做标记，记录留到过期，会话内的表名不会被新文件重复使用 + deleted: bool = False + + +class DataFileList(BaseModel): + files: list[DataFileInfo] + + +class DataFileLimits(BaseModel): + max_size_mb: int + max_files_per_conversation: int + formats: list[DataFormat] + # 文件选择框的 accept，只用于前端过滤；后端按文件内容识别格式 + accept: str + + +class RequestDataTable(DataTableInfo): + """本次请求可用的表：多出工具要用的列概况和 Parquet 路径，不带预览""" + + profile: dict[str, JsonValue] + path: str + + +class RequestDataFile(BaseModel): + """本次请求可用的文件：服务端按会话读取后放进请求，前端传的同名字段会被覆盖""" + + file_id: str + name: str + # 本轮消息的 attachments 里带了这个文件，通常是刚上传的 + attached_this_turn: bool + tables: list[RequestDataTable] + warnings: list[ParseWarning] diff --git a/app/schemas/input_context.py b/app/schemas/input_context.py index e329770..67463df 100644 --- a/app/schemas/input_context.py +++ b/app/schemas/input_context.py @@ -3,6 +3,7 @@ from pydantic import Field from .base import JsonObject +from .data_file import RequestDataFile from .request import DesignRequest @@ -12,3 +13,5 @@ class InputContext(DesignRequest): clarification_history: list[JsonObject] = Field(default_factory=list) # 同一会话此前各轮的用户输入与助手回复，由服务端按 conversation_id 读取并按 token 预算裁剪，按时间从早到晚排列 history: list[JsonObject] = Field(default_factory=list) + # 本会话里未过期的数据文件，由服务端读取并校验 attachments 后填入 + data_files: list[RequestDataFile] = Field(default_factory=list) diff --git a/app/schemas/request.py b/app/schemas/request.py index 1448c2a..ede87c7 100644 --- a/app/schemas/request.py +++ b/app/schemas/request.py @@ -17,5 +17,6 @@ class DesignRequest(ExtensibleModel): prompt: NonBlankString screen: Screen context: RequestContext = Field(default_factory=RequestContext) - # 只预留 JSON 描述入口，暂不规定材料种类，也不读取文件或远程资源 + # 本条消息附带的材料；{"kind": "data_file", "file_id": "..."} 引用本会话上传的数据文件， + # 其他种类只原样进入摘要，不读取文件或远程资源 attachments: list[JsonObject] = Field(default_factory=list) diff --git a/app/services/agent.py b/app/services/agent.py index e7427cb..e19f19a 100644 --- a/app/services/agent.py +++ b/app/services/agent.py @@ -9,6 +9,7 @@ from loguru import logger from app.config import settings from app.core.exceptions import BizException +from app.datafiles.service import DataFileService from app.observability.tracing import astream_screen from app.processor.history import trim_history from app.schemas.agent import AgentResult, ResumeRequest @@ -18,9 +19,10 @@ from app.services.conversation import ConversationStore class AgentService: - def __init__(self, graph, store: ConversationStore): + def __init__(self, graph, store: ConversationStore, data_files: DataFileService): self.graph = graph self.store = store + self.data_files = data_files # 同一时刻，一个任务和一个会话都只允许一个执行，避免历史顺序错乱 self._active: set[str] = set() self._conversation_of: dict[str, str] = {} @@ -40,6 +42,10 @@ class AgentService: turns = await self.store.load_turns( conversation_id, settings.conversation_history_turns ) + # 附件引用的文件不属于本会话时在流开始前返回 404 + data_files = await self.data_files.request_files( + user_id, conversation_id, request.attachments + ) except BaseException: self.release(thread_id) raise @@ -57,6 +63,8 @@ class AgentService: ], settings.history_max_tokens, ) + # 同样只信任服务端读取的结果；会话内的文件每一轮都可用，不附带也能查询 + payload["data_files"] = [item.model_dump(mode="json") for item in data_files] return ( thread_id, diff --git a/pyproject.toml b/pyproject.toml index 563e3f6..127ed94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,4 +20,12 @@ dependencies = [ "langchain-mcp-adapters>=0.3.0", "deepagents>=0.7.21", "langchain-core>=1.6.6", + "python-calamine>=0.8.2", + "pyarrow>=25.0.1", + "duckdb>=1.2", +] + +[dependency-groups] +dev = [ + "openpyxl>=3.1.5", ] diff --git a/uv.lock b/uv.lock index 4ceec1c..05c4f66 100644 --- a/uv.lock +++ b/uv.lock @@ -491,6 +491,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, ] +[[package]] +name = "duckdb" +version = "1.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/0b/d65ea3be00ea79aa276a8388bec588a9cbf409ce637c6d306e5316210d15/duckdb-1.5.6.tar.gz", hash = "sha256:166a91dbfacfc0c9f08cc76c0243cb6d3d4296bfab5bad72a3cfb63140a5b7c8", size = 18032957, upload-time = "2026-09-28T13:38:37.978Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/e5/01e03d30b7ba33a030a4269fdca16ce445ce10f9d29b84a10fdbe0636ad2/duckdb-1.5.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c88700d0ee68ad149a0cc624df21b0f21efc136ea2449aaadd7cd0c9a564962a", size = 32757482, upload-time = "2026-09-28T13:37:29.916Z" }, + { url = "https://files.pythonhosted.org/packages/ba/4f/7f7be626a4649a3948ca646c84d6afc1a00121f292f98e6f0d9ed68330df/duckdb-1.5.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03e4f1b10a8b8ff476eb2b73955590fadbcef978da1167c593114c5edf763960", size = 17372997, upload-time = "2026-09-28T13:37:32.363Z" }, + { url = "https://files.pythonhosted.org/packages/1a/66/9d57573729348d800a0eebdd508f1a833d3714f72e984fef79b47f0e6c45/duckdb-1.5.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34623eaabd2c66ba5c20f1a39486321c3b7d32e4e0e001ced95f81e3372dd361", size = 15514224, upload-time = "2026-09-28T13:37:34.467Z" }, + { url = "https://files.pythonhosted.org/packages/57/ec/97f595214b3a27b4ca42b8cab6d8121c06f3537dcc4d2da7bca0332de4c5/duckdb-1.5.6-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56c0f71c6bee982e9c30568bb12371bf66b26bf129c75d8d7f60bc69d6590a2c", size = 19428776, upload-time = "2026-09-28T13:37:36.689Z" }, + { url = "https://files.pythonhosted.org/packages/68/4a/ab59f4c1f76fb89e28d23f19b2729538e0723c8d328a07e1b8c37f9ee128/duckdb-1.5.6-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73b108c04c932b36c2fa4e41110cc1c3c8cd510eb49f065f92d050be8e6929fd", size = 21537771, upload-time = "2026-09-28T13:37:39.548Z" }, + { url = "https://files.pythonhosted.org/packages/31/4f/9306c442ecad76f2a4d19f249e7fc8861f139dcf748315102eb69de8ca56/duckdb-1.5.6-cp311-cp311-win_amd64.whl", hash = "sha256:dda311932cf5aae955a53fe28a4fc1700c2ab5fa02dc1f165abdd5ec6c39141e", size = 13179009, upload-time = "2026-09-28T13:37:41.981Z" }, + { url = "https://files.pythonhosted.org/packages/a0/40/8a370e998293d3ebbbac4d926db30bb4ac5f700851a06ac31e7093bee386/duckdb-1.5.6-cp311-cp311-win_arm64.whl", hash = "sha256:df5ae02af278e084f54a9730a9f4f211ed736d0bd8f3bc12af925c2effb5b33d", size = 14046340, upload-time = "2026-09-28T13:37:44.187Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d5/d0ab77a0a1702a43171c93874f44c1f6481e30038bd3987df0d77a16a5c6/duckdb-1.5.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:48d07d0651aaeac2c3974afd37599970154b7b79b54c18f27c319c14ccf98d9d", size = 32810486, upload-time = "2026-09-28T13:37:47.254Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cd/b22201de5377faa3be6c38d5f3eaa504cb480392a448bed6a4d2239469b4/duckdb-1.5.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:79de3dfa8705b1ba0d59e7e3252e40ff399e0afd12f485502a6c7bf7c2fd809a", size = 17405278, upload-time = "2026-09-28T13:37:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/f9cfb1493bbdc2f095693a402e42dce1192077f9e11573f00baed6a748de/duckdb-1.5.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dcccce20965e6986cd083fdf192c461685ad0b93cd1ccd0b2a8207f1185f078b", size = 15532943, upload-time = "2026-09-28T13:37:52.927Z" }, + { url = "https://files.pythonhosted.org/packages/53/04/f65ccfaa5a833f2e570c4a140f03c8f95da416da9fe8ed08401f81f8242a/duckdb-1.5.6-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce89a1025a5317ebe9c520876c48032b5247ac574865486648b1a004f6009875", size = 19454940, upload-time = "2026-09-28T13:37:55.732Z" }, + { url = "https://files.pythonhosted.org/packages/4c/99/be75c788a492f8d77b7a1cdc1b19939ae7be0007f2028691ad371a1a33ee/duckdb-1.5.6-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc9619ed7d4ffa117b5155d84b44794366bb6635178d78ed5e13a6024845c757", size = 21568087, upload-time = "2026-09-28T13:37:58.191Z" }, + { url = "https://files.pythonhosted.org/packages/b5/95/889f8508960e47c0a7c75cc5bf57cde8512fc24f8db7b3129cca5388da42/duckdb-1.5.6-cp312-cp312-win_amd64.whl", hash = "sha256:09ff51b230219f0d8b47fc8a1e17fb595ba9fab0c3d96a6de4d00b8ff86b3cf1", size = 13190189, upload-time = "2026-09-28T13:38:00.407Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c9/baab503364a68309f8368c88e77f5341e7d94927bdf3e6d703f0e5035f3e/duckdb-1.5.6-cp312-cp312-win_arm64.whl", hash = "sha256:b8d795c8b2d5634b3269f974aa97f1fdf878f62f032317a52252a151b693fb1e", size = 14021977, upload-time = "2026-09-28T13:38:02.682Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/a476197fcba557738a588ec844747a19bc0a24b0e6f1809e308f29d68c0e/duckdb-1.5.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ae352646374cacf48e9981cf031191c494865192fc436d13667a2531fc5d1da3", size = 32810376, upload-time = "2026-09-28T13:38:05.148Z" }, + { url = "https://files.pythonhosted.org/packages/0c/6d/5466a2b53ddd557644dfa47a763f68748efccdf282e6ae7c4f1bcfb3da69/duckdb-1.5.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a1261e90785e9d29953293e44f60fa073bd1137098924e8de21a037a861b051", size = 17405385, upload-time = "2026-09-28T13:38:07.363Z" }, + { url = "https://files.pythonhosted.org/packages/d4/a0/bf87071170835ee4a34fe764fc11c1c6e7040a0e021b36c1b6f834a4c22f/duckdb-1.5.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97dd7a555b8f5298b76bc7d48a11cb2c64336e8de9bfde783cffb86ea9f54807", size = 15533132, upload-time = "2026-09-28T13:38:09.681Z" }, + { url = "https://files.pythonhosted.org/packages/31/e0/38095c8e140ecfbe847519ac07bcba94301b8fbb76b2870015e33e07f179/duckdb-1.5.6-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:364992ba1089a2b327391cfcb68fd0bd0ce9090cf293baef861a0ba6847abfee", size = 19454994, upload-time = "2026-09-28T13:38:11.836Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/61dd2876bbaa69cf77d7b5c620e52e8b25faae7096f4d2e4a812b52095d7/duckdb-1.5.6-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:644f54ce99b3b61844bc9a3fe80e0aecb1ea4084b1fffc4396d1569db6111679", size = 21568700, upload-time = "2026-09-28T13:38:14.258Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/100730e7785e85268be4d4d5bd62cfc8314e261d2f42efa208243eef35cb/duckdb-1.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:ced693d33ddcee2e5345f077d342c87d2aaa80e41c514e64c9ff2d4e5963c251", size = 13190707, upload-time = "2026-09-28T13:38:16.875Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2e/bc7f44eab4e89ee5c1cb427bb1168ad021d985042e6841ec0694c3d3d501/duckdb-1.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:41ecc75bb9328d72d154a705c1a653d2c5c60f686a5c0c6578aa80020753c884", size = 14020962, upload-time = "2026-09-28T13:38:19.007Z" }, + { url = "https://files.pythonhosted.org/packages/fb/62/a8a30a4c6b94c0861d348ed5633b963f6745a5525527530f02f3c1a7c931/duckdb-1.5.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:aa21d2ad803b2524326e8622d7d96b2bb1ff1d5b60368e1978ee805df9c21fb3", size = 32828003, upload-time = "2026-09-28T13:38:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/71/b7/1dcca0005eb8c67adf9fc06bf0cbb1d2bf4ea1974cc89e7a7c2ad66aac28/duckdb-1.5.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8a1b2ad27d414068cbca06c55cfa802eece10f86ea4812ff082f8ab4cb25fc85", size = 17413912, upload-time = "2026-09-28T13:38:23.915Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/e3ac175443550f3464f2d95731a8b0aae9b4dc3875c3a186c352262b43c2/duckdb-1.5.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c79c6d222b1d015cde73b5139087186b00db65357fb4e2c94c2308fbbf465a72", size = 15543122, upload-time = "2026-09-28T13:38:26.317Z" }, + { url = "https://files.pythonhosted.org/packages/9d/08/cc510a7952aba69d5cdca17f3ef61c95713d86143f2ee9aa3e097d38f50b/duckdb-1.5.6-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1052b8050ef5696e2c0d8c836949c72f3dd11f0690466acbea739613e8e2750b", size = 19457946, upload-time = "2026-09-28T13:38:28.877Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a5/6f8099d9a5a02ddff89e5c85875df3465054845b0920fb0703fbdf8dd2ec/duckdb-1.5.6-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c5e485e59613b8878d1670bcaa7a010f53c5a4da5ae8e08863e5e529ca6182", size = 21575132, upload-time = "2026-09-28T13:38:31.231Z" }, + { url = "https://files.pythonhosted.org/packages/9f/58/762f7159662d7859e201fa05ca29f306795daeabf84f3e087215a966b001/duckdb-1.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:ebcbd09cd8578ab1093393e9b16289cda0e8f1791ac595bf00eb5bad75c3cf00", size = 13713963, upload-time = "2026-09-28T13:38:33.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/69/64d165db322de13f5c3e75d377b6b9694df1821155ad1fa4b14b04601abc/duckdb-1.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:820a8384faef11cd86068ea48c5da57ce2d8f1c7b3d2bdb9be3398317a7c3728", size = 14514368, upload-time = "2026-09-28T13:38:35.676Z" }, +] + [[package]] name = "email-validator" version = "2.3.0" @@ -504,6 +540,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "fastapi" version = "0.141.1" @@ -1499,6 +1544,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "deepagents" }, + { name = "duckdb" }, { name = "fastapi", extra = ["standard"] }, { name = "httpx" }, { name = "langchain" }, @@ -1510,16 +1556,24 @@ dependencies = [ { name = "langgraph-checkpoint-mongodb" }, { name = "loguru" }, { name = "mcp" }, + { name = "pyarrow" }, { name = "pydantic" }, { name = "pymongo" }, + { name = "python-calamine" }, { name = "python-dotenv" }, { name = "typesafe-sdk" }, { name = "uvicorn" }, ] +[package.dev-dependencies] +dev = [ + { name = "openpyxl" }, +] + [package.metadata] requires-dist = [ { name = "deepagents", specifier = ">=0.7.21" }, + { name = "duckdb", specifier = ">=1.2" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.141.1" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "langchain", specifier = ">=1.4.3" }, @@ -1531,13 +1585,18 @@ requires-dist = [ { name = "langgraph-checkpoint-mongodb", specifier = ">=0.5.0" }, { name = "loguru", specifier = ">=0.7.3" }, { name = "mcp", specifier = ">=1.28.1" }, + { name = "pyarrow", specifier = ">=25.0.1" }, { name = "pydantic", specifier = ">=2.13.5,<3" }, { name = "pymongo", specifier = ">=4.17.0" }, + { name = "python-calamine", specifier = ">=0.8.2" }, { name = "python-dotenv", specifier = ">=1.2.3" }, { name = "typesafe-sdk", specifier = ">=0.7.1" }, { name = "uvicorn", specifier = ">=0.54.0" }, ] +[package.metadata.requires-dev] +dev = [{ name = "openpyxl", specifier = ">=3.1.5" }] + [[package]] name = "numpy" version = "2.4.6" @@ -1718,6 +1777,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/72/2b1fe3e303db76ab2ba0c2de2b57822869bcb588db82334d4ff3aea8acf2/openai-3.16.2-py3-none-any.whl", hash = "sha256:660a7f8307e6605342ae84ff4425412e7cd19e5e9b298f975502df2a1a9a5d5f", size = 2053568, upload-time = "2026-09-18T21:27:20.885Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.45.0" @@ -1964,6 +2035,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/04/d52c7016b04b6c5108f26691f9d33ec82a9b65d041f1a9c771137693d618/protobuf-7.36.2-py3-none-any.whl", hash = "sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e", size = 179806, upload-time = "2026-09-17T20:07:58.211Z" }, ] +[[package]] +name = "pyarrow" +version = "25.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180, upload-time = "2026-08-10T12:37:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787, upload-time = "2026-08-10T12:37:25.795Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633, upload-time = "2026-08-10T12:37:33.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507, upload-time = "2026-08-10T12:37:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690, upload-time = "2026-08-10T12:37:46.644Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198, upload-time = "2026-08-10T12:37:52.531Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263, upload-time = "2026-08-10T12:37:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e2/9ab15b88cbfac28e16419ce5439ec29234c5172cb8259301b4ba639bdec0/pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9", size = 35861559, upload-time = "2026-08-10T12:38:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/58/79/a0036dbe1eabe1f73127427342f1d99982584c4a2cde2651d6c93499c6f6/pyarrow-25.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:cc4aa407fde9fc660be3939e49ea31f50f3e9fec17c0ec63159f7711edd3efc9", size = 37628383, upload-time = "2026-08-10T12:38:09.083Z" }, + { url = "https://files.pythonhosted.org/packages/13/49/d93a57d375f4bf0cf82913dd6bb54acafde83dd993be2282c81ac5616cad/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:4340f0ba6c1d2e13f21658de1d7c662ca2545018568d0030a1e9afca159d87e3", size = 46820190, upload-time = "2026-08-10T12:38:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/60/c9/711ca85d79f1ec98f29a5eae2b051e25b4ecec5de3e3c0e2d5c5dcb15664/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5389cdf79447ed1515c9e31620e6e1e2302249564d603f2ad727d4f6d313e4c3", size = 50102437, upload-time = "2026-08-10T12:38:22.487Z" }, + { url = "https://files.pythonhosted.org/packages/80/53/8fb8359ff17cfb6263a1cf3ebf7caec9fe197de118719e84fcb1d0618026/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d51592cb7561e87877c506113e7adbf1342ab579e6c21f0ef44b8ba41cb74c80", size = 49942424, upload-time = "2026-08-10T12:38:28.755Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/4e5ae02a9341571b18a6fca380ac7a58ce6ddae7ab3c060208c0a1e79f02/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6109c94d8b9f3b17a041daca16cacb2f651ad8f1ef70a4232c2c0f37a23da2a8", size = 53144206, upload-time = "2026-08-10T12:38:34.862Z" }, + { url = "https://files.pythonhosted.org/packages/65/ee/197cbf47e49f83e6ebeb946a5259a48a638dea27ac774db42fe78022179d/pyarrow-25.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:8858d7bfc22e3f51529aeaa4077225029724623e4595dc9eff8c793935c34140", size = 27953934, upload-time = "2026-08-10T12:38:39.808Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8d/8f271a7a034c834910ec925d56fa4b29733b1380f5289419f5aaa3b02777/pyarrow-25.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c7c534ec03c358a76ea3e505e74c1b6aef290af90c444dfd092dbfe23e755b85", size = 35855328, upload-time = "2026-08-10T12:38:45.489Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/5bac242f4e841b9971d5eb94fdfe2577e2b70be983e27401e72055786037/pyarrow-25.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dda9470024204d7bbf2042b47c6e8a0e47a3eeb8e34405882dfaea6577e0c153", size = 37622415, upload-time = "2026-08-10T12:38:51.107Z" }, + { url = "https://files.pythonhosted.org/packages/63/1f/96d03b4e1506524f7087adb0fd6b2f69f0c9c7aaff1ec36d8030082e15a5/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:44a9120ce5bd81936b8ab9a88076e3fd47c2c6838e0e43630fed83626aca81d9", size = 46813813, upload-time = "2026-08-10T12:38:57.773Z" }, + { url = "https://files.pythonhosted.org/packages/98/d6/33a411115b61dbfc16ad6ad73e71730f6fea654ee3667673bc53ab0e2fe7/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f", size = 50104452, upload-time = "2026-08-10T12:39:04.579Z" }, + { url = "https://files.pythonhosted.org/packages/33/ae/b1b97c9ca87f9f9ddbb5230c798df94eccce61bd79b9b45458c69a478588/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f89685964f46e4216103c75483aac0c0692a5f72212d7ca835adba5ede56ce3", size = 49951343, upload-time = "2026-08-10T12:39:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/98/9e/a112df5cfd5a68cb1d9fc31cfe38c28d5aec9f10865ce37ecef2e4450873/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6943e2fe7954d29d84de45d29d34c8dc36ce96570e67d89aa9976e650a4a9138", size = 53144784, upload-time = "2026-08-10T12:39:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/31/24/97e8bd98f1e3b07e2ba08bcdff690674fbe16d69a7d2712cc3884665e615/pyarrow-25.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:31e49a7888fcdf3a835da33ae777f6bb9a866334e5a789282fc26dcf426f7f15", size = 27870159, upload-time = "2026-08-10T12:39:26.161Z" }, + { url = "https://files.pythonhosted.org/packages/36/4c/b525824ad3094076919273cd97db61fb3d78252dee76fa3b8dc8f76774aa/pyarrow-25.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:bf0b672390cdcb640d7288f96b826d71ff4e9abb254a86c89890baf51a29cee6", size = 35885255, upload-time = "2026-08-10T12:39:32.366Z" }, + { url = "https://files.pythonhosted.org/packages/08/62/448bb0e940de41aec31d1a956e63ad9c54afdf122a103cc3ab20c2a3ce33/pyarrow-25.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:38a9a4b4b9613380e200641891495a56c3d5a98a092db4a870af9975e220471d", size = 37644461, upload-time = "2026-08-10T12:39:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9a/13587e38bd4806fd218f50fd13b8903fab60588a699ff0c406372e5b4043/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b", size = 46877146, upload-time = "2026-08-10T12:39:43.722Z" }, + { url = "https://files.pythonhosted.org/packages/8d/61/1c5d1229fa21da4cff5365e41e57177aaac57c563c727f35419b8513d1c1/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:9171748cdf796972d85a4b60157c279913e242992e350c90c7450182a9838b2a", size = 50131616, upload-time = "2026-08-10T12:39:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/43/20/291e1d65cc0b09aa19f03cf25cf51a2f5fa94b5db315178f2d254ed5cad4/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b7a296aac7a71fa0886c08e155ddb6c636a50013f801f6178daafa0f9e726188", size = 50008879, upload-time = "2026-08-10T12:39:56.891Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7c/1b7c9ec28e76576337e4f97b31141c9a181b89b6d1d6221e9d8205621a58/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0fe7c8b6c03969b49c8c66182e4a18e3819ab92d07cfab5d8370c531b9369ef0", size = 53170864, upload-time = "2026-08-10T12:40:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/75/f3d789dc06011a765d14d86bda799cf72ac1d715b6a6edecaa0d73d95062/pyarrow-25.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f", size = 28620729, upload-time = "2026-08-10T12:40:51.41Z" }, + { url = "https://files.pythonhosted.org/packages/fc/05/647a8ee6f7c2662feb6921315617bc04dcd6034763fb61b1199720bf6162/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:59a2de54c0cbd954da861eee4d1d330f8e909c45b53455baef696380f2c55033", size = 36130288, upload-time = "2026-08-10T12:40:11.014Z" }, + { url = "https://files.pythonhosted.org/packages/93/f8/c9ee997554d7bea94520667dd1933f109ac1da3ee3556d2b49381e023484/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:35935cd5de130aa5cf4dea052a63e6bf2e17006c35c3a468194242b9b2bf5956", size = 37762187, upload-time = "2026-08-10T12:40:16.592Z" }, + { url = "https://files.pythonhosted.org/packages/a2/08/a28c01c7fe9e96e8233ce2d13df1d402f4f999f848f51d2daacd6bb4c036/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44", size = 46888003, upload-time = "2026-08-10T12:40:23.242Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b9/58612e977d28dc58c878448866838369ee8da2f1e7cc8ed2c84b952aafee/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a1fdfc6659b6b19022f2e50627fb5cf7156a66c46bf4299379955cbe742382a", size = 50079036, upload-time = "2026-08-10T12:40:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/72/13/66e1402dcc860e1dc2760b1e0292c9a569b62b3bccab69def1b3e907d006/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:169d3429d5be7c752125890620f75a60776d38b0035eddae939651640822332e", size = 50040226, upload-time = "2026-08-10T12:40:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/78/10/3f1a5497a7ef732ab0f03ecca3e66d89d9c0f57fdc61b4794c456b781f01/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:119297a6dc197e45d9c6d4415f7814a67ffa36c180d26f68c154c58067ae782d", size = 53149035, upload-time = "2026-08-10T12:40:41.454Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, +] + [[package]] name = "pyasn1" version = "0.6.4" @@ -2239,6 +2353,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/ee/5d3f952a7fc8d0bc73706a92e08c3ba13a5fe2435456758d9c936a12541f/pymongo_search_utils-0.3.1-py3-none-any.whl", hash = "sha256:1865e5a0cc01c4b0c4a366e6f1142baa92c0dbfa4b7e7e91603fa83da92bf5b8", size = 20144, upload-time = "2026-09-22T12:34:59.797Z" }, ] +[[package]] +name = "python-calamine" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/36/473a55063a6af3739cdf4b879869d3067e677f3dfb52ec8fadac23941a33/python_calamine-0.8.2.tar.gz", hash = "sha256:b2000c085722afd01d973af3d58325d26cfd798b3665bb0924e3a658351ebfad", size = 227187, upload-time = "2026-07-13T13:58:26.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/a0/33beab8ace41d872de57da04d06ab001e31b42d283defd42a81f5653526b/python_calamine-0.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:29b2ee38dfa98e430ae4f32849fb46bd47befd8adbed0a08c290326371d195fe", size = 860315, upload-time = "2026-07-13T13:56:08.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/e6/cbd088173589d7f78b5aaab821721a790e8d621a653512de3c412f92ac0a/python_calamine-0.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e44f54daa383a64704b8f975e0ee793c429bcc0de67bd231b1762179a539475", size = 853168, upload-time = "2026-07-13T13:56:09.83Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/612f005c24ead4ca1dc844371a443a6b74a8b53e66f506d0cf8edf9097bb/python_calamine-0.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6dd67cf7806e2b29b04d32453a1c6c4b304b2e86120813f3a96696245a8ed461", size = 927629, upload-time = "2026-07-13T13:56:11.519Z" }, + { url = "https://files.pythonhosted.org/packages/74/7b/03a261f4489eb04df88d770d317e021858e37d70a958188dabab19a9fe58/python_calamine-0.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:28ceb21f80bd251e48265a7742235c87488a4cac9ff91f523fb3fdc1e56195dc", size = 920851, upload-time = "2026-07-13T13:56:13.209Z" }, + { url = "https://files.pythonhosted.org/packages/c1/30/f84cdb77a2d1034703ec5f2c9a7bee469c7d99f1e54374b247ed319a3164/python_calamine-0.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22434c553f4f6b945f63caa271351ec9f4d6fd11bdb97cafc094c474f13aabc0", size = 1090840, upload-time = "2026-07-13T13:56:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b4/531f2878776b601e3e2895a7f8a3b41729ca7447fc6d6e59cf90d55cb03a/python_calamine-0.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca82ceef14d9194caf0a62e02749b8449d92430b11523c99c9670b7a763fa5e", size = 989963, upload-time = "2026-07-13T13:56:16.484Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9e/f8879a6c1334997825d6f5cfba1552f7fe9b1dd650e3b03e8bc0b1f5e875/python_calamine-0.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:819c86fbfc638eca46b7c06134632959a2ce8ae20aeb580d89b4b6f6cfc5fb94", size = 924123, upload-time = "2026-07-13T13:56:18.099Z" }, + { url = "https://files.pythonhosted.org/packages/b8/88/de03268656a12fd9a73974cd836558f113db152202c93160e57fe0dc10e5/python_calamine-0.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ccccafe6767027eae0ab5493b0afcbc35da38aa64a9b85657bbe8660cb60dd98", size = 1003606, upload-time = "2026-07-13T13:56:19.772Z" }, + { url = "https://files.pythonhosted.org/packages/ed/d8/7ec6aa10bea0d2f4c535d6b7685a58f7e4ba24f7323dfb05aaa789f5baba/python_calamine-0.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:43648db3d6773d4e56b23d405fa3e91ed3398e3b05b9244d50dbb61ffb67d34a", size = 1103990, upload-time = "2026-07-13T13:56:21.582Z" }, + { url = "https://files.pythonhosted.org/packages/cc/aa/969ca12f2412ae4b1c6830b42cc6c5a419704b76ddabbb5e7fec803275aa/python_calamine-0.8.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:ebf7a362249f80ff1519e6f568f3046266cbb3157fda262cd725013e17b18d9f", size = 1195526, upload-time = "2026-07-13T13:56:23.407Z" }, + { url = "https://files.pythonhosted.org/packages/20/81/23d9f1ca602a194a57aa7bced4a554ec57532377fbe3955dcfcd227fefff/python_calamine-0.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:71d5ffc0d81bf9d6b8c5051a256fd4e6d0f0bcaf65700ad7d6fb35e35712d932", size = 1139854, upload-time = "2026-07-13T13:56:24.875Z" }, + { url = "https://files.pythonhosted.org/packages/ca/9c/b3e73e321c2fa78cf7f64720801fc5e74e8be722129293a1c5fe0c3d7e71/python_calamine-0.8.2-cp311-cp311-win32.whl", hash = "sha256:ac23687b41bbe4a6964a69420051cc59e81d4fcf32e5e82342fd3c970d141f24", size = 738563, upload-time = "2026-07-13T13:56:26.429Z" }, + { url = "https://files.pythonhosted.org/packages/05/e3/d13fc340b0a0b0010d3c5799ed6c578e13cbc6501d629ee301e7e37468dd/python_calamine-0.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:c94abc66f8b544e5fc126dfaa6b41b77a394adfe09dac95e20679823e41e38be", size = 777797, upload-time = "2026-07-13T13:56:28.112Z" }, + { url = "https://files.pythonhosted.org/packages/53/30/b00a62b6c910d83511454f1474896432c839ccf53183f184ef5cbf822fa5/python_calamine-0.8.2-cp311-cp311-win_arm64.whl", hash = "sha256:0288fa6488f6ced089d3848d572fc41098e4e7c9e09976d269043957ca432b7b", size = 749055, upload-time = "2026-07-13T13:56:29.749Z" }, + { url = "https://files.pythonhosted.org/packages/13/18/adfb951c33dedf37c3c9f5804af04eb8a18ba12dc0d3901c98563462f565/python_calamine-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0efea8e1fc617d43351abe7c9f8ce7585b3a546d08e5c26d87d1629369a4b6e5", size = 864109, upload-time = "2026-07-13T13:56:31.591Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5e/d9b2cee430feb7eeee2d84b19e61266cda0e2880d126441a7f19b31772c7/python_calamine-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a99eb1a74f9a93af4c5a5586e9fa72de1879721c47fd21aa9e042deb6832e30a", size = 846205, upload-time = "2026-07-13T13:56:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/de/a5/4cbd1a01279372a20319629ddc832806914827bd1edc8801ee7956c275fe/python_calamine-0.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96daed873be1f70d22795fbf197d3d833655aad6a149c15154651b7773d3a1ad", size = 923414, upload-time = "2026-07-13T13:56:34.952Z" }, + { url = "https://files.pythonhosted.org/packages/45/1b/9d646373c241082ce4488a0a297f8834a2513e17c6ce098506f91bb0f4c4/python_calamine-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:17314a59fe01bb1bb7010240a578bf85e49b47af955622ae5f06f286d6d6954f", size = 918317, upload-time = "2026-07-13T13:56:36.467Z" }, + { url = "https://files.pythonhosted.org/packages/34/0d/73f4bcba43fc7a89ec48cbe1459d0436d6205a85ce720af54682bd14116e/python_calamine-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1396287f66a54ec2e68c6448f0c782173dbb3e8c2aca4089195d8c9a52b46c72", size = 1087802, upload-time = "2026-07-13T13:56:38.113Z" }, + { url = "https://files.pythonhosted.org/packages/47/54/b71763efd1c868d096fbad6f080de39ac83d832b456bc0ff285e74237723/python_calamine-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da1b26061783d2bc1d7aabc83cce63706cdacb6b03c6c6a37d3ea33ad34b7b4b", size = 986451, upload-time = "2026-07-13T13:56:39.743Z" }, + { url = "https://files.pythonhosted.org/packages/27/61/deb1b368933ddedeedbd7bef8aa71185eaff66dc058b44ca9f4a419925d5/python_calamine-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a148bf41efabb39ee99eac2552223c3605d29beaed6bd7a14a2990e07022e3b", size = 920176, upload-time = "2026-07-13T13:56:41.308Z" }, + { url = "https://files.pythonhosted.org/packages/c9/df/7645e4d3ea983cac1bfe2643661fd03b92db2736cd97ba1bda367d86a61e/python_calamine-0.8.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:21afb508d71f83c12c1c01fef21d9285f91820d052998e44706571bebd835cf6", size = 996879, upload-time = "2026-07-13T13:56:42.951Z" }, + { url = "https://files.pythonhosted.org/packages/af/b1/a88e1c7bc883f2767cc09247d395ce530c903a6bd2cc40e1f2000e6eaf85/python_calamine-0.8.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0cd1c5592b4a2cc7b0a92c5289f0eeef74e8ce3c5af9cc12361ae75625b14926", size = 1098763, upload-time = "2026-07-13T13:56:44.708Z" }, + { url = "https://files.pythonhosted.org/packages/0c/59/de0fb08a1745d30d85cbef6b474e975a92bb7c6daf6aa37fc63960ae4674/python_calamine-0.8.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:70e6e7aa5e74234254cae87e60a7ccb525e6962f73741f2b1927536aa7cb0093", size = 1192807, upload-time = "2026-07-13T13:56:46.379Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f5/d6956857490924ab9945d3be6624dadee1918d8630288009a2b7ef7700e1/python_calamine-0.8.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2c2d65dd0cff571ad9463fc964194507e31295387bc1bbc76c2604959d694c0f", size = 1136033, upload-time = "2026-07-13T13:56:48.346Z" }, + { url = "https://files.pythonhosted.org/packages/12/06/1992fa129dde6f7c5feed07d3b74df37a231782922af2ca6ff89917cc676/python_calamine-0.8.2-cp312-cp312-win32.whl", hash = "sha256:8938f66e6797d9d778504e0f827641d5067784dc68f18d2c2307abe615fcb85e", size = 734193, upload-time = "2026-07-13T13:56:50.546Z" }, + { url = "https://files.pythonhosted.org/packages/63/31/aed4b50630ba23bb3372b4cebe930a92201dfd2c1c072e2d1570a042542e/python_calamine-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:7a64237cc6b3d56b478686adff16502aecaf356543204bd6c70d754023a3d6aa", size = 774599, upload-time = "2026-07-13T13:56:52.278Z" }, + { url = "https://files.pythonhosted.org/packages/84/c0/23b910999d2d9e1b9145fc3be645add1e1cbc90cba4a39a9ca9d23ebe678/python_calamine-0.8.2-cp312-cp312-win_arm64.whl", hash = "sha256:24607bde04b76f8329dae4963d22830a78f1eaa0622325217e53c5377143384c", size = 731385, upload-time = "2026-07-13T13:56:54.041Z" }, + { url = "https://files.pythonhosted.org/packages/0a/31/134e7b2ecdde486ec4e5fae763f84973fac6a097971aeb9648c0031ee365/python_calamine-0.8.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:923825ea049f7a16bfa0333e237909dfd109c7a44ad156c6d83921a768382020", size = 863158, upload-time = "2026-07-13T13:56:55.91Z" }, + { url = "https://files.pythonhosted.org/packages/a2/aa/36ec3c85c50b08c8f4df4adfb762a6978ef0da08264d5813ff3b55518137/python_calamine-0.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed9a48c892aed565e6250a9a38222806db49334868e174b30e6670caa4231c4a", size = 845854, upload-time = "2026-07-13T13:56:57.584Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/4ef12d9c4d3774410eec3a1b377542523ee18b606b91b0281f8db5ac8819/python_calamine-0.8.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f28ebfaa0ca306ec25df2cf6d5cef4838769ebd5536edd1eb0aa9a77935189fa", size = 923463, upload-time = "2026-07-13T13:56:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/df/a1/41db6b31011ac5c4ad3c05f4f8078a7d24d81f4288c756829cbdc941cbcc/python_calamine-0.8.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0a229632dde778a03d3c1ff159da4762e04d8b5e69be94edbf005da152de944", size = 916869, upload-time = "2026-07-13T13:57:01.063Z" }, + { url = "https://files.pythonhosted.org/packages/72/70/b582691901b90b52884b8f525d4be7e27e423040b17ec4f50d0f8ac8f4e5/python_calamine-0.8.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b67798d130fe8c89891c3e75f9334e0e18c2c8e6527d276954c31c9dc426d39b", size = 1086769, upload-time = "2026-07-13T13:57:02.761Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5c/a05493fdcbf8aede4362150edd0dfde4e87db53c23f1668cc6aa4d4b40a3/python_calamine-0.8.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eebc266e7c40d337112a9cfc82f9c9d7587121fa4e86673da1e97f8855039c", size = 986561, upload-time = "2026-07-13T13:57:04.346Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/8c12291275f63d314723d1480be017f0c32d5fd4dea76b66205ebd05dee2/python_calamine-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efe476889191a80b05a514b37f89a7d364e1ad19783918e93b0a053e2e2e9f08", size = 919538, upload-time = "2026-07-13T13:57:06.049Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5d/dc12ce14a748b180cfa22d6a9e5dc372808050466666ab9776623f437c61/python_calamine-0.8.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:220b1a2962f5394edc0f5918d91e4dd3e9f3c6c521d0f2a4ccaf3d3ace980a28", size = 995727, upload-time = "2026-07-13T13:57:08.315Z" }, + { url = "https://files.pythonhosted.org/packages/ab/da/72986560b082a005d88e5b74cc89686fa8f767a5173436b9dd3d0f43a33a/python_calamine-0.8.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:02d52d6d7792f82d97f1b8e15001fc08f9ab75a57a62fd849313c0a6b64d38af", size = 1098601, upload-time = "2026-07-13T13:57:10.071Z" }, + { url = "https://files.pythonhosted.org/packages/00/9e/e5ca987e17e043079a391a1fff7c13f2b5ed6908a25b54baac2e41d2db84/python_calamine-0.8.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:c55112b2b0f65f7e6757629e89d0b063689d8fa821736d9b81a1468db08d0085", size = 1191819, upload-time = "2026-07-13T13:57:11.758Z" }, + { url = "https://files.pythonhosted.org/packages/78/23/4cc7cc53d342ad7b74e3780aefd60a6af0e132f3e5d964a1847ccfe32ea5/python_calamine-0.8.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:2e8f0753f1ec064704e256a2d5793f42f0cb256710501cf060f5f3a47e5dabc8", size = 1135376, upload-time = "2026-07-13T13:57:13.391Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8e/1a9649ffba117731b44ea57cd6c0b7925cc5a04b9d6f7858031cd9987f01/python_calamine-0.8.2-cp313-cp313-win32.whl", hash = "sha256:f276aa353f5be38e15968f861f7102abf53d1fe261638846e7e9c9913a124d7c", size = 733696, upload-time = "2026-07-13T13:57:15.028Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d0/55b921a2459e4b1cba914a2ed94d5ad0d711c2015e33ab201038f4b1627e/python_calamine-0.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:62b7e41046056bfba7d2957a8a1566fc22a010a2929f0c8174c7a911555d0cf9", size = 774314, upload-time = "2026-07-13T13:57:16.753Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ea/11d2d984d71673c6070a23d9a5fdd9bf174f83eb7e7c8909e27a4f9af740/python_calamine-0.8.2-cp313-cp313-win_arm64.whl", hash = "sha256:024a367c90a61c02743f674025bcdd686a1cbe6466bb8fc67398ef5fbbfac4a2", size = 731178, upload-time = "2026-07-13T13:57:18.459Z" }, + { url = "https://files.pythonhosted.org/packages/c8/51/982cd7edc5e8886090adcc3b436e87a8ff8932a748269fba30ac6151551f/python_calamine-0.8.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:612904f321fa1c399f03efab0f4a557914d7b8a155d25f244cb816a051eec82e", size = 866292, upload-time = "2026-07-13T13:57:20.229Z" }, + { url = "https://files.pythonhosted.org/packages/f2/30/816181f86ec744c142e8efc6fdac49ab61221a1387fbd13798424a72995b/python_calamine-0.8.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5dd0d8934c5b65392b7fe6f149c5b2eec0ce422ec50f35a389b579f53a3ad94c", size = 847675, upload-time = "2026-07-13T13:57:22.034Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/2c82689a076e438fa310fa8119e5e1d1166e2419a122f6137808a1a70a64/python_calamine-0.8.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccd41bed56ae3a7ff0de6f71860cd13edcf3ebc42eefb58eab201b5cc6ec7660", size = 923867, upload-time = "2026-07-13T13:57:23.739Z" }, + { url = "https://files.pythonhosted.org/packages/d8/3d/8e5da593780efb767f0d0fcaccec9f4bcc6f9beba9281969b2c51fee05f8/python_calamine-0.8.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:19c8d87444fe83ca0cebd95e01cb7166f1844455120319de4ef8581e03f63b22", size = 917529, upload-time = "2026-07-13T13:57:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f7/2f/56334bdb7c1bbdb2474ff927afeff72ebb558a301e2dfbb1691e6969e0e1/python_calamine-0.8.2-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5be7dca31f391cc2bc3d86a9e1ef88d51240565e9849b23642b3b95bd7ce29ad", size = 1089145, upload-time = "2026-07-13T13:57:27.658Z" }, + { url = "https://files.pythonhosted.org/packages/2d/99/908da083c1faa6ee215be079238f7d25022eaa577dd9fa3e88814f39dfec/python_calamine-0.8.2-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40f1c657eaa988a50638fc584226391e6897ba676ec3e35aaed0db7d8b02e3ed", size = 987785, upload-time = "2026-07-13T13:57:29.57Z" }, + { url = "https://files.pythonhosted.org/packages/52/90/f7a71c8991911a431ed31f83cd9f3a49bf0e0e3395a27d3a47f60dd55826/python_calamine-0.8.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d3cfce465ce82eb9100e5e90673a5844fd46eb7b8148c5404c70f941fd8280b", size = 921563, upload-time = "2026-07-13T13:57:31.268Z" }, + { url = "https://files.pythonhosted.org/packages/b8/2e/768d8686fa4d87d6bcdee21959845d767e4913066e562cddd048516f9edd/python_calamine-0.8.2-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3655c08901562d7c44853ba966a62104637f216a67aa5ba5628115067811b3f2", size = 998078, upload-time = "2026-07-13T13:57:33.483Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b3/12273f5ea9df335eeb896e8f8d54dfd888b52d57f03d0e6163c86be28ac1/python_calamine-0.8.2-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c72d5310e28a517cbb14812cd60a41c264d65b2424e6413c08f5f7160eebafd4", size = 1099253, upload-time = "2026-07-13T13:57:35.404Z" }, + { url = "https://files.pythonhosted.org/packages/02/64/3f222cc693eb6bbae0cf1bfa3c45300d6024cb515a8380779ca46ea096db/python_calamine-0.8.2-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:2e88eef40b2f39aa9c32cf6d2d7e6bf0c8669547f8a196883bae6d202458b358", size = 1193093, upload-time = "2026-07-13T13:57:37.147Z" }, + { url = "https://files.pythonhosted.org/packages/56/a0/91d06d24caa01cbb7d020155a0ba565d5acb15b873705c72c69dc36351fc/python_calamine-0.8.2-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:8e9b57dc1553bf86e2793d86e0b9268855f5382391f01be78c9c06aa30bad3a7", size = 1138084, upload-time = "2026-07-13T13:57:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ba/279277863792e26982c6a9c9b9700c36b4319e7ad3424fc4e8164b6db5a7/python_calamine-0.8.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:74f5038df6c40316e7f1d20c7d478baf6a6bd227e91f17aae2723014156a895f", size = 515427, upload-time = "2026-07-13T13:57:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/5d/39/29677d6a4b5bffc78b35bb6a30e904ec0af71dc0bfc105a342150fdce70c/python_calamine-0.8.2-cp314-cp314-win32.whl", hash = "sha256:853e6789c781184998bc13c9a1affde29ef76d07112e339e379c45f4ffe0d911", size = 722761, upload-time = "2026-07-13T13:57:42.399Z" }, + { url = "https://files.pythonhosted.org/packages/c7/71/bd69ce8cc651d1db6a6c2051c1d34fa22fc6ea6bbfb3cbf1feb49fb2b25a/python_calamine-0.8.2-cp314-cp314-win_amd64.whl", hash = "sha256:408c60e8c5671c87dec3fda2268475bafcd05dfc7684c72ad15adb050ae5dc45", size = 776909, upload-time = "2026-07-13T13:57:44.451Z" }, + { url = "https://files.pythonhosted.org/packages/97/00/1da6c1d4a9a83be80273e2136b10a665da53012a86388895e9554d65a7e5/python_calamine-0.8.2-cp314-cp314-win_arm64.whl", hash = "sha256:16630004b39bf4a0dcce09653fd28390781a2db07e9d8bcb0f85d8e5d1814707", size = 746538, upload-time = "2026-07-13T13:57:46.195Z" }, + { url = "https://files.pythonhosted.org/packages/33/ce/a089dee93f684398101a807177cc9fd5c0f04ea98dbf1c6b12d761a3e0b1/python_calamine-0.8.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:1d1db66f7043d13fd6f6c5d77c7b34598864ad14e613d02cc538ec3e9b46d296", size = 861568, upload-time = "2026-07-13T13:57:48.146Z" }, + { url = "https://files.pythonhosted.org/packages/64/ab/6f5a02a28d06070bcaefaa60f980097ea10ce1302912822e9cad976afc83/python_calamine-0.8.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3478b5f950e9722d284271b3cf7a7e83c43a0f93cb665474f8a158e12699a52e", size = 844446, upload-time = "2026-07-13T13:57:49.849Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9b/01bccf504edeb5419c4a138302004c7a3b14db537554f0da8b4b217c94d1/python_calamine-0.8.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c307f6e5f5c55a58a3a64b96dd3a7ca437570efea8e1b8515108b1cfd9adf46", size = 920817, upload-time = "2026-07-13T13:57:51.816Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/c555639d38376908a2258ad7c06e8f12ffb774343531dddd5acce1a7be9f/python_calamine-0.8.2-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7dc9d63b44b0681a3ff4be1810bf63042d3492bba3e4dbf0273318170fb82e59", size = 915360, upload-time = "2026-07-13T13:57:53.576Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a2/ba37bc638ecf4dbf651cb0e9db4fac495881739434c7eb106653bc686f8f/python_calamine-0.8.2-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd903b60575bf8ec45c5797a64d1960e46cafe0d5ae479131d739aba6f094775", size = 1086101, upload-time = "2026-07-13T13:57:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/6e/41/732501f6a965284dc429c56d90f56328ad5c9ec57ccc6404b6bd5a07605a/python_calamine-0.8.2-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5770e6fb816792ff329505d4a6a0eb3439b0b5e5c05f601268fb3bae26a52844", size = 985151, upload-time = "2026-07-13T13:57:57.357Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/90f3646da7c8546153229b9df2f97cd3dcb644d747bc28e0ae7773e6f3fa/python_calamine-0.8.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7351f2e8833ff6d6e2e6bf7bf1327a69527513f5e8cfb8d51f13ea32d2bcfb16", size = 921694, upload-time = "2026-07-13T13:57:59.071Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7b/96aedc02df1cb762578b44af05413d51cae9b041319c0c200ddfc4aa36c3/python_calamine-0.8.2-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1804917c652b674e9456419f4c80add237b2edc94e192e9b6b7ccd7b2a7c4105", size = 996333, upload-time = "2026-07-13T13:58:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/c5/9a/b152b3d25f778cf5b90b60df9fec57ddb2f3d5c2c16ed08e751421a64d30/python_calamine-0.8.2-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:4ae30729fa4032f7a5d3fdca6968f4b5774967ea67bb62d7e5d198350419adc4", size = 1096283, upload-time = "2026-07-13T13:58:02.669Z" }, + { url = "https://files.pythonhosted.org/packages/e7/25/9c8c877f1632e70681437471a63e4d0ec150f72219665e1bcc0d2ba9d14b/python_calamine-0.8.2-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:67dad173dce245859eeb8335e8808835b53245bab75bf5a5b9f20435357c5156", size = 1191124, upload-time = "2026-07-13T13:58:04.747Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/035b56decd6c0855203a21235c501c9d1c0965a0cd80f5299a561f05228b/python_calamine-0.8.2-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3013f0872b6e4f52173d98fa0a6dbb83f1858440c1b858690893339e7d813c38", size = 1137036, upload-time = "2026-07-13T13:58:06.457Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ff/8bda87f105165a4535615d07f923f8490c2daa2a0bc2489549e9b182fe56/python_calamine-0.8.2-cp314-cp314t-win_amd64.whl", hash = "sha256:288a8f4556ae8df3c813c61da2749eb627b1cfb97222118f5413bb8c7a0c7f02", size = 775005, upload-time = "2026-07-13T13:58:08.221Z" }, + { url = "https://files.pythonhosted.org/packages/bb/4c/f39b08d79e452bd3c7edb15285e096fc8b1fe67a76ad10c4622d4baacfb1/python_calamine-0.8.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f57c2cfc7854fac87fb0c8e4fba5eb25184f75f7bf340847deac6726d2c3b601", size = 744119, upload-time = "2026-07-13T13:58:10.018Z" }, + { url = "https://files.pythonhosted.org/packages/6d/76/7c558e6eb904fd05cf9c3e59464df5d03f507d87ce564df0452f15d848cf/python_calamine-0.8.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1d13023efd46b242c7ade9ea043bee11184bd8cb4f1da76832ebc18282eda858", size = 866176, upload-time = "2026-07-13T13:58:11.928Z" }, + { url = "https://files.pythonhosted.org/packages/56/4f/047d131bdea829e147d62526fce45c0ce1abb385ef3a1b8dfc135f5b0016/python_calamine-0.8.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:cd2fe069bd3a7894eb78259252493b1272be5ed82edcc35131565daff8e74758", size = 855816, upload-time = "2026-07-13T13:58:13.785Z" }, + { url = "https://files.pythonhosted.org/packages/3b/23/ae50c3d6bfe8d0e0d2c9fbcc242fa23a0847217bb8316cc7e59f5425fe09/python_calamine-0.8.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d5ab4a3e6e710d8c77c314a70b0c89e6ccbe934e10fb97f1d5970416ababf5d", size = 929717, upload-time = "2026-07-13T13:58:15.746Z" }, + { url = "https://files.pythonhosted.org/packages/43/fe/0fc0f06d82e8743c8f95cdaada31d7c54b114b787e9577fe319cd6f81c4a/python_calamine-0.8.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a9fd05a000d4e28ad6cc87f258ea46681c24568c2738cda24ec44fae9aa20a3", size = 928256, upload-time = "2026-07-13T13:58:17.578Z" }, + { url = "https://files.pythonhosted.org/packages/59/e0/db2a1b9fc7d2bdd8300a376c16f45edb16049eeb151a59bf04ac6cc80ad7/python_calamine-0.8.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4f4323d0f05be5f729fa8c165bdc6f19c89114f538ab994f0de5da90adc05302", size = 1006762, upload-time = "2026-07-13T13:58:19.55Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f6/7e1b30dad9a0bb18d71bb4ebffab46fc835711e07450b40c987825315d7c/python_calamine-0.8.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e38b7285af2e7e7628173754671feffecdc9705f007fd3b01b6a899be61a4be8", size = 1106333, upload-time = "2026-07-13T13:58:21.384Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a8/7a3ad60bff627d81f74ba9e552c86d8821fcdd4f24b67511487c1e578eb5/python_calamine-0.8.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c5cfc20448c5fcfee42d26e70bb9416892b1b26aad9fd7126de31cceb13826a3", size = 1199755, upload-time = "2026-07-13T13:58:23.35Z" }, + { url = "https://files.pythonhosted.org/packages/80/a2/7d953a413905314ef2d03f9b6d552f7f8685fb28a175905b0de51d03b94c/python_calamine-0.8.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:9b5fc52cc2fb496c038781ad6ca86d5017a615075ae04a45c7de6160bfe4beb9", size = 1145928, upload-time = "2026-07-13T13:58:25.242Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.3"