feat:完成大部分数据文件上传解析功能
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
"""数据文件的本地工具:describe_data 查看列概况,query_data 执行只读 SQL;问答和 screen agent 共用
|
||||
|
||||
工具只能看到 DataContext 里的表,也就是当前用户在本会话里未过期的文件,查询引擎也只为这些表建视图
|
||||
工具通过 runtime.context.data 读取 DataContext,用到它们的 agent 上下文都要有 data 字段;
|
||||
上下文类型因 agent 而异,所以写 ToolRuntime[Any](不写类型参数时按 None 处理,序列化会告警)
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langchain_core.messages import ToolMessage
|
||||
from pydantic import Field
|
||||
|
||||
from app.config import settings
|
||||
from app.datafiles.query import QueryError, run_query
|
||||
from app.schemas.data_file import RequestDataFile, RequestDataTable
|
||||
|
||||
# 列超过这么多时,不指定 columns 只返回列名和类型,避免一次塞进太多概况
|
||||
BRIEF_COLUMNS = 50
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DataContext:
|
||||
files: tuple[RequestDataFile, ...] = ()
|
||||
|
||||
def find(self, name: str) -> tuple[RequestDataFile, RequestDataTable] | None:
|
||||
for file in self.files:
|
||||
for table in file.tables:
|
||||
if table.table == name:
|
||||
return file, table
|
||||
return None
|
||||
|
||||
def paths(self) -> dict[str, Path]:
|
||||
return {
|
||||
table.table: Path(table.path)
|
||||
for file in self.files
|
||||
for table in file.tables
|
||||
}
|
||||
|
||||
|
||||
def _error(runtime: ToolRuntime, message: str) -> ToolMessage:
|
||||
# 返回给模型修正,不中断 agent
|
||||
return ToolMessage(message, tool_call_id=runtime.tool_call_id, status="error")
|
||||
|
||||
|
||||
def _table_list(data: DataContext) -> str:
|
||||
return "、".join(data.paths()) or "无"
|
||||
|
||||
|
||||
@tool
|
||||
def describe_data(
|
||||
table: str,
|
||||
runtime: ToolRuntime[Any],
|
||||
columns: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="只看这些列;不传时看全部列"),
|
||||
] = None,
|
||||
) -> dict[str, Any] | ToolMessage:
|
||||
"""查看一张数据表的列概况(类型、空值率、去重值个数、数值范围和四分位、常见值、样例)、随机样例行,以及解析时做的调整
|
||||
|
||||
分析或取数前先看要用的表;表名见请求 data_files 里的 table。统计值来自全表,样例只是其中几行
|
||||
列很多时只返回列名和类型,再用 columns 指定要看的列
|
||||
"""
|
||||
data: DataContext = runtime.context.data
|
||||
found = data.find(table)
|
||||
if found is None:
|
||||
return _error(runtime, f"没有表 {table},可用的表:{_table_list(data)}")
|
||||
file, item = found
|
||||
profiled: list[dict[str, Any]] = item.profile["columns"]
|
||||
result: dict[str, Any] = {
|
||||
"table": item.table,
|
||||
"label": item.label,
|
||||
"rows": item.rows,
|
||||
}
|
||||
if columns is None and len(profiled) > BRIEF_COLUMNS:
|
||||
result["columns"] = [
|
||||
{key: column[key] for key in ("name", "type", "format") if key in column}
|
||||
for column in profiled
|
||||
]
|
||||
result["note"] = (
|
||||
f"共 {len(profiled)} 列,只列出列名和类型;用 columns 指定列查看详细概况和样例"
|
||||
)
|
||||
else:
|
||||
index = {column["name"]: position for position, column in enumerate(profiled)}
|
||||
names = list(index) if columns is None else list(dict.fromkeys(columns))
|
||||
positions = [index[name] for name in names if name in index]
|
||||
result["columns"] = [profiled[position] for position in positions]
|
||||
# 样例行的值和 columns 一一对应
|
||||
result["sample_rows"] = [
|
||||
[row[position] for position in positions] for row in item.profile["sample"]
|
||||
]
|
||||
if missing := [name for name in names if name not in index]:
|
||||
result["missing_columns"] = missing
|
||||
# 针对这张表的警告,以及针对整个文件的警告(例如跳过的坏行)
|
||||
result["warnings"] = [
|
||||
warning.message
|
||||
for warning in file.warnings
|
||||
if warning.table in (item.table, None)
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
@tool
|
||||
def query_data(sql: str, runtime: ToolRuntime[Any]) -> dict[str, Any] | ToolMessage:
|
||||
"""对上传的数据表执行一条只读 SQL(DuckDB 语法),表名是请求 data_files 里的 table(t1、t2……)
|
||||
|
||||
中文或带符号的列名用双引号括起来;宽表转长表用 UNPIVOT,PIVOT 要用 IN 写明取值
|
||||
最多返回 200 行,truncated 为 true 时结果不完整,改用聚合或 LIMIT
|
||||
回答和图表里的数字以查询结果为准,不要自己心算
|
||||
"""
|
||||
data: DataContext = runtime.context.data
|
||||
paths = data.paths()
|
||||
# 任务执行期间被删除或清理的文件跳过,不影响查询其他表
|
||||
existing = {name: path for name, path in paths.items() if path.exists()}
|
||||
try:
|
||||
return run_query(sql, existing, timeout=settings.data_query_timeout).to_dict()
|
||||
except QueryError as error:
|
||||
message = str(error)
|
||||
if gone := [name for name in paths if name not in existing]:
|
||||
message += f";{'、'.join(gone)} 的文件已删除或过期,不能再查询"
|
||||
return _error(runtime, message)
|
||||
|
||||
|
||||
DATA_TOOLS = [describe_data, query_data]
|
||||
+16
-10
@@ -5,7 +5,7 @@
|
||||
组件目录和物料在物料 MCP,各类任务的做法在 skill;系统提示只放每次都要用、别处放不下的原则
|
||||
"""
|
||||
|
||||
from typing import Literal, Self
|
||||
from typing import Any, Literal, Self
|
||||
|
||||
from langchain.agents import create_agent
|
||||
from langchain.agents.middleware import ModelCallLimitMiddleware, wrap_model_call
|
||||
@@ -14,6 +14,7 @@ from langchain_core.messages import AIMessage, HumanMessage
|
||||
from loguru import logger
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from app.agent.data_tools import DATA_TOOLS
|
||||
from app.agent.skills import skills_prompt
|
||||
from app.agent.tools import (
|
||||
ScreenAgentContext,
|
||||
@@ -98,16 +99,22 @@ async def require_tool_call(request, handler):
|
||||
return response
|
||||
|
||||
|
||||
_agent = None
|
||||
# 是否挂数据工具 → agent
|
||||
_agents: dict[bool, Any] = {}
|
||||
|
||||
|
||||
async def get_screen_agent():
|
||||
"""第一次使用时组装,之后复用;系统提示是常量,不随物料目录变化"""
|
||||
global _agent
|
||||
if _agent is None:
|
||||
_agent = create_agent(
|
||||
async def get_screen_agent(with_data: bool = False):
|
||||
"""第一次使用时组装,之后复用;系统提示是常量,不随物料目录变化
|
||||
|
||||
会话里有数据文件时用挂了 describe_data、query_data 的实例,没有时不让模型看到用不了的工具
|
||||
"""
|
||||
if with_data not in _agents:
|
||||
tools = [get_nodes, apply_actions, *await get_mcp_tools()]
|
||||
if with_data:
|
||||
tools.extend(DATA_TOOLS)
|
||||
_agents[with_data] = create_agent(
|
||||
model=get_llm(),
|
||||
tools=[get_nodes, apply_actions, *await get_mcp_tools()],
|
||||
tools=tools,
|
||||
system_prompt=f"{SYSTEM_PROMPT}\n\n{skills_prompt()}",
|
||||
# 用工具调用给出结论:试过原生结构化输出(ProviderStrategy),歧义时模型更常直接执行而不追问,评测多删了组件
|
||||
response_format=ToolStrategy(AgentReply),
|
||||
@@ -124,5 +131,4 @@ async def get_screen_agent():
|
||||
checkpointer=False,
|
||||
name="screen_agent",
|
||||
)
|
||||
|
||||
return _agent
|
||||
return _agents[with_data]
|
||||
|
||||
+3
-1
@@ -3,7 +3,7 @@
|
||||
它们读写的是本次请求的大屏和 agent 状态,所以放在本地;物料等能力由 MCP 提供
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Annotated, NotRequired
|
||||
|
||||
from langchain.agents import AgentState
|
||||
@@ -13,6 +13,7 @@ from langgraph.types import Command
|
||||
from pydantic import Field
|
||||
|
||||
from app.actions.batch import BatchError, run_batch
|
||||
from app.agent.data_tools import DataContext
|
||||
from app.schemas.actions import Action, AddAction, AgentAction
|
||||
from app.schemas.component_resource import ComponentCatalog
|
||||
from app.schemas.screen import Screen
|
||||
@@ -22,6 +23,7 @@ from app.schemas.screen import Screen
|
||||
class ScreenAgentContext:
|
||||
screen: Screen # 本次请求的原始大屏,只读
|
||||
catalog: ComponentCatalog # 当前物料目录,只用于补全和校验
|
||||
data: DataContext = field(default_factory=DataContext) # 本会话可用的数据表
|
||||
|
||||
|
||||
def _latest(_, new):
|
||||
|
||||
Reference in New Issue
Block a user