feat：记忆&上下文管理

app/processor/history.py

@@ -1,21 +1,89 @@
-"""会话历史：数据库保存完整对话，每次请求按 token 预算只带最近的完整轮次
+"""会话历史：数据库保存完整对话，每次请求按 token 预算带最近的轮次，最近一轮总是带上
 
 裁剪在服务层组装请求时做一次，意图判断、问答和 screen agent 看到的是同一段历史
+每轮是 {user, assistant, outcome}，有轮次记录时另带 changes、queries、clarifications（字段同 Turn）：
+用户原话和回复转成对话消息，记录由 history_records 放进本次请求的 JSON，两者共用一个预算
 """
 
-from langchain_core.messages import (
-    AIMessage,
-    BaseMessage,
-    HumanMessage,
-    trim_messages,
-)
+import json
+from functools import cache
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
 from langchain_core.messages.utils import count_tokens_approximately
+from loguru import logger
 
 from app.schemas.base import JsonObject
+from app.schemas.screen import Screen
+
+RECORD_KEYS = ("changes", "queries", "clarifications")
+# 放进请求的单条 SQL 上限（字符）；更长的不带，原文仍在会话记录里
+RECORD_SQL_MAX_CHARS = 2000
+# 每条消息在请求里的固定开销（角色、分隔），计入历史预算
+MESSAGE_OVERHEAD = 4
+# 最近一轮超过预算被截短时，回复和用户原话各自至少保留的 token
+MIN_KEPT_TOKENS = 200
+
+
+@cache
+def _encoding():
+    """o200k 近似各家模型的 token；编码表第一次使用时由 tiktoken 下载并缓存，拿不到时退回按字符估算"""
+    try:
+        import tiktoken
+
+        return tiktoken.get_encoding("o200k_base")
+    except Exception:  # noqa: BLE001 计数只用来裁剪历史，不能让请求失败
+        logger.exception("tiktoken 编码表加载失败，历史预算改按字符估算，中文会低估")
+        return None
+
+
+def count_tokens(text: str) -> int:
+    encoding = _encoding()
+    if encoding is None:
+        return count_tokens_approximately([HumanMessage(content=text)])
+    return len(encoding.encode(text, disallowed_special=()))
+
+
+def _record(turn: JsonObject, ago: int, present: set[str] | None) -> JsonObject | None:
+    """一轮放进请求的记录，没有的字段不出现；present 是当前大屏的组件 id，None 时按都不在算（只用于计数）
+
+    过长的 SQL 截断了就不能执行，这条查询整条不带，原文仍在会话记录里
+    """
+    record: JsonObject = {key: turn[key] for key in RECORD_KEYS if turn.get(key)}
+    if "queries" in record:
+        record["queries"] = [
+            query
+            for query in record["queries"]
+            if len(query["sql"]) <= RECORD_SQL_MAX_CHARS
+        ] or None
+    if "changes" in record:
+        record["changes"] = [
+            {**change, "on_screen": _on_screen(change, present or set())}
+            for change in record["changes"]
+        ]
+    record = {key: value for key, value in record.items() if value}
+    return {"ago": ago, **record} if record else None
+
+
+def _on_screen(change: JsonObject, present: set[str]) -> bool:
+    if change["type"] == "canvas":
+        return True
+    return change["node_id"] in present
+
+
+def turn_tokens(turn: JsonObject) -> int:
+    """一轮在请求里的大小：两条消息（含每条的固定开销）和放进请求的记录"""
+    tokens = (
+        count_tokens(turn["user"])
+        + count_tokens(turn["assistant"])
+        + MESSAGE_OVERHEAD * 2
+    )
+    if record := _record(turn, ago=1, present=None):
+        tokens += count_tokens(json.dumps(record, ensure_ascii=False))
+    return tokens
 
 
 def history_messages(history: list[JsonObject]) -> list[BaseMessage]:
-    """每轮转成一条用户消息和一条助手消息，按时间从早到晚排列"""
+    """每轮转成一条用户消息和一条助手消息，按时间从早到晚排列；记录不进消息"""
     messages: list[BaseMessage] = []
     for turn in history:
         messages.append(HumanMessage(content=turn["user"]))
@@ -24,16 +92,72 @@ def history_messages(history: list[JsonObject]) -> list[BaseMessage]:
 
 
 def trim_history(history: list[JsonObject], max_tokens: int) -> list[JsonObject]:
-    """从最近一轮往前保留，总量不超过 max_tokens；只保留完整的轮次，不从一段回复中间截断
+    """从最近一轮往前保留，消息和记录合计不超过 max_tokens；更早的轮次放不下就整轮不带
 
-    单独一轮就超过预算时这一轮也不保留
+    最近一轮总是保留：它单独就超过预算时截短再带上（见 _fit），不能因为一条长回答把整段历史丢掉
     """
-    kept = trim_messages(
-        history_messages(history),
-        max_tokens=max_tokens,
-        token_counter=count_tokens_approximately,
-        strategy="last",
-        # 从用户消息开始，助手回复放不下时连同它的用户消息一起去掉
-        start_on="human",
-    )
-    return history[len(history) - len(kept) // 2 :]
+    if not history:
+        return []
+    latest = _fit(history[-1], max_tokens)
+    used = turn_tokens(latest)
+    kept = [latest]
+    for turn in reversed(history[:-1]):
+        used += turn_tokens(turn)
+        if used > max_tokens:
+            break
+        kept.append(turn)
+    return kept[::-1]
+
+
+def _fit(turn: JsonObject, max_tokens: int) -> JsonObject:
+    """一轮超过预算时先截回复、再截用户原话，都是去掉中间、保留开头和结尾（提议和结论通常在回复末尾）
+
+    记录不截：查询最多 10 条、每条 SQL 不超过 RECORD_SQL_MAX_CHARS，默认预算下放得下
+    """
+    for key in ("assistant", "user"):
+        over = turn_tokens(turn) - max_tokens
+        if over <= 0:
+            break
+        keep = max(count_tokens(turn[key]) - over, MIN_KEPT_TOKENS)
+        turn = {**turn, key: truncate_middle(turn[key], keep)}
+    return turn
+
+
+def truncate_middle(text: str, max_tokens: int) -> str:
+    """去掉中间、保留开头和结尾各一半，中间写明省略了多少字；结果不超过 max_tokens"""
+    total = count_tokens(text)
+    if total <= max_tokens:
+        return text
+    keep = len(text) * max_tokens // total
+    while keep > 0:
+        head, tail = text[: keep // 2], text[len(text) - (keep - keep // 2) :]
+        shortened = f"{head}\n…（中间省略 {len(text) - keep} 字）…\n{tail}"
+        if count_tokens(shortened) <= max_tokens:
+            return shortened
+        keep = keep * 9 // 10
+    return f"…（省略 {len(text)} 字）…"
+
+
+def intent_history(history: list[JsonObject]) -> list[JsonObject]:
+    """意图判断看的历史：原话、回复、结果和澄清问答；SQL 和修改记录不给"""
+    projected = []
+    for turn in history:
+        item = {key: turn[key] for key in ("user", "assistant", "outcome")}
+        if turn.get("clarifications"):
+            item["clarifications"] = turn["clarifications"]
+        projected.append(item)
+    return projected
+
+
+def history_records(history: list[JsonObject], screen: Screen) -> list[JsonObject]:
+    """保留下来的各轮的记录，最近的在前；ago 是往前数第几轮，1 是上一轮
+
+    on_screen 按当前大屏现算：只表示这个 id 现在还在大屏上，不代表那次修改已经生效
+    澄清恢复时可能换了大屏，所以在节点里调用，不在组装请求时算好
+    """
+    present = {node.id for node in screen.nodes}
+    records = []
+    for ago, turn in enumerate(reversed(history), start=1):
+        if record := _record(turn, ago, present):
+            records.append(record)
+    return records

app/processor/nodes/input_node.py

@@ -52,5 +52,7 @@ class InputNode(BaseNode):
         updated_state["clarification"] = None
         updated_state["answer"] = None
         updated_state["suggestions"] = None
+        updated_state["changes"] = None
+        updated_state["queries"] = None
         updated_state["status"] = "running"
         return updated_state

app/processor/nodes/intent_node.py

@@ -2,6 +2,7 @@
 
 from app.clients.decision import get_decision_engine
 from app.processor.base import BaseNode
+from app.processor.history import intent_history
 from app.processor.intent_policy import resolve_intent
 from app.processor.state import QueryState
 from app.processor.summary import request_summary
@@ -16,8 +17,11 @@ class IntentNode(BaseNode):
         updated_state: QueryState = state.copy()
         context = state["input_context"]
         request = DecisionRequest(
-            # Jev 只接收 JSON，会话历史作为 history 字段和本次请求放在一起
-            state={"history": context.history, **request_summary(context)},
+            # Jev 只接收 JSON，会话历史作为 history 字段和本次请求放在一起；SQL、修改记录不给，state 越大越不准
+            state={
+                "history": intent_history(context.history),
+                **request_summary(context),
+            },
             decisions={"intent": build_intent_question()},
         )
         response = await get_decision_engine().run(request)

app/processor/nodes/qa_node.py

@@ -6,21 +6,31 @@
 import json
 
 from langchain.agents.middleware.model_call_limit import ModelCallLimitExceededError
-from langchain_core.messages import AIMessageChunk, HumanMessage
+from langchain_core.messages import (
+    AIMessage,
+    AIMessageChunk,
+    BaseMessage,
+    HumanMessage,
+    ToolMessage,
+)
 from langchain_core.output_parsers import StrOutputParser
 from langgraph.config import get_stream_writer
 
 from app.agent.qa_agent import QAContext, get_qa_agent
 from app.processor.base import BaseNode
-from app.processor.history import history_messages
+from app.processor.history import history_messages, history_records
 from app.processor.state import QueryState, run_data
 from app.processor.summary import data_file_summary
+from app.schemas.base import JsonObject
+from app.schemas.conversation import TurnQuery
 
 # 模型在一次回答里分几次输出文字（例如查询前后各说一段）时，段落之间空一行
 PARAGRAPH_BREAK = "\n\n"
 # 带数据的问答回答完后给的后续请求：意图判断会把"展示一下 9 月销售额"这类说法分到问答，
 # 用户其实想在大屏上看到；回答之后给一个按钮，点了按普通请求走大屏 agent，会话历史里有这次回答
 PUT_ON_SCREEN = "把上面的结果放到大屏上"
+# 一轮最多记下的查询；Langfuse 里每条 trace 的 query_data 调用 p50 3 次、p90 7 次（含重试）
+MAX_QUERY_RECORDS = 10
 
 
 class QANode(BaseNode):
@@ -46,11 +56,17 @@ class QANode(BaseNode):
             ]
         agent = get_qa_agent(with_data=bool(context.data_files))
 
+        # 此前各轮的修改、查询和澄清记录跟着本次请求，不拼进历史消息；前端带来的同名字段不保留
+        payload.pop("history_records", None)
+        if records := history_records(context.history, context.screen):
+            payload["history_records"] = records
+
         parser = StrOutputParser()
         parts: list[str] = []
         last_message_id = None
+        messages: list[BaseMessage] = []
         try:
-            async for chunk, metadata in agent.astream(
+            async for mode, data in agent.astream(
                 {
                     "messages": [
                         # 会话历史作为此前的对话消息，本次请求是最后一条用户消息
@@ -59,8 +75,13 @@ class QANode(BaseNode):
                     ]
                 },
                 context=QAContext(data=run_data()),
-                stream_mode="messages",
+                # messages 推送文字，values 留下完整的消息，用来记下执行过的查询
+                stream_mode=["messages", "values"],
             ):
+                if mode == "values":
+                    messages = data["messages"]
+                    continue
+                chunk, metadata = data
                 # 工具结果、工具调用和中间件插入的消息都不推送
                 if metadata.get("langgraph_node") != "model" or not isinstance(
                     chunk, AIMessageChunk
@@ -98,4 +119,41 @@ class QANode(BaseNode):
         updated_state["status"] = "completed"
         if context.data_files:
             updated_state["suggestions"] = [PUT_ON_SCREEN]
+            # 只记成功的问答：步数用尽时回答里也没有数字可用
+            updated_state["queries"] = query_records(messages)
         return updated_state
+
+
+def query_records(messages: list[BaseMessage]) -> list[JsonObject]:
+    """问答里成功执行的 query_data：调用参数和工具结果按 tool_call_id 配对
+
+    只存 SQL 和结果的形状；按执行顺序，相同的 SQL 只留最后一次，最多 MAX_QUERY_RECORDS 条（留最近的）
+    """
+    sql_of = {
+        call["id"]: call["args"].get("sql")
+        for message in messages
+        if isinstance(message, AIMessage)
+        for call in message.tool_calls
+        if call["name"] == "query_data"
+    }
+    found: dict[str, JsonObject] = {}
+    for message in messages:
+        if not isinstance(message, ToolMessage) or message.status == "error":
+            continue
+        sql = sql_of.get(message.tool_call_id)
+        if not isinstance(sql, str):
+            continue
+        try:
+            result = json.loads(str(message.content))
+            record = TurnQuery(
+                sql=sql,
+                columns=result["columns"],
+                rows=result["row_count"],
+                truncated=result["truncated"],
+            ).model_dump(mode="json")
+        except (ValueError, KeyError, TypeError):
+            continue
+        # 重新插入，相同 SQL 的位置按最后一次执行算；按原文比较，字面量里的空白也有意义
+        found.pop(sql, None)
+        found[sql] = record
+    return list(found.values())[-MAX_QUERY_RECORDS:]

app/processor/nodes/screen_agent_node.py

@@ -18,11 +18,15 @@ from app.agent.screen_agent import (
 )
 from app.agent.tools import ScreenAgentContext, last_write_failed
 from app.processor.base import BaseNode
-from app.processor.history import history_messages
+from app.processor.history import history_messages, history_records
 from app.processor.state import QueryState, run_data
 from app.processor.summary import request_summary
 from app.resources.component import get_catalog_cache
+from app.resources.material_fields import identifying_texts
+from app.schemas.base import JsonObject
 from app.schemas.component_resource import ComponentCatalog
+from app.schemas.conversation import TurnChange
+from app.schemas.screen import Screen, ScreenNode
 
 # 评测按这句话识别"草稿因最后一次提交失败没有交付"，单独核对草稿本身是否满足要求
 DRAFT_REJECTED = "修改没有通过校验，这次没有生效，可以换个说法再试"
@@ -57,6 +61,44 @@ def reply_update(
     return {"status": "completed", "actions": actions, "answer": reply.message}
 
 
+def screen_changes(
+    before: Screen, after: Screen, catalog: ComponentCatalog | None
+) -> list[JsonObject]:
+    """原始大屏和交付后的大屏按组件做净差，同一组件的多个动作合并成一条
+
+    不重放动作：新增后又删掉的组件不出现，布局展开产生的大量移动、缩放合并成一条 modified；
+    交付后的大屏就是草稿，交付记录在原始大屏上重放得到的也是它
+    """
+
+    def change(node: ScreenNode, kind: str) -> JsonObject:
+        texts = identifying_texts(catalog, node.model_dump(mode="json"))
+        return TurnChange(
+            node_id=node.id,
+            type=node.type,
+            title=texts[0] if texts else node.name,
+            change=kind,
+        ).model_dump(mode="json")
+
+    old = {node.id: node for node in before.nodes}
+    new = {node.id: node for node in after.nodes}
+    changes: list[JsonObject] = []
+    if before.canvas.model_dump(mode="json") != after.canvas.model_dump(mode="json"):
+        changes.append(
+            TurnChange(node_id="canvas", type="canvas", change="modified").model_dump(
+                mode="json"
+            )
+        )
+    for node_id, node in new.items():
+        if node_id not in old:
+            changes.append(change(node, "added"))
+        elif node.model_dump(mode="json") != old[node_id].model_dump(mode="json"):
+            changes.append(change(node, "modified"))
+    changes.extend(
+        change(node, "deleted") for node_id, node in old.items() if node_id not in new
+    )
+    return changes
+
+
 class ScreenAgentNode(BaseNode):
     name = "screen_agent_node"
 
@@ -70,14 +112,16 @@ class ScreenAgentNode(BaseNode):
 
         context = state["input_context"]
         agent = await get_screen_agent(with_data=bool(context.data_files))
+        request = request_summary(context)
+        # 此前各轮的修改、查询和澄清记录跟着本次请求，不拼进历史消息
+        if records := history_records(context.history, context.screen):
+            request["history_records"] = records
         result = await agent.ainvoke(
             {
                 # 会话历史作为此前的对话消息，本次请求是最后一条用户消息
                 "messages": [
                     *history_messages(context.history),
-                    HumanMessage(
-                        content=json.dumps(request_summary(context), ensure_ascii=False)
-                    ),
+                    HumanMessage(content=json.dumps(request, ensure_ascii=False)),
                 ]
             },
             context=ScreenAgentContext(
@@ -97,5 +141,11 @@ class ScreenAgentNode(BaseNode):
             )
             reply = await finalize_reply(result, limit_hit=limit_hit)
             result = {**result, "structured_response": reply}
-        updated_state.update(reply_update(result, limit_hit=limit_hit, catalog=catalog))
+        update = reply_update(result, limit_hit=limit_hit, catalog=catalog)
+        if update.get("actions"):
+            # reply_update 拿不到原始大屏，修改记录在这里按原始大屏和草稿算
+            update["changes"] = screen_changes(
+                context.screen, result["working_screen"], catalog
+            )
+        updated_state.update(update)
         return updated_state

app/processor/state.py

@@ -29,6 +29,9 @@ class QueryState(TypedDict):
     ]  # 待合并的用户回答或取消指令
     answer: NotRequired[str | None]  # 问答回答，或 agent 给用户的说明
     suggestions: NotRequired[list[str] | None]  # 可以一键发送的后续请求，前端显示成按钮
+    # 轮次记录，字段同 TurnChange、TurnQuery；用普通 dict，检查点不用为它们扩类型白名单
+    changes: NotRequired[list[JsonObject] | None]  # 交付的修改，按组件合并
+    queries: NotRequired[list[JsonObject] | None]  # 问答里成功执行的查询
     status: NotRequired[
         Literal["running", "waiting_for_input", "completed", "cancelled", "failed"]
     ]  # 后端处理、等待补充、处理完成、用户取消或失败，完成不代表前端已执行