feat：增加分析进度接口

2026-10-10 07:26:24 -07:00
parent e3699d1a30
commit cd5a2b4d8f
15 changed files with 632 additions and 143 deletions
@@ -0,0 +1,154 @@
+"""上传解析任务的进度：排队、解析比例、预计剩余时间
+
+任务只存在进程内存里：单实例部署，重启后进行中的任务随进程丢失，原文件目录由过期清理删除
+所有方法都在事件循环里调用；解析子进程的进度由上传流程用 call_soon_threadsafe 转进来
+"""
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from itertools import count
+
+from app.schemas.data_file import UploadProgress, UploadStage, UploadStatus
+
+# 失败的任务保留这么久，刷新页面后还能看到失败原因；成功的任务不保留，文件已经有记录
+FAILED_RETENTION = 600
+# 解析中的进度不到 1，1 只留给已完成
+MAX_PARSING_PROGRESS = 0.99
+# 进度太小或解析刚开始时估不准，不给预计时间
+ETA_MIN_PROGRESS = 0.05
+ETA_MIN_ELAPSED = 1.0
+# 预计时间的平滑系数：新的估算占多少，其余沿用上一次，免得数字来回跳
+ETA_SMOOTHING = 0.3
+
+
+@dataclass
+class UploadJob:
+    file_id: str
+    owner_id: int
+    conversation_id: str
+    name: str
+    seq: int
+    status: UploadStatus = "queued"
+    stage: UploadStage | None = None
+    progress: float = 0.0
+    error: tuple[int, str] | None = None
+    # 以下时间都来自 UploadBoard 的时钟
+    started_at: float | None = None
+    failed_at: float | None = None
+    eta: float | None = None
+    eta_at: float = 0.0
+
+    @property
+    def active(self) -> bool:
+        return self.status in ("queued", "parsing")
+
+
+class UploadBoard:
+    def __init__(self, clock: Callable[[], float] = time.monotonic):
+        self._clock = clock
+        self._jobs: dict[str, UploadJob] = {}
+        self._numbers = count()
+
+    def create(
+        self, file_id: str, owner_id: int, conversation_id: str, name: str
+    ) -> UploadJob:
+        job = UploadJob(file_id, owner_id, conversation_id, name, next(self._numbers))
+        self._jobs[file_id] = job
+        return job
+
+    def active_count(self, conversation_id: str) -> int:
+        """会话里排队和解析中的任务数，它们都占着会话内的文件数名额"""
+        return sum(
+            job.active and job.conversation_id == conversation_id
+            for job in self._jobs.values()
+        )
+
+    def start(self, job: UploadJob) -> None:
+        job.status = "parsing"
+        job.started_at = self._clock()
+
+    def report(self, job: UploadJob, stage: UploadStage, value: float) -> None:
+        """解析子进程报告进度：只增不减，预计时间按已用时间和进度外推后平滑"""
+        if job.status != "parsing" or job.started_at is None:
+            return
+        now = self._clock()
+        job.stage = stage
+        job.progress = min(max(value, job.progress), MAX_PARSING_PROGRESS)
+        elapsed = now - job.started_at
+        if job.progress < ETA_MIN_PROGRESS or elapsed < ETA_MIN_ELAPSED:
+            return
+        raw = elapsed * (1 - job.progress) / job.progress
+        previous = self._eta(job, now)
+        job.eta = (
+            raw
+            if previous is None
+            else ETA_SMOOTHING * raw + (1 - ETA_SMOOTHING) * previous
+        )
+        job.eta_at = now
+
+    def fail(self, job: UploadJob, code: int, message: str) -> None:
+        job.status = "failed"
+        job.stage = None
+        job.error = (code, message)
+        job.failed_at = self._clock()
+
+    def remove(self, job: UploadJob) -> None:
+        self._jobs.pop(job.file_id, None)
+
+    def get(self, file_id: str) -> UploadJob | None:
+        self._purge()
+        return self._jobs.get(file_id)
+
+    def for_conversation(self, owner_id: int, conversation_id: str) -> list[UploadJob]:
+        self._purge()
+        return [
+            job
+            for job in self._jobs.values()
+            if job.owner_id == owner_id and job.conversation_id == conversation_id
+        ]
+
+    def view(self, job: UploadJob) -> UploadProgress:
+        return UploadProgress(
+            file_id=job.file_id,
+            name=job.name,
+            status=job.status,
+            progress=round(job.progress, 3),
+            stage=job.stage,
+            eta_seconds=self._eta_seconds(job),
+            queued_ahead=self._queued_ahead(job),
+            error=(
+                None
+                if job.error is None
+                else {"code": job.error[0], "msg": job.error[1]}
+            ),
+        )
+
+    def _eta(self, job: UploadJob, now: float) -> float | None:
+        """上次估算之后又过了一段时间，剩余时间相应减少"""
+        if job.eta is None:
+            return None
+        return max(job.eta - (now - job.eta_at), 0.0)
+
+    def _eta_seconds(self, job: UploadJob) -> float | None:
+        if job.status != "parsing":
+            return None
+        eta = self._eta(job, self._clock())
+        return None if eta is None else round(eta, 1)
+
+    def _queued_ahead(self, job: UploadJob) -> int:
+        if job.status != "queued":
+            return 0
+        return sum(
+            other.status == "queued" and other.seq < job.seq
+            for other in self._jobs.values()
+        )
+
+    def _purge(self) -> None:
+        now = self._clock()
+        for job in [
+            job
+            for job in self._jobs.values()
+            if job.failed_at is not None and now - job.failed_at > FAILED_RETENTION
+        ]:
+            self.remove(job)