feat：新增大屏存取发布接口
This commit is contained in:
@@ -0,0 +1,562 @@
|
||||
"""大屏的业务逻辑：存取、乐观锁、发布、历史、复制、封面、线上读取和数据源取数
|
||||
|
||||
存储层只提供"匹配并更新"这类原语，404 / 409 的判断都在这里
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import secrets
|
||||
import weakref
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.exceptions import BizException
|
||||
from app.schemas.saved_screen import (
|
||||
NAME_MAX_LENGTH,
|
||||
ListStatus,
|
||||
PublishEntry,
|
||||
PublishInfo,
|
||||
PublishSnapshot,
|
||||
ScreenRecord,
|
||||
ScreenSummary,
|
||||
SortField,
|
||||
)
|
||||
from app.schemas.screen import Screen
|
||||
from app.services.datasource_proxy import FetchCache, fetch_json
|
||||
from app.services.screen_store import ScreenStore
|
||||
from app.services.thumbnails import ThumbnailStorage
|
||||
|
||||
# 事件代码和数据源转换函数是字符串形式的 JS，线上会在浏览器里执行；写入权限只给 owner，再限制长度
|
||||
SCRIPT_MAX_CHARS = 64 * 1024
|
||||
COPY_SUFFIX = " 副本"
|
||||
EMPTY_PAGE = {
|
||||
"canvas": {"width": 1920, "height": 1080, "backgroundColor": "#ffffff10"},
|
||||
"nodes": [],
|
||||
"dataSource": [],
|
||||
}
|
||||
|
||||
# 回收站过期清理：一批最多清多少个，两次扫描的间隔（秒）
|
||||
PURGE_BATCH = 100
|
||||
TRASH_SWEEP_INTERVAL = 3600
|
||||
|
||||
STALE = "大屏已在别处修改"
|
||||
# 下线 / 删除 / 换封面遇到并发修改时的重读次数
|
||||
UPDATE_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class FilePinner(Protocol):
|
||||
"""数据文件服务里和大屏有关的两个方法"""
|
||||
|
||||
async def sync_screen_files(
|
||||
self, user_id: int, screen_id: str, file_ids: set[str]
|
||||
) -> None: ...
|
||||
|
||||
async def release_screen_files(self, screen_id: str) -> None: ...
|
||||
|
||||
|
||||
def referenced_file_ids(page: dict[str, Any]) -> set[str]:
|
||||
"""页面里图表绑定的数据文件：nodes[].fileQuery.file_id"""
|
||||
ids = set()
|
||||
nodes = page.get("nodes")
|
||||
for node in nodes if isinstance(nodes, list) else []:
|
||||
query = node.get("fileQuery") if isinstance(node, dict) else None
|
||||
file_id = query.get("file_id") if isinstance(query, dict) else None
|
||||
if isinstance(file_id, str) and file_id:
|
||||
ids.add(file_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _publish_is(publish: PublishInfo | None) -> dict[str, Any]:
|
||||
""" "线上指针还是我读到的那个"的更新条件"""
|
||||
if publish is None:
|
||||
return {"publish": None}
|
||||
return {"publish.version": publish.version, "publish.status": publish.status}
|
||||
|
||||
|
||||
def _not_found() -> BizException:
|
||||
# 别人的大屏与不存在同样返回 404
|
||||
return BizException(404, "大屏不存在或已删除")
|
||||
|
||||
|
||||
def validate_page(page: dict[str, Any]) -> dict[str, Any]:
|
||||
"""按 Screen 校验结构，但返回的仍是原始 JSON：model_dump 会给没传的字段补出 null，前端读回来就变了"""
|
||||
try:
|
||||
Screen.model_validate(page)
|
||||
except ValidationError as error:
|
||||
first = error.errors()[0]
|
||||
where = ".".join(str(part) for part in first["loc"])
|
||||
raise BizException(422, f"大屏数据不合法：{where}：{first['msg']}") from None
|
||||
_check_script_length(page)
|
||||
return page
|
||||
|
||||
|
||||
def _check_script_length(page: dict[str, Any]) -> None:
|
||||
scripts = [
|
||||
event.get("code")
|
||||
for node in page["nodes"]
|
||||
for event in (node.get("events") or [])
|
||||
if isinstance(event, dict)
|
||||
] + [
|
||||
source.get("transform")
|
||||
for source in page.get("dataSource", [])
|
||||
if isinstance(source, dict)
|
||||
]
|
||||
if any(isinstance(text, str) and len(text) > SCRIPT_MAX_CHARS for text in scripts):
|
||||
raise BizException(
|
||||
422, f"事件代码和数据转换函数不能超过 {SCRIPT_MAX_CHARS // 1024} KB"
|
||||
)
|
||||
|
||||
|
||||
def public_page(page: dict[str, Any]) -> dict[str, Any]:
|
||||
"""线上页面里去掉数据源的 headers：它们常放 API key，线上取数由服务端按快照里的完整配置发起，浏览器用不着"""
|
||||
sources = page.get("dataSource")
|
||||
if not isinstance(sources, list):
|
||||
return {**page}
|
||||
return {
|
||||
**page,
|
||||
"dataSource": [
|
||||
{key: value for key, value in item.items() if key != "headers"}
|
||||
if isinstance(item, dict)
|
||||
else item
|
||||
for item in sources
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class ScreenService:
|
||||
def __init__(
|
||||
self,
|
||||
store: ScreenStore,
|
||||
thumbnails: ThumbnailStorage,
|
||||
files: FilePinner | None = None,
|
||||
trash_days: int = 30,
|
||||
):
|
||||
self._store = store
|
||||
self._thumbnails = thumbnails
|
||||
# 已保存的大屏引用的数据文件不能过期，否则"刷新数据"会 404；没有文件服务时不管
|
||||
self._files = files
|
||||
if trash_days < 1:
|
||||
raise ValueError("trash_days 必须是正整数")
|
||||
self.trash_days = trash_days
|
||||
# 按大屏的锁：保存 / 回滚 / 删除 / 还原都是"改库再同步固定的文件"两步，
|
||||
# 两个请求交错会让固定的文件跟着先开始、后结束的那个请求的旧页面走。单实例部署，进程内锁就够
|
||||
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
|
||||
weakref.WeakValueDictionary()
|
||||
)
|
||||
self._datasource_cache = FetchCache()
|
||||
|
||||
def _lock(self, screen_id: str) -> asyncio.Lock:
|
||||
lock = self._locks.get(screen_id)
|
||||
if lock is None:
|
||||
lock = self._locks[screen_id] = asyncio.Lock()
|
||||
return lock
|
||||
|
||||
async def _pin_files(self, owner_id: int, screen_id: str, page: dict[str, Any]):
|
||||
"""让数据文件跟着大屏的引用走。主操作已经成功，这一步失败只记日志，不让保存报错"""
|
||||
if self._files is None:
|
||||
return
|
||||
try:
|
||||
await self._files.sync_screen_files(
|
||||
owner_id, screen_id, referenced_file_ids(page)
|
||||
)
|
||||
except Exception as error:
|
||||
logger.opt(exception=error).error(
|
||||
"同步大屏 {} 引用的数据文件失败", screen_id
|
||||
)
|
||||
|
||||
async def _release_files(self, screen_id: str) -> None:
|
||||
if self._files is None:
|
||||
return
|
||||
try:
|
||||
await self._files.release_screen_files(screen_id)
|
||||
except Exception as error:
|
||||
logger.opt(exception=error).error(
|
||||
"放开大屏 {} 固定的数据文件失败", screen_id
|
||||
)
|
||||
|
||||
# ---- 草稿 ----
|
||||
|
||||
async def create(
|
||||
self, owner_id: int, name: str, page: dict[str, Any] | None
|
||||
) -> ScreenRecord:
|
||||
now = _now()
|
||||
record = ScreenRecord(
|
||||
screen_id=f"s_{uuid4().hex}",
|
||||
owner_id=owner_id,
|
||||
name=name,
|
||||
page=validate_page(copy.deepcopy(EMPTY_PAGE) if page is None else page),
|
||||
revision=1,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
await self._store.insert(record)
|
||||
await self._pin_files(owner_id, record.screen_id, record.page)
|
||||
return record
|
||||
|
||||
async def get(self, screen_id: str, owner_id: int) -> ScreenRecord:
|
||||
record = await self._store.get(screen_id, owner_id)
|
||||
if record is None:
|
||||
raise _not_found()
|
||||
return record
|
||||
|
||||
async def save(
|
||||
self, screen_id: str, owner_id: int, page: dict[str, Any], revision: int
|
||||
) -> ScreenSummary:
|
||||
validate_page(page)
|
||||
async with self._lock(screen_id):
|
||||
saved = await self._update_with_revision(
|
||||
screen_id,
|
||||
owner_id,
|
||||
{"page": page, "revision": revision + 1, "updated_at": _now()},
|
||||
revision,
|
||||
)
|
||||
await self._pin_files(owner_id, screen_id, page)
|
||||
return saved
|
||||
|
||||
async def _update_with_revision(
|
||||
self, screen_id: str, owner_id: int, changes: dict[str, Any], revision: int
|
||||
) -> ScreenSummary:
|
||||
updated = await self._store.update(
|
||||
screen_id, owner_id, changes, revision=revision
|
||||
)
|
||||
if updated is not None:
|
||||
return updated
|
||||
# 没更新成功：大屏不存在是 404，存在但 revision 不一致是 409
|
||||
if await self._store.get_summary(screen_id, owner_id) is None:
|
||||
raise _not_found()
|
||||
raise BizException(409, STALE)
|
||||
|
||||
async def rename(self, screen_id: str, owner_id: int, name: str) -> None:
|
||||
# 改名不算改内容，revision 不动
|
||||
changes = {"name": name, "updated_at": _now()}
|
||||
if await self._store.update(screen_id, owner_id, changes) is None:
|
||||
raise _not_found()
|
||||
|
||||
async def _update_from_summary(
|
||||
self,
|
||||
screen_id: str,
|
||||
owner_id: int,
|
||||
build: Callable[[ScreenSummary], tuple[dict[str, Any], dict[str, Any]] | None],
|
||||
) -> None:
|
||||
"""读出当前状态，据此算出修改，再带着"状态没变"的条件写回；并发改了就重读再来
|
||||
|
||||
下线、删除要整段改写 publish，不带条件就会把并发发布写进去的新指针盖回旧值
|
||||
build 返回 (changes, where)；返回 None 表示已经是目标状态，什么都不用做
|
||||
"""
|
||||
for _ in range(UPDATE_ATTEMPTS):
|
||||
summary = await self._store.get_summary(screen_id, owner_id)
|
||||
if summary is None:
|
||||
raise _not_found()
|
||||
built = build(summary)
|
||||
if built is None:
|
||||
return
|
||||
changes, where = built
|
||||
if await self._store.update(screen_id, owner_id, changes, where=where):
|
||||
return
|
||||
raise BizException(409, "操作冲突，请重试")
|
||||
|
||||
async def delete(self, screen_id: str, owner_id: int) -> None:
|
||||
def build(summary: ScreenSummary):
|
||||
changes: dict[str, Any] = {"deleted_at": _now()}
|
||||
if summary.publish is None:
|
||||
return changes, {"publish": None}
|
||||
# 线上版本同时下线，分享链接立即 404
|
||||
changes["publish"] = summary.publish.model_copy(
|
||||
update={"status": "offline"}
|
||||
)
|
||||
return changes, _publish_is(summary.publish)
|
||||
|
||||
async with self._lock(screen_id):
|
||||
await self._update_from_summary(screen_id, owner_id, build)
|
||||
# 进了回收站的大屏不再固定文件，还原时再固定回来
|
||||
await self._release_files(screen_id)
|
||||
|
||||
async def restore_deleted(self, screen_id: str, owner_id: int) -> None:
|
||||
async with self._lock(screen_id):
|
||||
record = await self._store.get(screen_id, owner_id, deleted=True)
|
||||
if record is None:
|
||||
raise _not_found()
|
||||
changes = {"deleted_at": None}
|
||||
if (
|
||||
await self._store.update(screen_id, owner_id, changes, deleted=True)
|
||||
is None
|
||||
):
|
||||
raise _not_found()
|
||||
await self._pin_files(owner_id, screen_id, record.page)
|
||||
|
||||
# ---- 回收站 ----
|
||||
|
||||
async def _purge(self, summary: ScreenSummary) -> bool:
|
||||
"""彻底删除回收站里的一个大屏
|
||||
|
||||
先删大屏本身（带"仍在回收站"的条件，清理期间被还原的不会误删），再清理附属数据
|
||||
"""
|
||||
if not await self._store.purge(summary.screen_id, summary.owner_id):
|
||||
return False
|
||||
await self._store.delete_publishes(summary.screen_id)
|
||||
self._thumbnails.delete(summary.thumbnail)
|
||||
await self._release_files(summary.screen_id)
|
||||
return True
|
||||
|
||||
async def empty_trash(self, owner_id: int) -> int:
|
||||
"""清空自己的回收站，返回彻底删除的个数"""
|
||||
count = 0
|
||||
previous: set[str] = set()
|
||||
while True:
|
||||
batch, _ = await self._store.search(
|
||||
owner_id,
|
||||
keyword=None,
|
||||
status=None,
|
||||
sort="updated_at",
|
||||
descending=True,
|
||||
limit=PURGE_BATCH,
|
||||
offset=0,
|
||||
deleted=True,
|
||||
)
|
||||
ids = {summary.screen_id for summary in batch}
|
||||
purged = [await self._purge(summary) for summary in batch]
|
||||
count += sum(purged)
|
||||
# 空批才算清完；整批都没删成（被并发还原了）就再查一次，
|
||||
# 但同一批连续两次都删不掉说明有东西卡住，别空转
|
||||
if not batch or (not any(purged) and ids == previous):
|
||||
return count
|
||||
previous = ids
|
||||
|
||||
async def purge_expired_trash(self) -> int:
|
||||
"""彻底删除在回收站里超过保留天数的大屏（所有人的）"""
|
||||
deadline = _now() - timedelta(days=self.trash_days)
|
||||
count = 0
|
||||
previous: set[str] = set()
|
||||
while True:
|
||||
batch = await self._store.list_expired_trash(deadline, PURGE_BATCH)
|
||||
ids = {summary.screen_id for summary in batch}
|
||||
purged = [await self._purge(summary) for summary in batch]
|
||||
count += sum(purged)
|
||||
if not batch or (not any(purged) and ids == previous):
|
||||
return count
|
||||
previous = ids
|
||||
|
||||
async def run_trash_sweeper(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
if removed := await self.purge_expired_trash():
|
||||
logger.info(
|
||||
"已彻底删除 {} 个超过 {} 天的回收站大屏",
|
||||
removed,
|
||||
self.trash_days,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.opt(exception=error).error("清理回收站失败")
|
||||
await asyncio.sleep(TRASH_SWEEP_INTERVAL)
|
||||
|
||||
# ---- 管理页 ----
|
||||
|
||||
async def list_screens(
|
||||
self,
|
||||
owner_id: int,
|
||||
*,
|
||||
keyword: str | None,
|
||||
status: ListStatus | None,
|
||||
sort: SortField,
|
||||
descending: bool,
|
||||
limit: int,
|
||||
offset: int,
|
||||
deleted: bool,
|
||||
) -> tuple[list[ScreenSummary], int]:
|
||||
return await self._store.search(
|
||||
owner_id,
|
||||
keyword=keyword,
|
||||
status=status,
|
||||
sort=sort,
|
||||
descending=descending,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
deleted=deleted,
|
||||
)
|
||||
|
||||
async def copy_screen(self, screen_id: str, owner_id: int, name: str | None) -> str:
|
||||
"""只复制草稿和封面，不复制发布状态和历史"""
|
||||
source = await self.get(screen_id, owner_id)
|
||||
base = source.name[: NAME_MAX_LENGTH - len(COPY_SUFFIX)]
|
||||
copied = await self.create(
|
||||
owner_id,
|
||||
name or f"{base}{COPY_SUFFIX}",
|
||||
copy.deepcopy(source.page),
|
||||
)
|
||||
if source.thumbnail is not None:
|
||||
thumbnail = self._thumbnails.copy(source.thumbnail)
|
||||
if thumbnail is not None:
|
||||
try:
|
||||
attached = await self._store.update(
|
||||
copied.screen_id, owner_id, {"thumbnail": thumbnail}
|
||||
)
|
||||
except BaseException:
|
||||
self._thumbnails.delete(thumbnail)
|
||||
raise
|
||||
if not attached:
|
||||
self._thumbnails.delete(thumbnail)
|
||||
return copied.screen_id
|
||||
|
||||
async def set_thumbnail(self, screen_id: str, owner_id: int, data: bytes) -> str:
|
||||
thumbnail = self._thumbnails.save(data)
|
||||
replaced: list[str | None] = []
|
||||
|
||||
def build(summary: ScreenSummary):
|
||||
# 封面只是展示用的，不算改内容：不动 revision 和 updated_at
|
||||
replaced[:] = [summary.thumbnail]
|
||||
return {"thumbnail": thumbnail}, {"thumbnail": summary.thumbnail}
|
||||
|
||||
try:
|
||||
await self._update_from_summary(screen_id, owner_id, build)
|
||||
except BaseException:
|
||||
self._thumbnails.delete(thumbnail)
|
||||
raise
|
||||
# 删的是这次真正换掉的那张；并发上传时各自换掉各自读到的，不会留下孤儿文件
|
||||
self._thumbnails.delete(replaced[0])
|
||||
return thumbnail
|
||||
|
||||
def open_thumbnail(self, thumbnail_id: str):
|
||||
return self._thumbnails.open(thumbnail_id)
|
||||
|
||||
# ---- 发布 ----
|
||||
|
||||
async def publish(
|
||||
self, screen_id: str, owner_id: int, revision: int
|
||||
) -> PublishInfo:
|
||||
record = await self.get(screen_id, owner_id)
|
||||
if record.revision != revision:
|
||||
raise BizException(409, "大屏有未保存或已在别处修改的内容，请先保存")
|
||||
previous = record.publish
|
||||
snapshot = PublishSnapshot(
|
||||
screen_id=screen_id,
|
||||
# 版本号取已有快照的最大值加一：写快照后没来得及更新指针留下的孤儿快照也不会撞号
|
||||
version=await self._store.last_publish_version(screen_id) + 1,
|
||||
revision=record.revision,
|
||||
page=record.page,
|
||||
published_at=_now(),
|
||||
published_by=owner_id,
|
||||
)
|
||||
if not await self._store.insert_publish(snapshot):
|
||||
raise BizException(409, "发布冲突，请重试")
|
||||
info = PublishInfo(
|
||||
status="published",
|
||||
version=snapshot.version,
|
||||
revision=snapshot.revision,
|
||||
# 第一次发布生成 share_id，之后沿用；下线再发布也沿用
|
||||
share_id=previous.share_id
|
||||
if previous
|
||||
else f"p_{secrets.token_urlsafe(16)}",
|
||||
published_at=snapshot.published_at,
|
||||
)
|
||||
# 条件：草稿没被再保存过（revision），线上指针还是读到的那个——并发发布只有一个能成，
|
||||
# 否则第一次发布时两边各生成一个 share_id，先返回给前端的那个会失效
|
||||
try:
|
||||
updated = await self._store.update(
|
||||
screen_id,
|
||||
owner_id,
|
||||
{"publish": info},
|
||||
revision=revision,
|
||||
where=_publish_is(previous),
|
||||
)
|
||||
except BaseException:
|
||||
await self._store.delete_publish(screen_id, snapshot.version)
|
||||
raise
|
||||
if updated is None:
|
||||
await self._store.delete_publish(screen_id, snapshot.version)
|
||||
raise BizException(409, STALE)
|
||||
return info
|
||||
|
||||
async def unpublish(self, screen_id: str, owner_id: int) -> None:
|
||||
def build(summary: ScreenSummary):
|
||||
if summary.publish is None or summary.publish.status == "offline":
|
||||
return None
|
||||
offline = summary.publish.model_copy(update={"status": "offline"})
|
||||
return {"publish": offline}, _publish_is(summary.publish)
|
||||
|
||||
await self._update_from_summary(screen_id, owner_id, build)
|
||||
|
||||
async def exists(self, screen_id: str, owner_id: int) -> bool:
|
||||
return await self._store.get_summary(screen_id, owner_id) is not None
|
||||
|
||||
async def list_publishes(
|
||||
self, screen_id: str, owner_id: int, limit: int, offset: int
|
||||
) -> tuple[list[PublishEntry], int]:
|
||||
if await self._store.get_summary(screen_id, owner_id) is None:
|
||||
raise _not_found()
|
||||
return await self._store.list_publishes(screen_id, limit, offset)
|
||||
|
||||
async def restore_publish(self, screen_id: str, owner_id: int, version: int) -> int:
|
||||
"""把历史版本的页面覆盖回草稿，不自动发布"""
|
||||
async with self._lock(screen_id):
|
||||
record = await self.get(screen_id, owner_id)
|
||||
snapshot = await self._store.get_publish(screen_id, version)
|
||||
if snapshot is None:
|
||||
raise BizException(404, "发布版本不存在")
|
||||
updated = await self._update_with_revision(
|
||||
screen_id,
|
||||
owner_id,
|
||||
{
|
||||
"page": snapshot.page,
|
||||
"revision": record.revision + 1,
|
||||
"updated_at": _now(),
|
||||
},
|
||||
record.revision,
|
||||
)
|
||||
await self._pin_files(owner_id, screen_id, snapshot.page)
|
||||
return updated.revision
|
||||
|
||||
# ---- 线上（匿名） ----
|
||||
|
||||
async def _live(self, share_id: str) -> tuple[ScreenSummary, PublishSnapshot]:
|
||||
"""线上版本：已下线 / 已删除 / 不存在都是 404；返回的是发布快照，不是草稿"""
|
||||
summary = await self._store.get_by_share_id(share_id)
|
||||
if (
|
||||
summary is None
|
||||
or summary.publish is None
|
||||
or summary.publish.status != "published"
|
||||
):
|
||||
raise _not_found()
|
||||
snapshot = await self._store.get_publish(
|
||||
summary.screen_id, summary.publish.version
|
||||
)
|
||||
if snapshot is None:
|
||||
raise _not_found()
|
||||
return summary, snapshot
|
||||
|
||||
async def get_live(self, share_id: str) -> tuple[ScreenSummary, PublishSnapshot]:
|
||||
return await self._live(share_id)
|
||||
|
||||
async def fetch_live_datasource(self, share_id: str, datasource_id: str):
|
||||
"""前端只传数据源 ID，请求地址取自已发布快照；不能做成"前端传 URL 后端转发"，否则是匿名开放代理"""
|
||||
_, snapshot = await self._live(share_id)
|
||||
source = next(
|
||||
(
|
||||
item
|
||||
for item in snapshot.page.get("dataSource", [])
|
||||
if isinstance(item, dict) and item.get("id") == datasource_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if source is None or source.get("type") != "api" or not source.get("url"):
|
||||
raise BizException(404, "数据源不存在")
|
||||
method = "POST" if source.get("method") == "POST" else "GET"
|
||||
params = source.get("params")
|
||||
headers = source.get("headers")
|
||||
return await self._datasource_cache.get(
|
||||
(share_id, snapshot.version, datasource_id),
|
||||
lambda: fetch_json(
|
||||
str(source["url"]),
|
||||
method,
|
||||
params if isinstance(params, dict) else None,
|
||||
headers if isinstance(headers, dict) else None,
|
||||
),
|
||||
)
|
||||
Reference in New Issue
Block a user