feat: 完善日志
This commit is contained in:
@@ -1,18 +1,15 @@
|
||||
# 定义业务异常与全局异常处理
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from loguru import logger
|
||||
from redis.exceptions import RedisError
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from app.common.response import error_content
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BizException(Exception):
|
||||
def __init__(self, code: int, msg: str = "请求失败") -> None:
|
||||
@@ -46,13 +43,13 @@ async def handle_dependency_exception(
|
||||
exc: SQLAlchemyError | RedisError,
|
||||
) -> JSONResponse:
|
||||
"""将数据库和 Redis 故障转换为服务器异常响应"""
|
||||
logger.error("外部依赖访问失败 error_type=%s", type(exc).__name__)
|
||||
logger.error("外部依赖访问失败 error_type={}", type(exc).__name__)
|
||||
return JSONResponse(status_code=500, content=error_content(500))
|
||||
|
||||
|
||||
async def handle_unexpected_exception(_request: Request, exc: Exception) -> JSONResponse:
|
||||
"""将未捕获异常转换为服务器异常响应"""
|
||||
logger.error("未处理的服务器异常 error_type=%s", type(exc).__name__)
|
||||
logger.error("未处理的服务器异常 error_type={}", type(exc).__name__)
|
||||
return JSONResponse(status_code=500, content=error_content(500))
|
||||
|
||||
|
||||
|
||||
+82
-45
@@ -1,14 +1,22 @@
|
||||
# 配置日志与敏感信息脱敏
|
||||
# 基于 Loguru 统一配置日志输出与敏感信息脱敏
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from logging.handlers import QueueHandler, QueueListener
|
||||
from queue import SimpleQueue
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from anyio import to_thread
|
||||
from loguru import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from loguru import Record
|
||||
|
||||
LOG_PATH = Path(__file__).resolve().parents[2] / "logs" / "app.log"
|
||||
LOG_FORMAT = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{line} | {message}"
|
||||
SENSITIVE_REPLACEMENTS = (
|
||||
(
|
||||
re.compile(r"(?i)(authorization)(['\"\s:=]+)(?:Bearer|Basic)[ \t]+[^\s,'\"}\]]+"),
|
||||
@@ -22,52 +30,81 @@ SENSITIVE_REPLACEMENTS = (
|
||||
)
|
||||
|
||||
|
||||
class SensitiveDataFilter(logging.Filter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
"""清理日志消息中的密码、Token 和连接凭据"""
|
||||
message = record.getMessage()
|
||||
for pattern, replacement in SENSITIVE_REPLACEMENTS:
|
||||
message = pattern.sub(replacement, message)
|
||||
record.msg = message
|
||||
record.args = ()
|
||||
return True
|
||||
def redact(text: str) -> str:
|
||||
"""清理文本中的密码、Token 和连接凭据"""
|
||||
for pattern, replacement in SENSITIVE_REPLACEMENTS:
|
||||
text = pattern.sub(replacement, text)
|
||||
return text
|
||||
|
||||
|
||||
class RedactingQueueHandler(QueueHandler):
|
||||
def prepare(self, record: logging.LogRecord) -> logging.LogRecord:
|
||||
"""在日志入队前对消息和格式化异常进行脱敏"""
|
||||
prepared = super().prepare(record)
|
||||
SensitiveDataFilter().filter(prepared)
|
||||
return prepared
|
||||
def redact_record(record: "Record") -> None:
|
||||
"""在输出前对日志消息和异常堆栈脱敏"""
|
||||
message = record["message"]
|
||||
exception = record["exception"]
|
||||
if exception is not None:
|
||||
# 异常文本可能携带凭据,转为已脱敏的消息文本后不再由 Loguru 渲染
|
||||
message += "\n" + "".join(traceback.format_exception(*exception))
|
||||
record["exception"] = None
|
||||
record["message"] = redact(message).rstrip("\n")
|
||||
|
||||
|
||||
class InterceptHandler(logging.Handler):
|
||||
"""将标准 logging 转发到 Loguru,兼容 Uvicorn 与第三方库"""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
"""转发标准日志记录并保持原调用位置"""
|
||||
try:
|
||||
level: str | int = logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = record.levelno
|
||||
frame, depth = inspect.currentframe(), 0
|
||||
while frame and (depth == 0 or frame.f_code.co_filename == logging.__file__):
|
||||
frame = frame.f_back
|
||||
depth += 1
|
||||
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||
|
||||
|
||||
def _setup_sinks(level: str) -> list[int]:
|
||||
"""添加 Loguru 输出并接管标准 logging,返回由本函数创建的输出编号"""
|
||||
LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
# 仅移除 Loguru 默认输出,保留其他调用方自行添加的输出
|
||||
with suppress(ValueError):
|
||||
logger.remove(0)
|
||||
logger.configure(patcher=redact_record)
|
||||
options = {
|
||||
"level": level.upper(),
|
||||
"format": LOG_FORMAT,
|
||||
"enqueue": True,
|
||||
"diagnose": False,
|
||||
"backtrace": False,
|
||||
}
|
||||
sink_ids = [
|
||||
logger.add(sys.stderr, colorize=True, **options),
|
||||
logger.add(
|
||||
LOG_PATH,
|
||||
rotation="20 MB",
|
||||
retention="14 days",
|
||||
compression="zip",
|
||||
encoding="utf-8",
|
||||
**options,
|
||||
),
|
||||
]
|
||||
# 清理已有处理器,让标准日志只经过根处理器输出一次
|
||||
logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)
|
||||
for name in list(logging.root.manager.loggerDict):
|
||||
existing = logging.getLogger(name)
|
||||
existing.handlers.clear()
|
||||
existing.propagate = True
|
||||
return sink_ids
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def configure_logging(level: str) -> AsyncIterator[None]:
|
||||
"""在应用生命周期内使用后台线程输出日志并在退出时排空队列"""
|
||||
app_logger = logging.getLogger("app")
|
||||
previous_handlers = app_logger.handlers[:]
|
||||
previous_level = app_logger.level
|
||||
previous_propagate = app_logger.propagate
|
||||
targets = previous_handlers or logging.getLogger().handlers[:]
|
||||
fallback = logging.StreamHandler() if not targets else None
|
||||
if fallback is not None:
|
||||
targets = [fallback]
|
||||
queue: SimpleQueue[logging.LogRecord] = SimpleQueue()
|
||||
queue_handler = RedactingQueueHandler(queue)
|
||||
listener = QueueListener(queue, *targets, respect_handler_level=True)
|
||||
listener.start()
|
||||
app_logger.handlers = [queue_handler]
|
||||
app_logger.setLevel(level.upper())
|
||||
app_logger.propagate = False
|
||||
"""在应用生命周期内配置日志并在退出时排空队列"""
|
||||
sink_ids = _setup_sinks(level)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
await to_thread.run_sync(listener.stop)
|
||||
finally:
|
||||
app_logger.handlers = previous_handlers
|
||||
app_logger.setLevel(previous_level)
|
||||
app_logger.propagate = previous_propagate
|
||||
queue_handler.close()
|
||||
if fallback is not None:
|
||||
fallback.close()
|
||||
await logger.complete()
|
||||
for sink_id in sink_ids:
|
||||
logger.remove(sink_id)
|
||||
|
||||
+4
-4
@@ -1,12 +1,12 @@
|
||||
# 检查启动时的数据库和 Redis 连接
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.core.database import check_database
|
||||
from app.core.redis import check_redis
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
STARTUP_CHECK_TIMEOUT_SECONDS = 10
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ async def check_startup_dependencies() -> None:
|
||||
except Exception as exc:
|
||||
reason = describe_connection_error(exc)
|
||||
logger.error(
|
||||
"%s连接失败,服务启动终止 error_type=%s reason=%s", name, type(exc).__name__, reason
|
||||
"{}连接失败,服务启动终止 error_type={} reason={}", name, type(exc).__name__, reason
|
||||
)
|
||||
raise RuntimeError(f"{name}连接失败:{reason}") from None
|
||||
logger.info("%s连接成功", name)
|
||||
logger.info("{}连接成功", name)
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
{
|
||||
"run_prefix": "bencha6ea70cf4a4e",
|
||||
"workers": 1,
|
||||
"logical_cpus": 20,
|
||||
"physical_cpus": 14,
|
||||
"stages": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"elapsed_s": 21.61,
|
||||
"success": 7,
|
||||
"errors": {},
|
||||
"success_rps": 0.32,
|
||||
"registration": {
|
||||
"count": 7,
|
||||
"mean_ms": 3085.21,
|
||||
"p50_ms": 3086.83,
|
||||
"p95_ms": 3095.23,
|
||||
"p99_ms": 3095.23,
|
||||
"max_ms": 3095.23
|
||||
},
|
||||
"authenticated_probe": {
|
||||
"count": 20,
|
||||
"mean_ms": 996.9,
|
||||
"p50_ms": 832.57,
|
||||
"p95_ms": 2472.89,
|
||||
"p99_ms": 2473.15,
|
||||
"max_ms": 2473.15
|
||||
},
|
||||
"probe_errors": {},
|
||||
"cpu_mean_percent_one_core": 6.1,
|
||||
"cpu_peak_percent_one_core": 23.0,
|
||||
"rss_peak_mb": 103.3
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"elapsed_s": 21.87,
|
||||
"success": 25,
|
||||
"errors": {},
|
||||
"success_rps": 1.14,
|
||||
"registration": {
|
||||
"count": 25,
|
||||
"mean_ms": 3357.33,
|
||||
"p50_ms": 3096.26,
|
||||
"p95_ms": 4733.97,
|
||||
"p99_ms": 4740.52,
|
||||
"max_ms": 4740.52
|
||||
},
|
||||
"authenticated_probe": {
|
||||
"count": 22,
|
||||
"mean_ms": 906.63,
|
||||
"p50_ms": 831.8,
|
||||
"p95_ms": 838.05,
|
||||
"p99_ms": 2476.45,
|
||||
"max_ms": 2476.45
|
||||
},
|
||||
"probe_errors": {},
|
||||
"cpu_mean_percent_one_core": 18.7,
|
||||
"cpu_peak_percent_one_core": 62.3,
|
||||
"rss_peak_mb": 161.6
|
||||
},
|
||||
{
|
||||
"concurrency": 8,
|
||||
"elapsed_s": 22.14,
|
||||
"success": 47,
|
||||
"errors": {},
|
||||
"success_rps": 2.12,
|
||||
"registration": {
|
||||
"count": 47,
|
||||
"mean_ms": 3627.04,
|
||||
"p50_ms": 3103.34,
|
||||
"p95_ms": 4803.8,
|
||||
"p99_ms": 6417.17,
|
||||
"max_ms": 6417.17
|
||||
},
|
||||
"authenticated_probe": {
|
||||
"count": 15,
|
||||
"mean_ms": 1377.47,
|
||||
"p50_ms": 836.83,
|
||||
"p95_ms": 2475.52,
|
||||
"p99_ms": 2475.52,
|
||||
"max_ms": 2475.52
|
||||
},
|
||||
"probe_errors": {},
|
||||
"cpu_mean_percent_one_core": 43.2,
|
||||
"cpu_peak_percent_one_core": 123.3,
|
||||
"rss_peak_mb": 364.0
|
||||
},
|
||||
{
|
||||
"concurrency": 16,
|
||||
"elapsed_s": 22.94,
|
||||
"success": 85,
|
||||
"errors": {},
|
||||
"success_rps": 3.71,
|
||||
"registration": {
|
||||
"count": 85,
|
||||
"mean_ms": 4081.2,
|
||||
"p50_ms": 3892.77,
|
||||
"p95_ms": 5114.77,
|
||||
"p99_ms": 5822.78,
|
||||
"max_ms": 5822.78
|
||||
},
|
||||
"authenticated_probe": {
|
||||
"count": 19,
|
||||
"mean_ms": 1135.8,
|
||||
"p50_ms": 1114.0,
|
||||
"p95_ms": 1992.33,
|
||||
"p99_ms": 1992.33,
|
||||
"max_ms": 1992.33
|
||||
},
|
||||
"probe_errors": {},
|
||||
"cpu_mean_percent_one_core": 64.7,
|
||||
"cpu_peak_percent_one_core": 101.4,
|
||||
"rss_peak_mb": 252.4
|
||||
},
|
||||
{
|
||||
"concurrency": 32,
|
||||
"elapsed_s": 25.38,
|
||||
"success": 96,
|
||||
"errors": {},
|
||||
"success_rps": 3.78,
|
||||
"registration": {
|
||||
"count": 96,
|
||||
"mean_ms": 7751.18,
|
||||
"p50_ms": 7630.97,
|
||||
"p95_ms": 9152.05,
|
||||
"p99_ms": 10273.37,
|
||||
"max_ms": 10273.37
|
||||
},
|
||||
"authenticated_probe": {
|
||||
"count": 11,
|
||||
"mean_ms": 2209.09,
|
||||
"p50_ms": 2158.04,
|
||||
"p95_ms": 3788.55,
|
||||
"p99_ms": 3788.55,
|
||||
"max_ms": 3788.55
|
||||
},
|
||||
"probe_errors": {},
|
||||
"cpu_mean_percent_one_core": 67.5,
|
||||
"cpu_peak_percent_one_core": 136.9,
|
||||
"rss_peak_mb": 112.3
|
||||
},
|
||||
{
|
||||
"concurrency": 64,
|
||||
"elapsed_s": 32.26,
|
||||
"success": 128,
|
||||
"errors": {},
|
||||
"success_rps": 3.97,
|
||||
"registration": {
|
||||
"count": 128,
|
||||
"mean_ms": 14727.87,
|
||||
"p50_ms": 15125.01,
|
||||
"p95_ms": 16402.9,
|
||||
"p99_ms": 17481.34,
|
||||
"max_ms": 17705.6
|
||||
},
|
||||
"authenticated_probe": {
|
||||
"count": 7,
|
||||
"mean_ms": 4525.38,
|
||||
"p50_ms": 4347.75,
|
||||
"p95_ms": 9358.62,
|
||||
"p99_ms": 9358.62,
|
||||
"max_ms": 9358.62
|
||||
},
|
||||
"probe_errors": {},
|
||||
"cpu_mean_percent_one_core": 63.5,
|
||||
"cpu_peak_percent_one_core": 223.1,
|
||||
"rss_peak_mb": 116.6
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"mysql_select_1": {
|
||||
"count": 10,
|
||||
"mean_ms": 276.5,
|
||||
"p50_ms": 276.42,
|
||||
"p95_ms": 277.11,
|
||||
"p99_ms": 277.11,
|
||||
"max_ms": 277.11
|
||||
},
|
||||
"redis_ping": {
|
||||
"count": 10,
|
||||
"mean_ms": 273.28,
|
||||
"p50_ms": 273.04,
|
||||
"p95_ms": 275.83,
|
||||
"p99_ms": 275.83,
|
||||
"max_ms": 275.83
|
||||
},
|
||||
"local_password_hash": {
|
||||
"count": 10,
|
||||
"mean_ms": 46.49,
|
||||
"p50_ms": 46.2,
|
||||
"p95_ms": 49.95,
|
||||
"p99_ms": 49.95,
|
||||
"max_ms": 49.95
|
||||
},
|
||||
"authenticated_request": {
|
||||
"count": 10,
|
||||
"mean_ms": 831.53,
|
||||
"p50_ms": 831.51,
|
||||
"p95_ms": 836.56,
|
||||
"p99_ms": 836.56,
|
||||
"max_ms": 836.56
|
||||
}
|
||||
},
|
||||
"cleanup": {
|
||||
"users_deleted": 389,
|
||||
"sessions_deleted": 1,
|
||||
"users_remaining": 0,
|
||||
"sessions_remaining": 0
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ dependencies = [
|
||||
"cryptography>=50.0.1",
|
||||
"email-validator>=2.2.0",
|
||||
"fastapi[standard]>=0.141.1",
|
||||
"loguru>=0.7.3",
|
||||
"pwdlib[argon2]>=0.3.0",
|
||||
"pydantic-settings>=2.10.0",
|
||||
"redis>=6.2.0",
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# 注册压测
|
||||
|
||||
在项目根目录运行:
|
||||
|
||||
```powershell
|
||||
uv sync
|
||||
uv run alembic upgrade head
|
||||
uv run python scripts/load_test.py --seconds 20
|
||||
```
|
||||
|
||||
脚本使用当前环境配置的真实 MySQL 和 Redis,自动启动本地单 worker Uvicorn
|
||||
服务(关闭 reload 和 access log),依次测试 1、4、8、16、32、64 个并发注册。
|
||||
每个档位持续发起请求 20 秒,然后等待在途请求完成;吞吐量包含收尾时间。
|
||||
可以使用 `--concurrency 1 8 32 --seconds 60` 调整档位和时长。
|
||||
|
||||
指标包括注册成功数、HTTP/网络错误、成功吞吐量、平均/P50/P95/P99/最大延迟,
|
||||
以及压测期间 `/api/users/me` 的响应延迟、错误数、服务进程树 CPU 和峰值 RSS。
|
||||
普通认证探针每次响应后等待 100ms 再发起请求,只有一个在途探针,不能代表大量普通用户。
|
||||
CPU 的 100% 代表占满一个逻辑核心;内存是服务进程树 RSS 之和,可能包含共享页重复计算。
|
||||
客户端与服务在同一台机器运行,资源采样间隔随探针响应耗时变化,峰值可能漏采。
|
||||
|
||||
基线分别测量 10 次热连接 `SELECT 1`、Redis PING、本机 Argon2 哈希和无注册负载的
|
||||
用户资料请求。网络指标包含远程服务处理时间,不等于纯网络延迟。
|
||||
P95/P99 使用 nearest-rank 算法;短测样本少,不能据此推断长期稳定性或生产容量。
|
||||
这是闭环并发测试,并发数表示同时在途的注册请求数,不是在线用户总数。
|
||||
|
||||
报告默认保存到 `artifacts/load-test.json`,可使用 `--output` 指定路径。
|
||||
报告不包含密码、令牌和连接地址。结束后删除本轮随机前缀用户及对应 Redis 会话,
|
||||
并记录残留计数;保留表结构和其他数据。异常时也会尝试清理,但强制结束整个脚本、
|
||||
断电或依赖不可用时无法保证清理成功,应检查报告中的 `cleanup` 字段。
|
||||
@@ -0,0 +1,334 @@
|
||||
"""通过真实 HTTP、MySQL 和 Redis 测量注册并发性能并清理数据"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
|
||||
def summarize(samples: list[float]) -> dict:
|
||||
"""汇总毫秒延迟并保留尾部耗时"""
|
||||
values = sorted(samples)
|
||||
if not values:
|
||||
return {}
|
||||
return {
|
||||
"count": len(values),
|
||||
"mean_ms": round(sum(values) / len(values), 2),
|
||||
"p50_ms": round(values[math.ceil(len(values) * 0.50) - 1], 2),
|
||||
"p95_ms": round(values[math.ceil(len(values) * 0.95) - 1], 2),
|
||||
"p99_ms": round(values[math.ceil(len(values) * 0.99) - 1], 2),
|
||||
"max_ms": round(values[-1], 2),
|
||||
}
|
||||
|
||||
|
||||
async def cleanup(prefix: str) -> dict:
|
||||
"""删除本轮用户及其 Redis 会话并核验残留"""
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import async_session_factory, close_database
|
||||
from app.core.redis import close_redis, redis_client
|
||||
from app.model.user import User
|
||||
|
||||
try:
|
||||
async with async_session_factory() as session:
|
||||
condition = User.username.startswith(prefix, autoescape=True)
|
||||
ids = set(map(str, (await session.scalars(select(User.id).where(condition))).all()))
|
||||
keys_deleted = 0
|
||||
pattern = f"{get_settings().redis_key_prefix}:session:*"
|
||||
async for key in redis_client.scan_iter(match=pattern, count=200):
|
||||
if await redis_client.get(key) in ids:
|
||||
keys_deleted += await redis_client.delete(key)
|
||||
result = await session.execute(delete(User).where(condition))
|
||||
await session.commit()
|
||||
remaining = await session.scalar(
|
||||
select(func.count()).select_from(User).where(condition)
|
||||
)
|
||||
remaining_sessions = 0
|
||||
async for key in redis_client.scan_iter(match=pattern, count=200):
|
||||
if await redis_client.get(key) in ids:
|
||||
remaining_sessions += 1
|
||||
return {
|
||||
"users_deleted": result.rowcount,
|
||||
"sessions_deleted": keys_deleted,
|
||||
"users_remaining": remaining,
|
||||
"sessions_remaining": remaining_sessions,
|
||||
}
|
||||
finally:
|
||||
await close_redis()
|
||||
await close_database()
|
||||
|
||||
|
||||
async def baseline() -> dict:
|
||||
"""分别测量热连接往返和本机密码计算作为诊断基线"""
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.database import engine
|
||||
from app.core.redis import redis_client
|
||||
from app.core.security import hash_password
|
||||
|
||||
results = {}
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text("SELECT 1"))
|
||||
samples = []
|
||||
for _ in range(10):
|
||||
start = time.perf_counter()
|
||||
await connection.execute(text("SELECT 1"))
|
||||
samples.append((time.perf_counter() - start) * 1000)
|
||||
results["mysql_select_1"] = summarize(samples)
|
||||
await redis_client.ping()
|
||||
samples = []
|
||||
for _ in range(10):
|
||||
start = time.perf_counter()
|
||||
await redis_client.ping()
|
||||
samples.append((time.perf_counter() - start) * 1000)
|
||||
results["redis_ping"] = summarize(samples)
|
||||
samples = []
|
||||
for _ in range(10):
|
||||
start = time.perf_counter()
|
||||
await hash_password("BenchmarkOnly123")
|
||||
samples.append((time.perf_counter() - start) * 1000)
|
||||
results["local_password_hash"] = summarize(samples)
|
||||
return results
|
||||
|
||||
|
||||
async def stage(
|
||||
client: httpx.AsyncClient,
|
||||
process: psutil.Process,
|
||||
prefix: str,
|
||||
password: str,
|
||||
token: str,
|
||||
concurrency: int,
|
||||
seconds: float,
|
||||
) -> dict:
|
||||
"""并行注册并采样已登录请求延迟与服务资源"""
|
||||
latencies: list[float] = []
|
||||
probe_latencies: list[float] = []
|
||||
errors: Counter = Counter()
|
||||
probe_errors: Counter = Counter()
|
||||
cpu: list[float] = []
|
||||
memory: list[float] = []
|
||||
sequence = 0
|
||||
completed = 0
|
||||
stop = asyncio.Event()
|
||||
start = time.perf_counter()
|
||||
deadline = start + seconds
|
||||
processes = [process, *process.children(recursive=True)]
|
||||
for item in processes:
|
||||
item.cpu_percent()
|
||||
|
||||
async def worker() -> None:
|
||||
"""保持一个在途注册请求直到本阶段停止发起请求"""
|
||||
nonlocal sequence, completed
|
||||
while time.perf_counter() < deadline:
|
||||
sequence += 1
|
||||
name = f"{prefix}{concurrency}_{sequence}"
|
||||
tick = time.perf_counter()
|
||||
try:
|
||||
response = await client.post(
|
||||
"/api/auth/register",
|
||||
json={
|
||||
"username": name,
|
||||
"email": f"{name}@example.com",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
if response.status_code == 200 and response.json().get("code") == 0:
|
||||
completed += 1
|
||||
else:
|
||||
errors[str(response.status_code)] += 1
|
||||
except httpx.HTTPError as exc:
|
||||
errors[type(exc).__name__] += 1
|
||||
latencies.append((time.perf_counter() - tick) * 1000)
|
||||
|
||||
async def probe() -> None:
|
||||
"""连续探测涉及数据库和 Redis 的普通认证请求"""
|
||||
while not stop.is_set():
|
||||
tick = time.perf_counter()
|
||||
try:
|
||||
response = await client.get(
|
||||
"/api/users/me", headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
if response.status_code != 200 or response.json().get("code") != 0:
|
||||
probe_errors[str(response.status_code)] += 1
|
||||
except httpx.HTTPError as exc:
|
||||
probe_errors[type(exc).__name__] += 1
|
||||
probe_latencies.append((time.perf_counter() - tick) * 1000)
|
||||
cpu.append(sum(item.cpu_percent() for item in processes if item.is_running()))
|
||||
memory.append(
|
||||
sum(item.memory_info().rss for item in processes if item.is_running()) / 1024**2
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
probe_task = asyncio.create_task(probe())
|
||||
try:
|
||||
await asyncio.gather(*(worker() for _ in range(concurrency)))
|
||||
finally:
|
||||
elapsed = time.perf_counter() - start
|
||||
stop.set()
|
||||
await probe_task
|
||||
return {
|
||||
"concurrency": concurrency,
|
||||
"elapsed_s": round(elapsed, 2),
|
||||
"success": completed,
|
||||
"errors": dict(errors),
|
||||
"success_rps": round(completed / elapsed, 2),
|
||||
"registration": summarize(latencies),
|
||||
"authenticated_probe": summarize(probe_latencies),
|
||||
"probe_errors": dict(probe_errors),
|
||||
"cpu_mean_percent_one_core": round(sum(cpu) / len(cpu), 1),
|
||||
"cpu_peak_percent_one_core": round(max(cpu), 1),
|
||||
"rss_peak_mb": round(max(memory), 1),
|
||||
}
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> None:
|
||||
"""启动独立单进程服务完成压测并确保结束后清理"""
|
||||
prefix = f"bench{secrets.token_hex(6)}"
|
||||
password = secrets.token_urlsafe(20) + "a9"
|
||||
with socket.socket() as bound:
|
||||
bound.bind(("127.0.0.1", 0))
|
||||
port = bound.getsockname()[1]
|
||||
report = {
|
||||
"run_prefix": prefix,
|
||||
"workers": 1,
|
||||
"logical_cpus": os.cpu_count(),
|
||||
"physical_cpus": psutil.cpu_count(logical=False),
|
||||
"stages": [],
|
||||
}
|
||||
process = await asyncio.to_thread(
|
||||
subprocess.Popen,
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"app.main:app",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(port),
|
||||
"--no-access-log",
|
||||
"--log-level",
|
||||
"error",
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=f"http://127.0.0.1:{port}",
|
||||
timeout=60,
|
||||
limits=httpx.Limits(max_connections=256),
|
||||
trust_env=False,
|
||||
) as client:
|
||||
for _ in range(150):
|
||||
if process.poll() is not None:
|
||||
raise RuntimeError("压测服务启动失败,请检查依赖和配置")
|
||||
try:
|
||||
if (await client.get("/openapi.json")).status_code == 200:
|
||||
break
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
await asyncio.sleep(0.2)
|
||||
else:
|
||||
raise RuntimeError("压测服务启动超时")
|
||||
report["baseline"] = await baseline()
|
||||
print(json.dumps(report["baseline"], ensure_ascii=False), flush=True)
|
||||
name = prefix + "probe"
|
||||
response = await client.post(
|
||||
"/api/auth/register",
|
||||
json={
|
||||
"username": name,
|
||||
"email": f"{name}@example.com",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"探针用户注册失败 HTTP {response.status_code}")
|
||||
response = await client.post(
|
||||
"/api/auth/login", json={"account": name, "password": password}
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"探针登录失败 HTTP {response.status_code}")
|
||||
token = response.json()["data"]["token"]
|
||||
samples = []
|
||||
for _ in range(10):
|
||||
tick = time.perf_counter()
|
||||
response = await client.get(
|
||||
"/api/users/me", headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError("认证基线请求失败")
|
||||
samples.append((time.perf_counter() - tick) * 1000)
|
||||
report["baseline"]["authenticated_request"] = summarize(samples)
|
||||
for concurrency in args.concurrency:
|
||||
result = await stage(
|
||||
client,
|
||||
psutil.Process(process.pid),
|
||||
prefix,
|
||||
password,
|
||||
token,
|
||||
concurrency,
|
||||
args.seconds,
|
||||
)
|
||||
report["stages"].append(result)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
except Exception as exc:
|
||||
report["error_type"] = type(exc).__name__
|
||||
raise
|
||||
finally:
|
||||
children = []
|
||||
if process.poll() is None:
|
||||
children = psutil.Process(process.pid).children(recursive=True)
|
||||
for child in children:
|
||||
child.terminate()
|
||||
process.terminate()
|
||||
await asyncio.to_thread(process.wait)
|
||||
await asyncio.to_thread(psutil.wait_procs, children, timeout=10)
|
||||
try:
|
||||
report["cleanup"] = await cleanup(prefix)
|
||||
except Exception as exc:
|
||||
report["cleanup"] = {"error_type": type(exc).__name__, "verified": False}
|
||||
raise
|
||||
finally:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
print(json.dumps(report["cleanup"], ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""解析压测档位和时长并运行真实依赖测试"""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--concurrency", nargs="+", type=int, default=[1, 4, 8, 16, 32, 64])
|
||||
parser.add_argument("--seconds", type=float, default=12)
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "artifacts/load-test.json")
|
||||
args = parser.parse_args()
|
||||
if args.seconds <= 0 or any(value < 1 or value > 200 for value in args.concurrency):
|
||||
parser.error("时长必须为正数,并发必须在 1 到 200 之间")
|
||||
try:
|
||||
asyncio.run(run(args))
|
||||
except Exception as exc:
|
||||
print(f"压测失败:{type(exc).__name__},请检查报告和服务配置", file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -205,6 +205,7 @@ dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "email-validator" },
|
||||
{ name = "fastapi", extra = ["standard"] },
|
||||
{ name = "loguru" },
|
||||
{ name = "pwdlib", extra = ["argon2"] },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "redis" },
|
||||
@@ -230,6 +231,7 @@ requires-dist = [
|
||||
{ name = "cryptography", specifier = ">=50.0.1" },
|
||||
{ name = "email-validator", specifier = ">=2.2.0" },
|
||||
{ name = "fastapi", extras = ["standard"], specifier = ">=0.141.1" },
|
||||
{ name = "loguru", specifier = ">=0.7.3" },
|
||||
{ name = "pwdlib", extras = ["argon2"], specifier = ">=0.3.0" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.10.0" },
|
||||
{ name = "redis", specifier = ">=6.2.0" },
|
||||
@@ -862,6 +864,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loguru"
|
||||
version = "0.7.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "win32-setctime", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mako"
|
||||
version = "1.4.1"
|
||||
@@ -1950,3 +1965,12 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/be/7e/75a0a491b512412e08333b9f8412757af6186fe1c598186261002de1a793/websockets-17.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8196d217eeca52b9235ee1f8a684a09885a5f953d5a31e80ef915bf2c5c94f9d", size = 217870, upload-time = "2026-08-26T17:25:29.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/41/63/23572870e01836a98346075b9e17a8bc24a6ddd9800a3204ceee58677f3c/websockets-17.1-py3-none-any.whl", hash = "sha256:f221081107b8c48184d99f7019604486376e7ef826037e70aad6b02540732c23", size = 211134, upload-time = "2026-08-26T17:25:31.397Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "win32-setctime"
|
||||
version = "1.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" },
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user