Files
paper-doc/backend/scripts/smoke_mcp.py
T
govin 199e328b48 refactor: call a heading a title, and its use a placement
`template_field_library` and `template_field` were one word apart and meant
opposite kinds of thing — a catalogue of rows versus one row — and neither name
said what a row actually is. Every row of the library is a **heading**: a piece
of text like 「1. Introduction」 with a level and typography, which is what the
reader sees at the top of a paragraph; and every row of the join table is that
heading *placed* in a template, `sort` included.

    template_field_library  -> template_title
    template_field          -> template_title_placement
    field_id (the column)   -> title_id
    TemplateFieldLibrary    -> TemplateTitle
    TemplateField           -> TemplateTitlePlacement

The join table keeps its own name rather than becoming `_link`, because it is
not a bare link: it carries `sort`, the only place display order is stored, and
it deliberately allows the same title twice in one template. `_link` would
describe two foreign keys and hide the column the ordering rule rests on.

Carried through every level, because a half-renamed layer is how the confusing
pair got there in the first place: HTTP paths (`/template-field-library` ->
`/template-titles`), the template payload (`fields`/`field_id` ->
`placements`/`title_id`, and `field_count` -> `placement_count`), the Pydantic
schemas, the CRUD modules, the seed script, the frontend API module and types,
and `docs/OVERVIEW.md`, which now states the naming rules outright.

Two deliberate exceptions:

* `paper_template_filed_sort` is untouched — it is a column of the paper module
  and is spelled as the feature was specified.
* The MCP tools keep their agent-facing names (`field_list`, `field_ids`, and
  the `fields`/`field_id` keys in their responses). Agents already call them by
  those names; the storage names they read underneath have changed.

The migration renames in place (data preserved: 4 templates, 45 titles, 63
placements) and is written to tolerate a database where part of it already ran,
which is not hypothetical — the first attempt renamed the tables and then failed
on the column, because Alembic's MySQL dialect wants an explicit type for
`alter_column` and will not rename a column without one. TiDB carries the
foreign key along with both the renamed table and the renamed column, rehearsed
against the cluster first, and it stays enforced.

Verified: seed re-runs clean, the paper smoke test's 45 checks and the MCP
smoke test's 47 pass, `vue-tsc` and the build are clean, and the UI was driven
in a headless browser through the template list, its preview drawer, the title
library page, the template form (which now posts `placements` with `title_id`)
and the paper document with no console errors.
2026-09-22 16:55:02 +08:00

516 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""End-to-end smoke test for the MCP server, spoken as a real MCP client.
Run it from ``backend/``::
.venv/bin/python scripts/smoke_mcp.py
It spawns ``scripts/mcp_server.py`` exactly the way Claude Code, Codex and the
Harness do — a child process, JSON-RPC over its stdin/stdout — and then walks
the whole writing loop through tool calls: create a paper on a template, read
the outline, write paragraphs by *heading*, write a whole generated document in
one call, edit one sentence without rewriting its paragraph, search across
papers, switch the template, and delete everything it created.
Speaking the real protocol is the point. Calling the tool functions directly
would test the domain layer, which ``scripts/smoke_papers.py`` already covers;
what is untested until a client connects is the part that only exists on the
wire — argument schemas, tool discovery, error results, and the stdio stream
staying clean enough to carry the protocol.
Exits non-zero on the first failed expectation and cleans up after itself.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
import anyio
BACKEND_DIR = Path(__file__).resolve().parents[1]
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from mcp import types # noqa: E402
from mcp.client.session import ClientSession # noqa: E402
from mcp.client.stdio import StdioServerParameters, stdio_client # noqa: E402
from sqlalchemy import or_, select # noqa: E402
from app.db.session import SessionLocal # noqa: E402
from app.models import Paper, Template # noqa: E402
SERVER_SCRIPT = BACKEND_DIR / "scripts" / "mcp_server.py"
_checks = 0
_created_papers: list[int] = []
_created_templates: list[int] = []
def check(label: str, condition: bool, detail: str = "") -> None:
"""Assert one expectation and report it."""
global _checks
_checks += 1
if not condition:
raise AssertionError(f"{label}: {detail}")
print(f" ok {label}")
def payload(result: types.CallToolResult) -> Any:
"""The JSON a tool returned, refusing an error result.
A tool that failed returns ``isError`` with the message as its content, not
an exception — so a smoke test that only looked at the content would read a
refusal as data. That is the failure mode this helper exists to prevent.
"""
text = "".join(
block.text for block in result.content if isinstance(block, types.TextContent)
)
if result.is_error:
raise AssertionError(f"tool returned an error: {text}")
return json.loads(text) if text else None
async def call(session: ClientSession, tool: str, **arguments: Any) -> Any:
"""Call one tool and return its JSON payload.
The parameter is ``tool``, not ``name``: several tools here take an
argument called ``name`` (``template_create``, ``field_create``) and a
helper whose own parameter shadowed it would fail with a TypeError that
looks nothing like the tool call it came from.
"""
return payload(await session.call_tool(tool, arguments))
async def expect_error(session: ClientSession, tool: str, **arguments: Any) -> str:
"""Call one tool expecting a refusal, and return the message."""
result = await session.call_tool(tool, arguments)
if not result.is_error:
raise AssertionError(f"{tool} should have refused, but returned {result.content}")
return "".join(
block.text for block in result.content if isinstance(block, types.TextContent)
)
def pick_template() -> tuple[int, list[dict[str, Any]]]:
"""The template with the most titles, and its outline as the tools see it."""
with SessionLocal() as db:
template = db.scalars(
select(Template).order_by(Template.id.asc())
).all()
best = max(template, key=lambda item: len(item.placements))
return best.id, [
{"position": item.sort, "name": item.title.name}
for item in best.placements
]
@asynccontextmanager
async def connect(url: str | None, token: str | None) -> AsyncIterator[ClientSession]:
"""Open a session over stdio (default) or against a running HTTP server.
Both paths yield the same :class:`ClientSession`, so every check below runs
unchanged on either transport — which is the point of testing the protocol
rather than the functions: the tools must behave identically whether a
client spawned them or connected to them.
"""
if url:
# headers live on the HTTP client, not on the transport call.
import httpx2
from mcp.client.streamable_http import streamable_http_client
headers = {"Authorization": f"Bearer {token}"} if token else {}
async with httpx2.AsyncClient(headers=headers) as http_client:
async with streamable_http_client(url, http_client=http_client) as (
read,
write,
):
async with ClientSession(read, write) as session:
yield session
return
parameters = StdioServerParameters(
command=sys.executable,
args=[str(SERVER_SCRIPT)],
cwd=str(BACKEND_DIR),
)
async with stdio_client(parameters) as (read, write):
async with ClientSession(read, write) as session:
yield session
async def run(url: str | None = None, token: str | None = None) -> int:
template_id, outline = pick_template()
check("a template with fields exists", len(outline) >= 3, f"got {len(outline)}")
first = outline[0]["name"]
second = outline[1]["name"]
async with connect(url, token) as session:
handshake = await session.initialize()
check(
"server announces itself",
handshake.server_info.name == "paper-doc",
str(handshake.server_info),
)
print("\n1. tool discovery")
listed = await session.list_tools()
names = {tool.name for tool in listed.tools}
check("29 tools are published", len(names) == 29, f"got {len(names)}")
check(
"every group is reachable",
{"paper_write", "paragraph_write", "sentence_add", "template_get", "field_list"}
<= names,
)
check(
"no tool returns a structured envelope",
all(tool.output_schema is None for tool in listed.tools),
)
write_tool = next(tool for tool in listed.tools if tool.name == "paper_write")
check(
"paper_write describes its paragraph array",
"paragraphs" in write_tool.input_schema["properties"],
)
print("\n2. create a paper on that template")
paper = await call(
session,
"paper_create",
title="SMOKE MCP 论文",
template_id=template_id,
author="smoke",
keywords="测试, smoke, 测试",
)
_created_papers.append(paper["id"])
check("created", paper["title"] == "SMOKE MCP 论文")
check("keywords are deduplicated", paper["keywords"] == "测试, smoke", paper["keywords"])
check(
"the whole structure exists with no content",
paper["paragraphs"] == 0 and paper["template_paragraphs"] == len(outline),
json.dumps(paper, ensure_ascii=False),
)
print("\n3. read the outline before writing")
skeleton = await call(session, "paper_outline", paper_id=paper["id"])
check(
"the outline carries every position",
len(skeleton["paragraphs"]) == len(outline),
json.dumps(skeleton, ensure_ascii=False),
)
check("nothing has been written", all(item["sentences"] == 0 for item in skeleton["paragraphs"]))
print("\n4. write paragraphs by heading")
written = await call(
session,
"paper_write",
paper_id=paper["id"],
paragraphs=[
{
"heading": first,
"sentences": ["第一句。", "第二句。"],
},
{
"heading": second.lstrip("0123456789. "),
"text": "这一整段只有一行,按句号切成两句。第二句在这里。",
"split": "sentence",
},
],
)
check("two paragraphs written", len(written["written"]) == 2, json.dumps(written, ensure_ascii=False))
check("four sentences total", written["total_sentences"] == 4, str(written["total_sentences"]))
check(
"both headings resolved to template positions",
all(item["heading"] for item in written["written"]),
json.dumps(written, ensure_ascii=False),
)
print("\n5. a heading the template does not have is refused, with candidates")
message = await expect_error(
session,
"paragraph_write",
paper_id=paper["id"],
heading="不存在的标题",
sentences=["x"],
)
check("the refusal lists the real headings", first in message, message)
print("\n6. read one paragraph back")
paragraph = await call(
session, "paragraph_get", paper_id=paper["id"], heading=first
)
check("two sentences stored", len(paragraph["sentences"]) == 2, json.dumps(paragraph, ensure_ascii=False))
check(
"CJK seams take no separator",
all(row["separator_before"] == "" for row in paragraph["sentences"]),
json.dumps(paragraph["sentences"], ensure_ascii=False),
)
print("\n7. append a sentence with a citation")
added = await call(
session,
"sentence_add",
paper_id=paper["id"],
heading=first,
content="A cited sentence.",
citations=[{"quote": "引用的原话", "reference_id": 7}],
)
check("appended", added["sentence"]["content"] == "A cited sentence.")
check("the citation is stored", added["sentence"]["citations"][0]["quote"] == "引用的原话")
after = await call(session, "paragraph_get", paper_id=paper["id"], heading=first)
check(
"the seam before it takes a space",
after["sentences"][-1]["separator_before"] == " ",
json.dumps(after["sentences"][-1], ensure_ascii=False),
)
print("\n8. edit one sentence without touching its paragraph")
edited = await call(
session,
"sentence_update",
paper_id=paper["id"],
sentence_id=after["sentences"][0]["id"],
content="改过的第一句。",
)
check("edited", edited["sentence"]["content"] == "改过的第一句。")
still = await call(session, "paragraph_get", paper_id=paper["id"], heading=first)
check("the other sentences kept their ids", still["sentences"][1]["id"] == after["sentences"][1]["id"])
print("\n9. delete one sentence")
before = await call(session, "sentence_list", paper_id=paper["id"])
await call(
session,
"sentence_delete",
paper_id=paper["id"],
sentence_id=still["sentences"][-1]["id"],
)
remaining = await call(session, "sentence_list", paper_id=paper["id"])
check(
"exactly one sentence fewer",
remaining["total"] == before["total"] - 1,
f"{before['total']} -> {remaining['total']}",
)
print("\n10. write a whole generated document in one call")
generated = "\n".join(
[f"# {item['name']}" + "\n" + f"{item['name']}的正文写在标题下面。这句话是第二句。"
for item in outline]
)
whole = await call(
session, "paper_write_text", paper_id=paper["id"], text=generated, split="sentence"
)
check(
"every heading in the document was written",
len(whole["written"]) == len(outline),
json.dumps(whole, ensure_ascii=False),
)
check("nothing was left unmatched", "unmatched" not in whole, json.dumps(whole, ensure_ascii=False))
print("\n11. read the paper as a document")
document = await call(session, "paper_document", paper_id=paper["id"])
check("the text carries every heading", all(item["name"] in document["text"] for item in outline))
check("and every paragraph", document["text"].count("正文写在标题下面") == len(outline))
structured = await call(
session, "paper_document", paper_id=paper["id"], format="json"
)
check(
"the json form carries sentence ids",
all(
row.get("id") is not None
for paragraph in structured["paragraphs"]
for row in paragraph["sentences"]
),
)
print("\n12. search across papers")
found = await call(session, "sentence_search", keyword="正文写在标题下面", paper_id=paper["id"])
check("the search finds what was written", found["total"] >= 1, str(found["total"]))
print("\n13. move a paragraph, then clear the one it moved onto")
home = outline[0]
target = outline[1]
relocation = await call(
session,
"paragraph_move",
paper_id=paper["id"],
heading=home["name"],
target_heading=target["name"],
)
check(
"the move reports both ends",
relocation["from"]["position"] == home["position"]
and relocation["to"]["position"] == target["position"],
json.dumps(relocation, ensure_ascii=False),
)
emptied = await call(session, "paragraph_get", paper_id=paper["id"], position=home["position"])
check("the source paragraph is empty now", emptied["sentences"] == [], json.dumps(emptied, ensure_ascii=False))
merged = await call(session, "paragraph_get", paper_id=paper["id"], position=target["position"])
check(
"the target holds both paragraphs' sentences",
len(merged["sentences"]) == 4,
json.dumps(merged["sentences"], ensure_ascii=False),
)
cleared = await call(
session, "paragraph_delete", paper_id=paper["id"], position=target["position"]
)
check("clearing reports how much it removed", cleared["deleted_sentences"] == 4, json.dumps(cleared, ensure_ascii=False))
after_clear = await call(session, "paragraph_get", paper_id=paper["id"], position=target["position"])
check("and the paragraph is empty", after_clear["sentences"] == [], json.dumps(after_clear, ensure_ascii=False))
print("\n14. the field library round-trips")
field = await call(session, "field_create", name="SMOKE MCP 字段", level=2, font_size=9.5)
check("created with its typography", field["level"] == 2 and field["font_size"] == 9.5, json.dumps(field, ensure_ascii=False))
renamed = await call(session, "field_update", field_id=field["id"], name="SMOKE MCP 字段(改)")
check("renamed", renamed["name"] == "SMOKE MCP 字段(改)", json.dumps(renamed, ensure_ascii=False))
dropped = await call(session, "field_delete", field_ids=field["id"])
check("deleted while no template places it", dropped["deleted"] == 1, json.dumps(dropped, ensure_ascii=False))
check("and it is reported as found, not missing", dropped["missing"] == [], json.dumps(dropped, ensure_ascii=False))
print("\n15. switch the template and keep the content")
spare = await call(
session,
"template_create",
name="SMOKE MCP 模板",
abstract="临时模板",
field_ids=[_field_id(outline[0]["name"])],
)
_created_templates.append(spare["id"])
check("a template can be built from field ids", spare["fields"][0]["position"] == 10, json.dumps(spare, ensure_ascii=False))
rebuilt = await call(
session,
"template_update",
template_id=spare["id"],
name="SMOKE MCP 模板",
field_ids=[_field_id(outline[0]["name"]), _field_id(outline[1]["name"])],
)
check(
"the outline is replaced wholesale and renumbered",
[item["position"] for item in rebuilt["fields"]] == [10, 20],
json.dumps(rebuilt, ensure_ascii=False),
)
sentences_before = (await call(session, "sentence_list", paper_id=paper["id"]))["total"]
switched = await call(
session, "paper_update", paper_id=paper["id"], template_id=spare["id"]
)
check("the switch is reported", switched["template_changed"] is True)
switched_document = await call(
session, "paper_document", paper_id=paper["id"], format="json"
)
check(
"content survives the switch, under 未设定",
sum(len(row["sentences"]) for row in switched_document["paragraphs"])
== sentences_before,
json.dumps(switched_document, ensure_ascii=False)[:400],
)
check(
"positions the new template does not define are marked unmatched",
any(row["matched"] is False for row in switched_document["paragraphs"]),
json.dumps(switched_document, ensure_ascii=False)[:400],
)
print("\n16. a template in use cannot be deleted")
message = await expect_error(session, "template_delete", template_ids=spare["id"])
check("the refusal names the blocker", "SMOKE MCP 模板" in message, message)
print("\n17. deleting a paper needs an explicit confirmation")
message = await expect_error(session, "paper_delete", paper_ids=paper["id"])
check("the refusal explains itself", "confirm=true" in message, message)
print("\n18. delete the paper and the template")
deleted = await call(session, "paper_delete", paper_ids=paper["id"], confirm=True)
check("deleted", deleted["deleted"] == 1, json.dumps(deleted, ensure_ascii=False))
_created_papers.remove(paper["id"])
gone = await call(session, "template_delete", template_ids=spare["id"])
check("the template is free to go now", gone["deleted"] == 1, json.dumps(gone, ensure_ascii=False))
_created_templates.remove(spare["id"])
print(f"\nall {_checks} checks passed")
return 0
def _field_id(name: str) -> int:
"""The library field behind a heading name, for the template-create step."""
from app.models import TemplateTitle
with SessionLocal() as db:
row = db.scalars(
select(TemplateTitle).where(TemplateTitle.name == name)
).first()
if row is None:
raise AssertionError(f"field {name!r} not in the library")
return row.id
def cleanup() -> None:
"""Remove anything this run created, even after a failed expectation."""
if _created_papers:
with SessionLocal() as db:
for paper in db.scalars(select(Paper).where(Paper.id.in_(_created_papers))).all():
db.delete(paper)
db.commit()
print(f"cleaned up papers {_created_papers}")
if _created_templates:
with SessionLocal() as db:
for template in db.scalars(
select(Template).where(Template.id.in_(_created_templates))
).all():
db.delete(template)
db.commit()
print(f"cleaned up templates {_created_templates}")
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
"""``--url`` switches the suite from spawning a server to dialling one."""
parser = argparse.ArgumentParser(description="paper-doc MCP 冒烟测试")
parser.add_argument(
"--url",
default=os.getenv("MCP_SMOKE_URL"),
help="连到已在运行的 streamable-http 服务,例如 http://127.0.0.1:8931/mcp;"
"不传则自己拉起 stdio 子进程",
)
parser.add_argument(
"--token",
default=os.getenv("MCP_HTTP_TOKEN"),
help="HTTP 传输的 Bearer 口令",
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
# The smoke test writes real rows, so it removes its own first: a previous
# run that died before cleanup would otherwise leave a duplicate title.
with SessionLocal() as db:
stale = db.scalars(
select(Paper).where(or_(Paper.title == "SMOKE MCP 论文"))
).all()
for paper in stale:
db.delete(paper)
stale_templates = db.scalars(
select(Template).where(Template.name == "SMOKE MCP 模板")
).all()
for template in stale_templates:
db.delete(template)
db.commit()
try:
return anyio.run(run, args.url, args.token)
except AssertionError as error:
print(f"\nFAILED: {error}", file=sys.stderr)
return 1
finally:
cleanup()
if __name__ == "__main__":
sys.exit(main())