199e328b48
`template_field_library` and `template_field` were one word apart and meant
opposite kinds of thing — a catalogue of rows versus one row — and neither name
said what a row actually is. Every row of the library is a **heading**: a piece
of text like 「1. Introduction」 with a level and typography, which is what the
reader sees at the top of a paragraph; and every row of the join table is that
heading *placed* in a template, `sort` included.
template_field_library -> template_title
template_field -> template_title_placement
field_id (the column) -> title_id
TemplateFieldLibrary -> TemplateTitle
TemplateField -> TemplateTitlePlacement
The join table keeps its own name rather than becoming `_link`, because it is
not a bare link: it carries `sort`, the only place display order is stored, and
it deliberately allows the same title twice in one template. `_link` would
describe two foreign keys and hide the column the ordering rule rests on.
Carried through every level, because a half-renamed layer is how the confusing
pair got there in the first place: HTTP paths (`/template-field-library` ->
`/template-titles`), the template payload (`fields`/`field_id` ->
`placements`/`title_id`, and `field_count` -> `placement_count`), the Pydantic
schemas, the CRUD modules, the seed script, the frontend API module and types,
and `docs/OVERVIEW.md`, which now states the naming rules outright.
Two deliberate exceptions:
* `paper_template_filed_sort` is untouched — it is a column of the paper module
and is spelled as the feature was specified.
* The MCP tools keep their agent-facing names (`field_list`, `field_ids`, and
the `fields`/`field_id` keys in their responses). Agents already call them by
those names; the storage names they read underneath have changed.
The migration renames in place (data preserved: 4 templates, 45 titles, 63
placements) and is written to tolerate a database where part of it already ran,
which is not hypothetical — the first attempt renamed the tables and then failed
on the column, because Alembic's MySQL dialect wants an explicit type for
`alter_column` and will not rename a column without one. TiDB carries the
foreign key along with both the renamed table and the renamed column, rehearsed
against the cluster first, and it stays enforced.
Verified: seed re-runs clean, the paper smoke test's 45 checks and the MCP
smoke test's 47 pass, `vue-tsc` and the build are clean, and the UI was driven
in a headless browser through the template list, its preview drawer, the title
library page, the template form (which now posts `placements` with `title_id`)
and the paper document with no console errors.
516 lines
21 KiB
Python
516 lines
21 KiB
Python
"""End-to-end smoke test for the MCP server, spoken as a real MCP client.
|
||
|
||
Run it from ``backend/``::
|
||
|
||
.venv/bin/python scripts/smoke_mcp.py
|
||
|
||
It spawns ``scripts/mcp_server.py`` exactly the way Claude Code, Codex and the
|
||
Harness do — a child process, JSON-RPC over its stdin/stdout — and then walks
|
||
the whole writing loop through tool calls: create a paper on a template, read
|
||
the outline, write paragraphs by *heading*, write a whole generated document in
|
||
one call, edit one sentence without rewriting its paragraph, search across
|
||
papers, switch the template, and delete everything it created.
|
||
|
||
Speaking the real protocol is the point. Calling the tool functions directly
|
||
would test the domain layer, which ``scripts/smoke_papers.py`` already covers;
|
||
what is untested until a client connects is the part that only exists on the
|
||
wire — argument schemas, tool discovery, error results, and the stdio stream
|
||
staying clean enough to carry the protocol.
|
||
|
||
Exits non-zero on the first failed expectation and cleans up after itself.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import sys
|
||
from collections.abc import AsyncIterator
|
||
from contextlib import asynccontextmanager
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import anyio
|
||
|
||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||
if str(BACKEND_DIR) not in sys.path:
|
||
sys.path.insert(0, str(BACKEND_DIR))
|
||
|
||
from mcp import types # noqa: E402
|
||
from mcp.client.session import ClientSession # noqa: E402
|
||
from mcp.client.stdio import StdioServerParameters, stdio_client # noqa: E402
|
||
from sqlalchemy import or_, select # noqa: E402
|
||
|
||
from app.db.session import SessionLocal # noqa: E402
|
||
from app.models import Paper, Template # noqa: E402
|
||
|
||
SERVER_SCRIPT = BACKEND_DIR / "scripts" / "mcp_server.py"
|
||
|
||
_checks = 0
|
||
_created_papers: list[int] = []
|
||
_created_templates: list[int] = []
|
||
|
||
|
||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||
"""Assert one expectation and report it."""
|
||
global _checks
|
||
_checks += 1
|
||
if not condition:
|
||
raise AssertionError(f"{label}: {detail}")
|
||
print(f" ok {label}")
|
||
|
||
|
||
def payload(result: types.CallToolResult) -> Any:
|
||
"""The JSON a tool returned, refusing an error result.
|
||
|
||
A tool that failed returns ``isError`` with the message as its content, not
|
||
an exception — so a smoke test that only looked at the content would read a
|
||
refusal as data. That is the failure mode this helper exists to prevent.
|
||
"""
|
||
text = "".join(
|
||
block.text for block in result.content if isinstance(block, types.TextContent)
|
||
)
|
||
if result.is_error:
|
||
raise AssertionError(f"tool returned an error: {text}")
|
||
return json.loads(text) if text else None
|
||
|
||
|
||
async def call(session: ClientSession, tool: str, **arguments: Any) -> Any:
|
||
"""Call one tool and return its JSON payload.
|
||
|
||
The parameter is ``tool``, not ``name``: several tools here take an
|
||
argument called ``name`` (``template_create``, ``field_create``) and a
|
||
helper whose own parameter shadowed it would fail with a TypeError that
|
||
looks nothing like the tool call it came from.
|
||
"""
|
||
return payload(await session.call_tool(tool, arguments))
|
||
|
||
|
||
async def expect_error(session: ClientSession, tool: str, **arguments: Any) -> str:
|
||
"""Call one tool expecting a refusal, and return the message."""
|
||
result = await session.call_tool(tool, arguments)
|
||
if not result.is_error:
|
||
raise AssertionError(f"{tool} should have refused, but returned {result.content}")
|
||
return "".join(
|
||
block.text for block in result.content if isinstance(block, types.TextContent)
|
||
)
|
||
|
||
|
||
def pick_template() -> tuple[int, list[dict[str, Any]]]:
|
||
"""The template with the most titles, and its outline as the tools see it."""
|
||
with SessionLocal() as db:
|
||
template = db.scalars(
|
||
select(Template).order_by(Template.id.asc())
|
||
).all()
|
||
best = max(template, key=lambda item: len(item.placements))
|
||
return best.id, [
|
||
{"position": item.sort, "name": item.title.name}
|
||
for item in best.placements
|
||
]
|
||
|
||
|
||
@asynccontextmanager
|
||
async def connect(url: str | None, token: str | None) -> AsyncIterator[ClientSession]:
|
||
"""Open a session over stdio (default) or against a running HTTP server.
|
||
|
||
Both paths yield the same :class:`ClientSession`, so every check below runs
|
||
unchanged on either transport — which is the point of testing the protocol
|
||
rather than the functions: the tools must behave identically whether a
|
||
client spawned them or connected to them.
|
||
"""
|
||
if url:
|
||
# headers live on the HTTP client, not on the transport call.
|
||
import httpx2
|
||
from mcp.client.streamable_http import streamable_http_client
|
||
|
||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||
async with httpx2.AsyncClient(headers=headers) as http_client:
|
||
async with streamable_http_client(url, http_client=http_client) as (
|
||
read,
|
||
write,
|
||
):
|
||
async with ClientSession(read, write) as session:
|
||
yield session
|
||
return
|
||
|
||
parameters = StdioServerParameters(
|
||
command=sys.executable,
|
||
args=[str(SERVER_SCRIPT)],
|
||
cwd=str(BACKEND_DIR),
|
||
)
|
||
async with stdio_client(parameters) as (read, write):
|
||
async with ClientSession(read, write) as session:
|
||
yield session
|
||
|
||
|
||
async def run(url: str | None = None, token: str | None = None) -> int:
|
||
template_id, outline = pick_template()
|
||
check("a template with fields exists", len(outline) >= 3, f"got {len(outline)}")
|
||
first = outline[0]["name"]
|
||
second = outline[1]["name"]
|
||
|
||
async with connect(url, token) as session:
|
||
handshake = await session.initialize()
|
||
check(
|
||
"server announces itself",
|
||
handshake.server_info.name == "paper-doc",
|
||
str(handshake.server_info),
|
||
)
|
||
|
||
print("\n1. tool discovery")
|
||
listed = await session.list_tools()
|
||
names = {tool.name for tool in listed.tools}
|
||
check("29 tools are published", len(names) == 29, f"got {len(names)}")
|
||
check(
|
||
"every group is reachable",
|
||
{"paper_write", "paragraph_write", "sentence_add", "template_get", "field_list"}
|
||
<= names,
|
||
)
|
||
check(
|
||
"no tool returns a structured envelope",
|
||
all(tool.output_schema is None for tool in listed.tools),
|
||
)
|
||
write_tool = next(tool for tool in listed.tools if tool.name == "paper_write")
|
||
check(
|
||
"paper_write describes its paragraph array",
|
||
"paragraphs" in write_tool.input_schema["properties"],
|
||
)
|
||
|
||
print("\n2. create a paper on that template")
|
||
paper = await call(
|
||
session,
|
||
"paper_create",
|
||
title="SMOKE MCP 论文",
|
||
template_id=template_id,
|
||
author="smoke",
|
||
keywords="测试, smoke, 测试",
|
||
)
|
||
_created_papers.append(paper["id"])
|
||
check("created", paper["title"] == "SMOKE MCP 论文")
|
||
check("keywords are deduplicated", paper["keywords"] == "测试, smoke", paper["keywords"])
|
||
check(
|
||
"the whole structure exists with no content",
|
||
paper["paragraphs"] == 0 and paper["template_paragraphs"] == len(outline),
|
||
json.dumps(paper, ensure_ascii=False),
|
||
)
|
||
|
||
print("\n3. read the outline before writing")
|
||
skeleton = await call(session, "paper_outline", paper_id=paper["id"])
|
||
check(
|
||
"the outline carries every position",
|
||
len(skeleton["paragraphs"]) == len(outline),
|
||
json.dumps(skeleton, ensure_ascii=False),
|
||
)
|
||
check("nothing has been written", all(item["sentences"] == 0 for item in skeleton["paragraphs"]))
|
||
|
||
print("\n4. write paragraphs by heading")
|
||
written = await call(
|
||
session,
|
||
"paper_write",
|
||
paper_id=paper["id"],
|
||
paragraphs=[
|
||
{
|
||
"heading": first,
|
||
"sentences": ["第一句。", "第二句。"],
|
||
},
|
||
{
|
||
"heading": second.lstrip("0123456789. "),
|
||
"text": "这一整段只有一行,按句号切成两句。第二句在这里。",
|
||
"split": "sentence",
|
||
},
|
||
],
|
||
)
|
||
check("two paragraphs written", len(written["written"]) == 2, json.dumps(written, ensure_ascii=False))
|
||
check("four sentences total", written["total_sentences"] == 4, str(written["total_sentences"]))
|
||
check(
|
||
"both headings resolved to template positions",
|
||
all(item["heading"] for item in written["written"]),
|
||
json.dumps(written, ensure_ascii=False),
|
||
)
|
||
|
||
print("\n5. a heading the template does not have is refused, with candidates")
|
||
message = await expect_error(
|
||
session,
|
||
"paragraph_write",
|
||
paper_id=paper["id"],
|
||
heading="不存在的标题",
|
||
sentences=["x"],
|
||
)
|
||
check("the refusal lists the real headings", first in message, message)
|
||
|
||
print("\n6. read one paragraph back")
|
||
paragraph = await call(
|
||
session, "paragraph_get", paper_id=paper["id"], heading=first
|
||
)
|
||
check("two sentences stored", len(paragraph["sentences"]) == 2, json.dumps(paragraph, ensure_ascii=False))
|
||
check(
|
||
"CJK seams take no separator",
|
||
all(row["separator_before"] == "" for row in paragraph["sentences"]),
|
||
json.dumps(paragraph["sentences"], ensure_ascii=False),
|
||
)
|
||
|
||
print("\n7. append a sentence with a citation")
|
||
added = await call(
|
||
session,
|
||
"sentence_add",
|
||
paper_id=paper["id"],
|
||
heading=first,
|
||
content="A cited sentence.",
|
||
citations=[{"quote": "引用的原话", "reference_id": 7}],
|
||
)
|
||
check("appended", added["sentence"]["content"] == "A cited sentence.")
|
||
check("the citation is stored", added["sentence"]["citations"][0]["quote"] == "引用的原话")
|
||
|
||
after = await call(session, "paragraph_get", paper_id=paper["id"], heading=first)
|
||
check(
|
||
"the seam before it takes a space",
|
||
after["sentences"][-1]["separator_before"] == " ",
|
||
json.dumps(after["sentences"][-1], ensure_ascii=False),
|
||
)
|
||
|
||
print("\n8. edit one sentence without touching its paragraph")
|
||
edited = await call(
|
||
session,
|
||
"sentence_update",
|
||
paper_id=paper["id"],
|
||
sentence_id=after["sentences"][0]["id"],
|
||
content="改过的第一句。",
|
||
)
|
||
check("edited", edited["sentence"]["content"] == "改过的第一句。")
|
||
still = await call(session, "paragraph_get", paper_id=paper["id"], heading=first)
|
||
check("the other sentences kept their ids", still["sentences"][1]["id"] == after["sentences"][1]["id"])
|
||
|
||
print("\n9. delete one sentence")
|
||
before = await call(session, "sentence_list", paper_id=paper["id"])
|
||
await call(
|
||
session,
|
||
"sentence_delete",
|
||
paper_id=paper["id"],
|
||
sentence_id=still["sentences"][-1]["id"],
|
||
)
|
||
remaining = await call(session, "sentence_list", paper_id=paper["id"])
|
||
check(
|
||
"exactly one sentence fewer",
|
||
remaining["total"] == before["total"] - 1,
|
||
f"{before['total']} -> {remaining['total']}",
|
||
)
|
||
|
||
print("\n10. write a whole generated document in one call")
|
||
generated = "\n".join(
|
||
[f"# {item['name']}" + "\n" + f"{item['name']}的正文写在标题下面。这句话是第二句。"
|
||
for item in outline]
|
||
)
|
||
whole = await call(
|
||
session, "paper_write_text", paper_id=paper["id"], text=generated, split="sentence"
|
||
)
|
||
check(
|
||
"every heading in the document was written",
|
||
len(whole["written"]) == len(outline),
|
||
json.dumps(whole, ensure_ascii=False),
|
||
)
|
||
check("nothing was left unmatched", "unmatched" not in whole, json.dumps(whole, ensure_ascii=False))
|
||
|
||
print("\n11. read the paper as a document")
|
||
document = await call(session, "paper_document", paper_id=paper["id"])
|
||
check("the text carries every heading", all(item["name"] in document["text"] for item in outline))
|
||
check("and every paragraph", document["text"].count("正文写在标题下面") == len(outline))
|
||
structured = await call(
|
||
session, "paper_document", paper_id=paper["id"], format="json"
|
||
)
|
||
check(
|
||
"the json form carries sentence ids",
|
||
all(
|
||
row.get("id") is not None
|
||
for paragraph in structured["paragraphs"]
|
||
for row in paragraph["sentences"]
|
||
),
|
||
)
|
||
|
||
print("\n12. search across papers")
|
||
found = await call(session, "sentence_search", keyword="正文写在标题下面", paper_id=paper["id"])
|
||
check("the search finds what was written", found["total"] >= 1, str(found["total"]))
|
||
|
||
print("\n13. move a paragraph, then clear the one it moved onto")
|
||
home = outline[0]
|
||
target = outline[1]
|
||
relocation = await call(
|
||
session,
|
||
"paragraph_move",
|
||
paper_id=paper["id"],
|
||
heading=home["name"],
|
||
target_heading=target["name"],
|
||
)
|
||
check(
|
||
"the move reports both ends",
|
||
relocation["from"]["position"] == home["position"]
|
||
and relocation["to"]["position"] == target["position"],
|
||
json.dumps(relocation, ensure_ascii=False),
|
||
)
|
||
emptied = await call(session, "paragraph_get", paper_id=paper["id"], position=home["position"])
|
||
check("the source paragraph is empty now", emptied["sentences"] == [], json.dumps(emptied, ensure_ascii=False))
|
||
merged = await call(session, "paragraph_get", paper_id=paper["id"], position=target["position"])
|
||
check(
|
||
"the target holds both paragraphs' sentences",
|
||
len(merged["sentences"]) == 4,
|
||
json.dumps(merged["sentences"], ensure_ascii=False),
|
||
)
|
||
|
||
cleared = await call(
|
||
session, "paragraph_delete", paper_id=paper["id"], position=target["position"]
|
||
)
|
||
check("clearing reports how much it removed", cleared["deleted_sentences"] == 4, json.dumps(cleared, ensure_ascii=False))
|
||
after_clear = await call(session, "paragraph_get", paper_id=paper["id"], position=target["position"])
|
||
check("and the paragraph is empty", after_clear["sentences"] == [], json.dumps(after_clear, ensure_ascii=False))
|
||
|
||
print("\n14. the field library round-trips")
|
||
field = await call(session, "field_create", name="SMOKE MCP 字段", level=2, font_size=9.5)
|
||
check("created with its typography", field["level"] == 2 and field["font_size"] == 9.5, json.dumps(field, ensure_ascii=False))
|
||
renamed = await call(session, "field_update", field_id=field["id"], name="SMOKE MCP 字段(改)")
|
||
check("renamed", renamed["name"] == "SMOKE MCP 字段(改)", json.dumps(renamed, ensure_ascii=False))
|
||
dropped = await call(session, "field_delete", field_ids=field["id"])
|
||
check("deleted while no template places it", dropped["deleted"] == 1, json.dumps(dropped, ensure_ascii=False))
|
||
check("and it is reported as found, not missing", dropped["missing"] == [], json.dumps(dropped, ensure_ascii=False))
|
||
|
||
print("\n15. switch the template and keep the content")
|
||
spare = await call(
|
||
session,
|
||
"template_create",
|
||
name="SMOKE MCP 模板",
|
||
abstract="临时模板",
|
||
field_ids=[_field_id(outline[0]["name"])],
|
||
)
|
||
_created_templates.append(spare["id"])
|
||
check("a template can be built from field ids", spare["fields"][0]["position"] == 10, json.dumps(spare, ensure_ascii=False))
|
||
|
||
rebuilt = await call(
|
||
session,
|
||
"template_update",
|
||
template_id=spare["id"],
|
||
name="SMOKE MCP 模板",
|
||
field_ids=[_field_id(outline[0]["name"]), _field_id(outline[1]["name"])],
|
||
)
|
||
check(
|
||
"the outline is replaced wholesale and renumbered",
|
||
[item["position"] for item in rebuilt["fields"]] == [10, 20],
|
||
json.dumps(rebuilt, ensure_ascii=False),
|
||
)
|
||
|
||
sentences_before = (await call(session, "sentence_list", paper_id=paper["id"]))["total"]
|
||
switched = await call(
|
||
session, "paper_update", paper_id=paper["id"], template_id=spare["id"]
|
||
)
|
||
check("the switch is reported", switched["template_changed"] is True)
|
||
switched_document = await call(
|
||
session, "paper_document", paper_id=paper["id"], format="json"
|
||
)
|
||
check(
|
||
"content survives the switch, under 未设定",
|
||
sum(len(row["sentences"]) for row in switched_document["paragraphs"])
|
||
== sentences_before,
|
||
json.dumps(switched_document, ensure_ascii=False)[:400],
|
||
)
|
||
check(
|
||
"positions the new template does not define are marked unmatched",
|
||
any(row["matched"] is False for row in switched_document["paragraphs"]),
|
||
json.dumps(switched_document, ensure_ascii=False)[:400],
|
||
)
|
||
|
||
print("\n16. a template in use cannot be deleted")
|
||
message = await expect_error(session, "template_delete", template_ids=spare["id"])
|
||
check("the refusal names the blocker", "SMOKE MCP 模板" in message, message)
|
||
|
||
print("\n17. deleting a paper needs an explicit confirmation")
|
||
message = await expect_error(session, "paper_delete", paper_ids=paper["id"])
|
||
check("the refusal explains itself", "confirm=true" in message, message)
|
||
|
||
print("\n18. delete the paper and the template")
|
||
deleted = await call(session, "paper_delete", paper_ids=paper["id"], confirm=True)
|
||
check("deleted", deleted["deleted"] == 1, json.dumps(deleted, ensure_ascii=False))
|
||
_created_papers.remove(paper["id"])
|
||
gone = await call(session, "template_delete", template_ids=spare["id"])
|
||
check("the template is free to go now", gone["deleted"] == 1, json.dumps(gone, ensure_ascii=False))
|
||
_created_templates.remove(spare["id"])
|
||
|
||
print(f"\nall {_checks} checks passed")
|
||
return 0
|
||
|
||
|
||
def _field_id(name: str) -> int:
|
||
"""The library field behind a heading name, for the template-create step."""
|
||
from app.models import TemplateTitle
|
||
|
||
with SessionLocal() as db:
|
||
row = db.scalars(
|
||
select(TemplateTitle).where(TemplateTitle.name == name)
|
||
).first()
|
||
if row is None:
|
||
raise AssertionError(f"field {name!r} not in the library")
|
||
return row.id
|
||
|
||
|
||
def cleanup() -> None:
|
||
"""Remove anything this run created, even after a failed expectation."""
|
||
if _created_papers:
|
||
with SessionLocal() as db:
|
||
for paper in db.scalars(select(Paper).where(Paper.id.in_(_created_papers))).all():
|
||
db.delete(paper)
|
||
db.commit()
|
||
print(f"cleaned up papers {_created_papers}")
|
||
if _created_templates:
|
||
with SessionLocal() as db:
|
||
for template in db.scalars(
|
||
select(Template).where(Template.id.in_(_created_templates))
|
||
).all():
|
||
db.delete(template)
|
||
db.commit()
|
||
print(f"cleaned up templates {_created_templates}")
|
||
|
||
|
||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||
"""``--url`` switches the suite from spawning a server to dialling one."""
|
||
parser = argparse.ArgumentParser(description="paper-doc MCP 冒烟测试")
|
||
parser.add_argument(
|
||
"--url",
|
||
default=os.getenv("MCP_SMOKE_URL"),
|
||
help="连到已在运行的 streamable-http 服务,例如 http://127.0.0.1:8931/mcp;"
|
||
"不传则自己拉起 stdio 子进程",
|
||
)
|
||
parser.add_argument(
|
||
"--token",
|
||
default=os.getenv("MCP_HTTP_TOKEN"),
|
||
help="HTTP 传输的 Bearer 口令",
|
||
)
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = parse_args(argv)
|
||
|
||
# The smoke test writes real rows, so it removes its own first: a previous
|
||
# run that died before cleanup would otherwise leave a duplicate title.
|
||
with SessionLocal() as db:
|
||
stale = db.scalars(
|
||
select(Paper).where(or_(Paper.title == "SMOKE MCP 论文"))
|
||
).all()
|
||
for paper in stale:
|
||
db.delete(paper)
|
||
stale_templates = db.scalars(
|
||
select(Template).where(Template.name == "SMOKE MCP 模板")
|
||
).all()
|
||
for template in stale_templates:
|
||
db.delete(template)
|
||
db.commit()
|
||
|
||
try:
|
||
return anyio.run(run, args.url, args.token)
|
||
except AssertionError as error:
|
||
print(f"\nFAILED: {error}", file=sys.stderr)
|
||
return 1
|
||
finally:
|
||
cleanup()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|