backend: add the reference library (参考文献)

One table per document the writer has read, and one for its contributors:

    reference          id, type, title, subtitle, title_translated,
                       container_title, volume, issue, pages, publisher,
                       publisher_place, event_*, publication_year/month/day,
                       accessed_date, doi, isbn, issn, url, patent_*,
                       standard_number, report_number, arxiv_id, abstract,
                       keywords, note, citation_key, language
    reference_author   reference_id, sort, name, family_name, given_name, role

The field set is the intersection of the formats a paper has to survive, taken
from the CSL JSON schema (103 fields, 45 types — what Zotero, Mendeley and
Pandoc exchange) against GB/T 7714-2015, BibTeX and RIS. The mapping is on
`app.models.reference` rather than in a commit message, because the next person
to add a column needs it more than this one does. Three decisions worth naming:

* **Partial dates.** Year, month and day are separate nullable integers, which
  is CSL's `issued`: a book has a year, a newspaper article a day, a patent its
  公告日期, and nothing invents the rest.
* **Contributors are rows, not a column.** A citation prints an ordered list
  with 编 / 译 marks, and styles rewrite names (「张三」 / "Zhang, S." /
  "S. Zhang"), so the printed form and its two halves are stored separately —
  display reads `name`, a reformatting style reads the halves, and an
  institutional author has neither.
* **`volume` / `issue` / `pages` are strings.** Journals print `12(Suppl.)`,
  `S1` and `e12345`, and a citation reproduces what is printed.

The library is global: the same article is cited from several papers, so the
entry cannot belong to one of them. The citation stays on the paper side in
`paper_sentence_reference`, which now carries a real foreign key to `reference`
(`ON DELETE RESTRICT`) — the column had been a loose integer waiting for this
table. A sentence may quote several documents and a document may be quoted by
many sentences, which is why the relation is a table and not a column on
`paper_sentence`.

Refusals are raised once, in the data layer, as `UnknownReference`, and
translated where each caller speaks: `400` with the offending ids for REST (one
exception handler, no route carries the check), a readable tool error for MCP
(one wrapper in the tool registry). Neither entry point can skip it, so the
foreign key never has to be the thing that complains.

API: `/references` CRUD plus batch delete, and `/references/{id}/citations` for
the reverse lookup — "where have I used this article?" — which is what tells a
writer whether an entry is safe to delete.

Verified: 29 new checks in `scripts/smoke_references.py` (partial dates, DOI
normalisation, author order and roles, search by author/type/year/cited, the
citation round trip, 400 on a phantom id, 409 on a cited entry), the paper
suite's 45 and the MCP suite's 47 still pass. The MCP suite now cites a real
entry instead of a phantom id, which is what an agent has to do now.
This commit is contained in:
2026-09-22 17:13:32 +08:00
parent 103b5544ba
commit d7bb777264
18 changed files with 1966 additions and 16 deletions
+50 -4
View File
@@ -26,8 +26,8 @@ import argparse
import json
import os
import sys
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from collections.abc import AsyncIterator, Iterator
from contextlib import asynccontextmanager, contextmanager
from pathlib import Path
from typing import Any
@@ -43,7 +43,7 @@ from mcp.client.stdio import StdioServerParameters, stdio_client # noqa: E402
from sqlalchemy import or_, select # noqa: E402
from app.db.session import SessionLocal # noqa: E402
from app.models import Paper, Template # noqa: E402
from app.models import Paper, Reference, Template # noqa: E402
SERVER_SCRIPT = BACKEND_DIR / "scripts" / "mcp_server.py"
@@ -97,6 +97,15 @@ async def expect_error(session: ClientSession, tool: str, **arguments: Any) -> s
)
class _Box:
"""A one-slot holder, so the citation helper can read what the context set."""
value: int | None = None
_CITATION_REFERENCE_ID = _Box()
def pick_template() -> tuple[int, list[dict[str, Any]]]:
"""The template with the most titles, and its outline as the tools see it."""
with SessionLocal() as db:
@@ -145,6 +154,11 @@ async def connect(url: str | None, token: str | None) -> AsyncIterator[ClientSes
async def run(url: str | None = None, token: str | None = None) -> int:
with reference_for_citation():
return await _run(url, token)
async def _run(url: str | None = None, token: str | None = None) -> int:
template_id, outline = pick_template()
check("a template with fields exists", len(outline) >= 3, f"got {len(outline)}")
first = outline[0]["name"]
@@ -257,7 +271,7 @@ async def run(url: str | None = None, token: str | None = None) -> int:
paper_id=paper["id"],
heading=first,
content="A cited sentence.",
citations=[{"quote": "引用的原话", "reference_id": 7}],
citations=[{"quote": "引用的原话", "reference_id": _reference_id()}],
)
check("appended", added["sentence"]["content"] == "A cited sentence.")
check("the citation is stored", added["sentence"]["citations"][0]["quote"] == "引用的原话")
@@ -436,6 +450,38 @@ async def run(url: str | None = None, token: str | None = None) -> int:
return 0
@contextmanager
def reference_for_citation() -> Iterator[int]:
"""A library entry the citation checks can point at.
A citation names real literature now, so the test has to have some: the MCP
surface has no reference tools yet, and this creates the row directly.
"""
with SessionLocal() as db:
reference = Reference(type="journal-article", title="SMOKE MCP 文献")
db.add(reference)
db.commit()
db.refresh(reference)
reference_id = reference.id
_CITATION_REFERENCE_ID.value = reference_id
try:
yield reference_id
finally:
with SessionLocal() as db:
stored = db.get(Reference, reference_id)
if stored is not None:
db.delete(stored)
db.commit()
_CITATION_REFERENCE_ID.value = None
def _reference_id() -> int:
"""The id the current citation check should use."""
if _CITATION_REFERENCE_ID.value is None:
raise AssertionError("no reference in scope for this citation check")
return _CITATION_REFERENCE_ID.value
def _field_id(name: str) -> int:
"""The library field behind a heading name, for the template-create step."""
from app.models import TemplateTitle
+38 -3
View File
@@ -63,6 +63,7 @@ def check(label: str, condition: bool, detail: str = "") -> None:
def main() -> int:
created_papers: list[int] = []
created_references: list[int] = []
try:
# The database may already hold real papers, so "nothing of ours is
@@ -117,6 +118,21 @@ def main() -> int:
first_sort = document["paragraphs"][0]["paper_template_filed_sort"]
print("4. write a paragraph: two sentences, citations on both")
# A citation names real literature now, so the library needs an entry
# first — and a citation with no id at all stays legal.
reference = call(
"POST",
"/references",
{
"type": "journal-article",
"title": "用于验证引用链路的文献",
"container_title": "测试期刊",
"publication_year": 2024,
"authors": [{"name": "测试作者"}],
},
expect=201,
)
created_references.append(reference["id"])
document = call(
"PUT",
f"/papers/{paper['id']}/paragraphs/{first_sort}",
@@ -124,12 +140,14 @@ def main() -> int:
"sentences": [
{
"content": " 第一句话,用于验证空白折叠。 ",
"citations": [{"reference_id": 7, "quote": "被引用的第一段内容"}],
"citations": [
{"reference_id": reference["id"], "quote": "被引用的第一段内容"}
],
},
{
"content": "第二句话。",
"citations": [
{"reference_id": 9, "quote": "引文甲"},
{"reference_id": reference["id"], "quote": "引文甲"},
{"quote": "只有引用内容、暂时没有引用 id"},
],
},
@@ -230,7 +248,16 @@ def main() -> int:
bad = call(
"PUT",
f"/papers/{paper['id']}/paragraphs/{first_sort}",
{"sentences": [{"content": "x", "citations": [{"reference_id": 3, "quote": " "}]}]},
{
"sentences": [
{
"content": "x",
"citations": [
{"reference_id": reference["id"], "quote": " "}
],
}
]
},
expect=422,
)
check("citation without content refused", "detail" in bad)
@@ -361,6 +388,9 @@ def main() -> int:
print("15. delete the paper")
call("DELETE", f"/papers/{paper['id']}", expect=204)
created_papers.clear()
for reference_id in created_references:
call("DELETE", f"/references/{reference_id}", expect=204)
created_references.clear()
check(
"gone from the list",
call("GET", "/papers")["total"] == baseline,
@@ -371,6 +401,11 @@ def main() -> int:
return 0
finally:
# Never leave test rows behind, even on a failed expectation.
for reference_id in created_references:
try:
call("DELETE", f"/references/{reference_id}", expect=204)
except Exception as error: # noqa: BLE001 - cleanup must not mask the failure
print(f"cleanup of reference {reference_id} failed: {error}", file=sys.stderr)
for paper_id in created_papers:
try:
call("DELETE", f"/papers/{paper_id}", expect=204)
+260
View File
@@ -0,0 +1,260 @@
"""End-to-end check of the reference library, against a running API.
Run it while the backend is up::
.venv/bin/python scripts/smoke_references.py
It walks the whole loop the library exists for — add literature, find it again
by author and by type, link a sentence to it, look the citation up from the
reference side, be refused when deleting something in use, then clean up — and
exits non-zero on the first failed expectation. Everything it creates is
deleted, including the throwaway paper it cites from.
"""
from __future__ import annotations
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
BASE = "http://127.0.0.1:8000/api"
_checks = 0
def call(method: str, path: str, body: Any = None, expect: int = 200) -> Any:
"""Perform one API call and assert its status code."""
data = json.dumps(body).encode() if body is not None else None
url = BASE + urllib.parse.quote(path, safe="/?&=%")
request = urllib.request.Request(
url,
data=data,
method=method,
headers={"Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(request, timeout=20) as response:
status = response.status
payload = response.read().decode()
except urllib.error.HTTPError as error:
status = error.code
payload = error.read().decode()
if status != expect:
raise AssertionError(f"{method} {path} -> {status}, expected {expect}: {payload}")
return json.loads(payload) if payload else None
def check(label: str, condition: bool, detail: str = "") -> None:
"""Assert one expectation and report it."""
global _checks
_checks += 1
if not condition:
raise AssertionError(f"{label}: {detail}")
print(f" ok {label}")
def main() -> int:
created_references: list[int] = []
created_papers: list[int] = []
try:
print("1. add an article with four authors")
article = call(
"POST",
"/references",
{
"type": "journal-article",
"title": "技术惯性与冲量的耦合机制",
"subtitle": "基于科学与技术融合网络的实证分析",
"title_translated": "Coupling inertia and impulse in technology evolution",
"container_title": "Technological Forecasting and Social Change",
"container_title_short": "Technol. Forecast. Soc. Change",
"volume": "198",
"issue": "3",
"pages": "122-141",
"publication_year": 2024,
"publication_month": 5,
"doi": "https://doi.org/10.1016/j.techfore.2024.123456",
"language": "zh",
"keywords": "技术演化, 惯性, 冲量",
"authors": [
{"name": "张三", "family_name": "张", "given_name": "三"},
{"name": "李四"},
{"name": "Wang Wu", "family_name": "Wang", "given_name": "Wu"},
{"name": "世界卫生组织", "role": "compiler"},
],
},
expect=201,
)
created_references.append(article["id"])
check("the DOI lost its URL prefix", article["doi"] == "10.1016/j.techfore.2024.123456", article["doi"])
check("four contributors stored", len(article["authors"]) == 4)
check(
"they keep the order they were sent in",
[a["name"] for a in article["authors"]] == ["张三", "李四", "Wang Wu", "世界卫生组织"],
str([a["name"] for a in article["authors"]]),
)
check(
"sort is numbered from 1",
[a["sort"] for a in article["authors"]] == [1, 2, 3, 4],
str([a["sort"] for a in article["authors"]]),
)
check("roles survive", article["authors"][3]["role"] == "compiler")
check("partial date kept as sent", article["publication_year"] == 2024 and article["publication_month"] == 5)
check("no day invented", article["publication_day"] is None)
check("not cited yet", article["citation_count"] == 0)
print("2. add a patent, which needs a country and a number")
patent = call(
"POST",
"/references",
{
"type": "patent",
"title": "一种基于多层聚类的人工智能技术识别方法",
"patent_country": "中国",
"patent_number": "CN114000000A",
"publication_year": 2023,
"authors": [{"name": "张三"}],
},
expect=201,
)
created_references.append(patent["id"])
check("patent stored", patent["type"] == "patent" and patent["patent_number"] == "CN114000000A")
print("3. a webpage without a URL is refused")
call(
"POST",
"/references",
{"type": "web-page", "title": "某网页"},
expect=422,
)
check("web page without URL refused", True)
print("4. a bad type, a phantom citation target and a missing title are refused")
call("POST", "/references", {"type": "blog-post", "title": "x"}, expect=422)
call("POST", "/references", {"title": " "}, expect=422)
call(
"POST",
"/references",
{"title": "引用了不存在的文献", "authors": [{"name": "甲"}], "type": "book"},
expect=201,
)
probe = call("GET", "/references?keyword=引用了不存在")
created_references.append(probe["items"][0]["id"])
check("a book needs nothing else", probe["total"] == 1)
print("5. find it again: by author, by type, by year")
by_author = call("GET", "/references?keyword=Wang")
check("search matches an author name", by_author["total"] == 1, str(by_author["total"]))
check(
"the row prints an author line",
by_author["items"][0]["authors_line"].startswith("张三, 李四, Wang Wu"),
by_author["items"][0]["authors_line"],
)
by_type = call("GET", "/references?type=patent")
check("filter by type", by_type["total"] == 1, str(by_type["total"]))
by_year = call("GET", "/references?year=2024")
check("filter by year", by_year["total"] == 1, str(by_year["total"]))
uncited = call("GET", "/references?cited=false")
check("filter by not-yet-cited", uncited["total"] == len(created_references))
print("6. edit it: replace the contributor list, change a field")
updated = call(
"PATCH",
f"/references/{article['id']}",
{
"pages": "122-145",
"authors": [
{"name": "张三", "family_name": "张", "given_name": "三"},
{"name": "李四"},
],
},
)
check("pages updated", updated["pages"] == "122-145")
check("author list replaced wholesale", len(updated["authors"]) == 2)
check("untouched fields kept", updated["container_title"] == "Technological Forecasting and Social Change")
print("7. cite it from a sentence")
templates = call("GET", "/templates?page_size=1")["items"]
paper = call(
"POST",
"/papers",
{"title": "参考文献冒烟测试论文", "template_id": templates[0]["id"]},
expect=201,
)
created_papers.append(paper["id"])
outline = call("GET", f"/templates/{templates[0]['id']}")
position = outline["placements"][0]["sort"]
call(
"PUT",
f"/papers/{paper['id']}/paragraphs/{position}",
{
"sentences": [
{
"content": "技术惯性约束了同时具有高冲量的可能。",
"citations": [
{"reference_id": article["id"], "quote": "高惯性技术几乎不可能同时具有高冲量。"}
],
}
]
},
)
check("a citation to a real reference is accepted", True)
missing = call(
"PUT",
f"/papers/{paper['id']}/paragraphs/{position}",
{"sentences": [{"content": "x", "citations": [{"reference_id": 999999, "quote": "y"}]}]},
expect=400,
)
check("a citation to nothing is refused by name", "999999" in missing["detail"], missing["detail"])
print("8. the reverse lookup: which sentences use this article?")
citations = call("GET", f"/references/{article['id']}/citations")
check("one citation found", len(citations) == 1, str(len(citations)))
check("it names the sentence", "技术惯性约束" in citations[0]["sentence_content"])
check("and the paper", citations[0]["paper_id"] == paper["id"])
check("and the paragraph heading", citations[0]["paragraph_name"] is not None, str(citations[0]))
cited = call("GET", "/references?cited=true")
check("the cited filter now finds it", cited["total"] == 1, str(cited["total"]))
print("9. a cited reference cannot be deleted; an uncited one can")
conflict = call("DELETE", f"/references/{article['id']}", expect=409)
check("refusal explains itself", "正被论文引用" in conflict["detail"], conflict["detail"])
call("DELETE", f"/references/{patent['id']}", expect=204)
created_references.remove(patent["id"])
check("an uncited reference deletes cleanly", True)
print("10. clean up")
call("DELETE", f"/papers/{paper['id']}", expect=204)
created_papers.clear()
for reference_id in created_references:
call("DELETE", f"/references/{reference_id}", expect=204)
created_references.clear()
remaining = call("GET", "/references")
check("nothing of ours is left", remaining["total"] == 0, str(remaining["total"]))
print(f"\nall {_checks} checks passed")
return 0
finally:
for paper_id in created_papers:
try:
call("DELETE", f"/papers/{paper_id}", expect=204)
except Exception as error: # noqa: BLE001 - cleanup must not mask the failure
print(f"cleanup of paper {paper_id} failed: {error}", file=sys.stderr)
for reference_id in created_references:
try:
call("DELETE", f"/references/{reference_id}", expect=204)
print(f"cleaned up reference {reference_id}")
except Exception as error: # noqa: BLE001
print(f"cleanup of reference {reference_id} failed: {error}", file=sys.stderr)
if __name__ == "__main__":
sys.exit(main())