backend: add the reference library (参考文献)
One table per document the writer has read, and one for its contributors:
reference id, type, title, subtitle, title_translated,
container_title, volume, issue, pages, publisher,
publisher_place, event_*, publication_year/month/day,
accessed_date, doi, isbn, issn, url, patent_*,
standard_number, report_number, arxiv_id, abstract,
keywords, note, citation_key, language
reference_author reference_id, sort, name, family_name, given_name, role
The field set is the intersection of the formats a paper has to survive, taken
from the CSL JSON schema (103 fields, 45 types — what Zotero, Mendeley and
Pandoc exchange) against GB/T 7714-2015, BibTeX and RIS. The mapping is on
`app.models.reference` rather than in a commit message, because the next person
to add a column needs it more than this one does. Three decisions worth naming:
* **Partial dates.** Year, month and day are separate nullable integers, which
is CSL's `issued`: a book has a year, a newspaper article a day, a patent its
公告日期, and nothing invents the rest.
* **Contributors are rows, not a column.** A citation prints an ordered list
with 编 / 译 marks, and styles rewrite names (「张三」 / "Zhang, S." /
"S. Zhang"), so the printed form and its two halves are stored separately —
display reads `name`, a reformatting style reads the halves, and an
institutional author has neither.
* **`volume` / `issue` / `pages` are strings.** Journals print `12(Suppl.)`,
`S1` and `e12345`, and a citation reproduces what is printed.
The library is global: the same article is cited from several papers, so the
entry cannot belong to one of them. The citation stays on the paper side in
`paper_sentence_reference`, which now carries a real foreign key to `reference`
(`ON DELETE RESTRICT`) — the column had been a loose integer waiting for this
table. A sentence may quote several documents and a document may be quoted by
many sentences, which is why the relation is a table and not a column on
`paper_sentence`.
Refusals are raised once, in the data layer, as `UnknownReference`, and
translated where each caller speaks: `400` with the offending ids for REST (one
exception handler, no route carries the check), a readable tool error for MCP
(one wrapper in the tool registry). Neither entry point can skip it, so the
foreign key never has to be the thing that complains.
API: `/references` CRUD plus batch delete, and `/references/{id}/citations` for
the reverse lookup — "where have I used this article?" — which is what tells a
writer whether an entry is safe to delete.
Verified: 29 new checks in `scripts/smoke_references.py` (partial dates, DOI
normalisation, author order and roles, search by author/type/year/cited, the
citation round trip, 400 on a phantom id, 409 on a cited entry), the paper
suite's 45 and the MCP suite's 47 still pass. The MCP suite now cites a real
entry instead of a phantom id, which is what an agent has to do now.
This commit is contained in:
@@ -26,8 +26,8 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from contextlib import asynccontextmanager, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -43,7 +43,7 @@ from mcp.client.stdio import StdioServerParameters, stdio_client # noqa: E402
|
||||
from sqlalchemy import or_, select # noqa: E402
|
||||
|
||||
from app.db.session import SessionLocal # noqa: E402
|
||||
from app.models import Paper, Template # noqa: E402
|
||||
from app.models import Paper, Reference, Template # noqa: E402
|
||||
|
||||
SERVER_SCRIPT = BACKEND_DIR / "scripts" / "mcp_server.py"
|
||||
|
||||
@@ -97,6 +97,15 @@ async def expect_error(session: ClientSession, tool: str, **arguments: Any) -> s
|
||||
)
|
||||
|
||||
|
||||
class _Box:
|
||||
"""A one-slot holder, so the citation helper can read what the context set."""
|
||||
|
||||
value: int | None = None
|
||||
|
||||
|
||||
_CITATION_REFERENCE_ID = _Box()
|
||||
|
||||
|
||||
def pick_template() -> tuple[int, list[dict[str, Any]]]:
|
||||
"""The template with the most titles, and its outline as the tools see it."""
|
||||
with SessionLocal() as db:
|
||||
@@ -145,6 +154,11 @@ async def connect(url: str | None, token: str | None) -> AsyncIterator[ClientSes
|
||||
|
||||
|
||||
async def run(url: str | None = None, token: str | None = None) -> int:
|
||||
with reference_for_citation():
|
||||
return await _run(url, token)
|
||||
|
||||
|
||||
async def _run(url: str | None = None, token: str | None = None) -> int:
|
||||
template_id, outline = pick_template()
|
||||
check("a template with fields exists", len(outline) >= 3, f"got {len(outline)}")
|
||||
first = outline[0]["name"]
|
||||
@@ -257,7 +271,7 @@ async def run(url: str | None = None, token: str | None = None) -> int:
|
||||
paper_id=paper["id"],
|
||||
heading=first,
|
||||
content="A cited sentence.",
|
||||
citations=[{"quote": "引用的原话", "reference_id": 7}],
|
||||
citations=[{"quote": "引用的原话", "reference_id": _reference_id()}],
|
||||
)
|
||||
check("appended", added["sentence"]["content"] == "A cited sentence.")
|
||||
check("the citation is stored", added["sentence"]["citations"][0]["quote"] == "引用的原话")
|
||||
@@ -436,6 +450,38 @@ async def run(url: str | None = None, token: str | None = None) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
@contextmanager
|
||||
def reference_for_citation() -> Iterator[int]:
|
||||
"""A library entry the citation checks can point at.
|
||||
|
||||
A citation names real literature now, so the test has to have some: the MCP
|
||||
surface has no reference tools yet, and this creates the row directly.
|
||||
"""
|
||||
with SessionLocal() as db:
|
||||
reference = Reference(type="journal-article", title="SMOKE MCP 文献")
|
||||
db.add(reference)
|
||||
db.commit()
|
||||
db.refresh(reference)
|
||||
reference_id = reference.id
|
||||
_CITATION_REFERENCE_ID.value = reference_id
|
||||
try:
|
||||
yield reference_id
|
||||
finally:
|
||||
with SessionLocal() as db:
|
||||
stored = db.get(Reference, reference_id)
|
||||
if stored is not None:
|
||||
db.delete(stored)
|
||||
db.commit()
|
||||
_CITATION_REFERENCE_ID.value = None
|
||||
|
||||
|
||||
def _reference_id() -> int:
|
||||
"""The id the current citation check should use."""
|
||||
if _CITATION_REFERENCE_ID.value is None:
|
||||
raise AssertionError("no reference in scope for this citation check")
|
||||
return _CITATION_REFERENCE_ID.value
|
||||
|
||||
|
||||
def _field_id(name: str) -> int:
|
||||
"""The library field behind a heading name, for the template-create step."""
|
||||
from app.models import TemplateTitle
|
||||
|
||||
@@ -63,6 +63,7 @@ def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
|
||||
def main() -> int:
|
||||
created_papers: list[int] = []
|
||||
created_references: list[int] = []
|
||||
|
||||
try:
|
||||
# The database may already hold real papers, so "nothing of ours is
|
||||
@@ -117,6 +118,21 @@ def main() -> int:
|
||||
first_sort = document["paragraphs"][0]["paper_template_filed_sort"]
|
||||
|
||||
print("4. write a paragraph: two sentences, citations on both")
|
||||
# A citation names real literature now, so the library needs an entry
|
||||
# first — and a citation with no id at all stays legal.
|
||||
reference = call(
|
||||
"POST",
|
||||
"/references",
|
||||
{
|
||||
"type": "journal-article",
|
||||
"title": "用于验证引用链路的文献",
|
||||
"container_title": "测试期刊",
|
||||
"publication_year": 2024,
|
||||
"authors": [{"name": "测试作者"}],
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
created_references.append(reference["id"])
|
||||
document = call(
|
||||
"PUT",
|
||||
f"/papers/{paper['id']}/paragraphs/{first_sort}",
|
||||
@@ -124,12 +140,14 @@ def main() -> int:
|
||||
"sentences": [
|
||||
{
|
||||
"content": " 第一句话,用于验证空白折叠。 ",
|
||||
"citations": [{"reference_id": 7, "quote": "被引用的第一段内容"}],
|
||||
"citations": [
|
||||
{"reference_id": reference["id"], "quote": "被引用的第一段内容"}
|
||||
],
|
||||
},
|
||||
{
|
||||
"content": "第二句话。",
|
||||
"citations": [
|
||||
{"reference_id": 9, "quote": "引文甲"},
|
||||
{"reference_id": reference["id"], "quote": "引文甲"},
|
||||
{"quote": "只有引用内容、暂时没有引用 id"},
|
||||
],
|
||||
},
|
||||
@@ -230,7 +248,16 @@ def main() -> int:
|
||||
bad = call(
|
||||
"PUT",
|
||||
f"/papers/{paper['id']}/paragraphs/{first_sort}",
|
||||
{"sentences": [{"content": "x", "citations": [{"reference_id": 3, "quote": " "}]}]},
|
||||
{
|
||||
"sentences": [
|
||||
{
|
||||
"content": "x",
|
||||
"citations": [
|
||||
{"reference_id": reference["id"], "quote": " "}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
expect=422,
|
||||
)
|
||||
check("citation without content refused", "detail" in bad)
|
||||
@@ -361,6 +388,9 @@ def main() -> int:
|
||||
print("15. delete the paper")
|
||||
call("DELETE", f"/papers/{paper['id']}", expect=204)
|
||||
created_papers.clear()
|
||||
for reference_id in created_references:
|
||||
call("DELETE", f"/references/{reference_id}", expect=204)
|
||||
created_references.clear()
|
||||
check(
|
||||
"gone from the list",
|
||||
call("GET", "/papers")["total"] == baseline,
|
||||
@@ -371,6 +401,11 @@ def main() -> int:
|
||||
return 0
|
||||
finally:
|
||||
# Never leave test rows behind, even on a failed expectation.
|
||||
for reference_id in created_references:
|
||||
try:
|
||||
call("DELETE", f"/references/{reference_id}", expect=204)
|
||||
except Exception as error: # noqa: BLE001 - cleanup must not mask the failure
|
||||
print(f"cleanup of reference {reference_id} failed: {error}", file=sys.stderr)
|
||||
for paper_id in created_papers:
|
||||
try:
|
||||
call("DELETE", f"/papers/{paper_id}", expect=204)
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
"""End-to-end check of the reference library, against a running API.
|
||||
|
||||
Run it while the backend is up::
|
||||
|
||||
.venv/bin/python scripts/smoke_references.py
|
||||
|
||||
It walks the whole loop the library exists for — add literature, find it again
|
||||
by author and by type, link a sentence to it, look the citation up from the
|
||||
reference side, be refused when deleting something in use, then clean up — and
|
||||
exits non-zero on the first failed expectation. Everything it creates is
|
||||
deleted, including the throwaway paper it cites from.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
BASE = "http://127.0.0.1:8000/api"
|
||||
|
||||
_checks = 0
|
||||
|
||||
|
||||
def call(method: str, path: str, body: Any = None, expect: int = 200) -> Any:
|
||||
"""Perform one API call and assert its status code."""
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
url = BASE + urllib.parse.quote(path, safe="/?&=%")
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=20) as response:
|
||||
status = response.status
|
||||
payload = response.read().decode()
|
||||
except urllib.error.HTTPError as error:
|
||||
status = error.code
|
||||
payload = error.read().decode()
|
||||
|
||||
if status != expect:
|
||||
raise AssertionError(f"{method} {path} -> {status}, expected {expect}: {payload}")
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
"""Assert one expectation and report it."""
|
||||
global _checks
|
||||
_checks += 1
|
||||
if not condition:
|
||||
raise AssertionError(f"{label}: {detail}")
|
||||
print(f" ok {label}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
created_references: list[int] = []
|
||||
created_papers: list[int] = []
|
||||
|
||||
try:
|
||||
print("1. add an article with four authors")
|
||||
article = call(
|
||||
"POST",
|
||||
"/references",
|
||||
{
|
||||
"type": "journal-article",
|
||||
"title": "技术惯性与冲量的耦合机制",
|
||||
"subtitle": "基于科学与技术融合网络的实证分析",
|
||||
"title_translated": "Coupling inertia and impulse in technology evolution",
|
||||
"container_title": "Technological Forecasting and Social Change",
|
||||
"container_title_short": "Technol. Forecast. Soc. Change",
|
||||
"volume": "198",
|
||||
"issue": "3",
|
||||
"pages": "122-141",
|
||||
"publication_year": 2024,
|
||||
"publication_month": 5,
|
||||
"doi": "https://doi.org/10.1016/j.techfore.2024.123456",
|
||||
"language": "zh",
|
||||
"keywords": "技术演化, 惯性, 冲量",
|
||||
"authors": [
|
||||
{"name": "张三", "family_name": "张", "given_name": "三"},
|
||||
{"name": "李四"},
|
||||
{"name": "Wang Wu", "family_name": "Wang", "given_name": "Wu"},
|
||||
{"name": "世界卫生组织", "role": "compiler"},
|
||||
],
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
created_references.append(article["id"])
|
||||
check("the DOI lost its URL prefix", article["doi"] == "10.1016/j.techfore.2024.123456", article["doi"])
|
||||
check("four contributors stored", len(article["authors"]) == 4)
|
||||
check(
|
||||
"they keep the order they were sent in",
|
||||
[a["name"] for a in article["authors"]] == ["张三", "李四", "Wang Wu", "世界卫生组织"],
|
||||
str([a["name"] for a in article["authors"]]),
|
||||
)
|
||||
check(
|
||||
"sort is numbered from 1",
|
||||
[a["sort"] for a in article["authors"]] == [1, 2, 3, 4],
|
||||
str([a["sort"] for a in article["authors"]]),
|
||||
)
|
||||
check("roles survive", article["authors"][3]["role"] == "compiler")
|
||||
check("partial date kept as sent", article["publication_year"] == 2024 and article["publication_month"] == 5)
|
||||
check("no day invented", article["publication_day"] is None)
|
||||
check("not cited yet", article["citation_count"] == 0)
|
||||
|
||||
print("2. add a patent, which needs a country and a number")
|
||||
patent = call(
|
||||
"POST",
|
||||
"/references",
|
||||
{
|
||||
"type": "patent",
|
||||
"title": "一种基于多层聚类的人工智能技术识别方法",
|
||||
"patent_country": "中国",
|
||||
"patent_number": "CN114000000A",
|
||||
"publication_year": 2023,
|
||||
"authors": [{"name": "张三"}],
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
created_references.append(patent["id"])
|
||||
check("patent stored", patent["type"] == "patent" and patent["patent_number"] == "CN114000000A")
|
||||
|
||||
print("3. a webpage without a URL is refused")
|
||||
call(
|
||||
"POST",
|
||||
"/references",
|
||||
{"type": "web-page", "title": "某网页"},
|
||||
expect=422,
|
||||
)
|
||||
check("web page without URL refused", True)
|
||||
|
||||
print("4. a bad type, a phantom citation target and a missing title are refused")
|
||||
call("POST", "/references", {"type": "blog-post", "title": "x"}, expect=422)
|
||||
call("POST", "/references", {"title": " "}, expect=422)
|
||||
call(
|
||||
"POST",
|
||||
"/references",
|
||||
{"title": "引用了不存在的文献", "authors": [{"name": "甲"}], "type": "book"},
|
||||
expect=201,
|
||||
)
|
||||
probe = call("GET", "/references?keyword=引用了不存在")
|
||||
created_references.append(probe["items"][0]["id"])
|
||||
check("a book needs nothing else", probe["total"] == 1)
|
||||
|
||||
print("5. find it again: by author, by type, by year")
|
||||
by_author = call("GET", "/references?keyword=Wang")
|
||||
check("search matches an author name", by_author["total"] == 1, str(by_author["total"]))
|
||||
check(
|
||||
"the row prints an author line",
|
||||
by_author["items"][0]["authors_line"].startswith("张三, 李四, Wang Wu"),
|
||||
by_author["items"][0]["authors_line"],
|
||||
)
|
||||
by_type = call("GET", "/references?type=patent")
|
||||
check("filter by type", by_type["total"] == 1, str(by_type["total"]))
|
||||
by_year = call("GET", "/references?year=2024")
|
||||
check("filter by year", by_year["total"] == 1, str(by_year["total"]))
|
||||
uncited = call("GET", "/references?cited=false")
|
||||
check("filter by not-yet-cited", uncited["total"] == len(created_references))
|
||||
|
||||
print("6. edit it: replace the contributor list, change a field")
|
||||
updated = call(
|
||||
"PATCH",
|
||||
f"/references/{article['id']}",
|
||||
{
|
||||
"pages": "122-145",
|
||||
"authors": [
|
||||
{"name": "张三", "family_name": "张", "given_name": "三"},
|
||||
{"name": "李四"},
|
||||
],
|
||||
},
|
||||
)
|
||||
check("pages updated", updated["pages"] == "122-145")
|
||||
check("author list replaced wholesale", len(updated["authors"]) == 2)
|
||||
check("untouched fields kept", updated["container_title"] == "Technological Forecasting and Social Change")
|
||||
|
||||
print("7. cite it from a sentence")
|
||||
templates = call("GET", "/templates?page_size=1")["items"]
|
||||
paper = call(
|
||||
"POST",
|
||||
"/papers",
|
||||
{"title": "参考文献冒烟测试论文", "template_id": templates[0]["id"]},
|
||||
expect=201,
|
||||
)
|
||||
created_papers.append(paper["id"])
|
||||
outline = call("GET", f"/templates/{templates[0]['id']}")
|
||||
position = outline["placements"][0]["sort"]
|
||||
|
||||
call(
|
||||
"PUT",
|
||||
f"/papers/{paper['id']}/paragraphs/{position}",
|
||||
{
|
||||
"sentences": [
|
||||
{
|
||||
"content": "技术惯性约束了同时具有高冲量的可能。",
|
||||
"citations": [
|
||||
{"reference_id": article["id"], "quote": "高惯性技术几乎不可能同时具有高冲量。"}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
check("a citation to a real reference is accepted", True)
|
||||
|
||||
missing = call(
|
||||
"PUT",
|
||||
f"/papers/{paper['id']}/paragraphs/{position}",
|
||||
{"sentences": [{"content": "x", "citations": [{"reference_id": 999999, "quote": "y"}]}]},
|
||||
expect=400,
|
||||
)
|
||||
check("a citation to nothing is refused by name", "999999" in missing["detail"], missing["detail"])
|
||||
|
||||
print("8. the reverse lookup: which sentences use this article?")
|
||||
citations = call("GET", f"/references/{article['id']}/citations")
|
||||
check("one citation found", len(citations) == 1, str(len(citations)))
|
||||
check("it names the sentence", "技术惯性约束" in citations[0]["sentence_content"])
|
||||
check("and the paper", citations[0]["paper_id"] == paper["id"])
|
||||
check("and the paragraph heading", citations[0]["paragraph_name"] is not None, str(citations[0]))
|
||||
|
||||
cited = call("GET", "/references?cited=true")
|
||||
check("the cited filter now finds it", cited["total"] == 1, str(cited["total"]))
|
||||
|
||||
print("9. a cited reference cannot be deleted; an uncited one can")
|
||||
conflict = call("DELETE", f"/references/{article['id']}", expect=409)
|
||||
check("refusal explains itself", "正被论文引用" in conflict["detail"], conflict["detail"])
|
||||
call("DELETE", f"/references/{patent['id']}", expect=204)
|
||||
created_references.remove(patent["id"])
|
||||
check("an uncited reference deletes cleanly", True)
|
||||
|
||||
print("10. clean up")
|
||||
call("DELETE", f"/papers/{paper['id']}", expect=204)
|
||||
created_papers.clear()
|
||||
for reference_id in created_references:
|
||||
call("DELETE", f"/references/{reference_id}", expect=204)
|
||||
created_references.clear()
|
||||
remaining = call("GET", "/references")
|
||||
check("nothing of ours is left", remaining["total"] == 0, str(remaining["total"]))
|
||||
|
||||
print(f"\nall {_checks} checks passed")
|
||||
return 0
|
||||
finally:
|
||||
for paper_id in created_papers:
|
||||
try:
|
||||
call("DELETE", f"/papers/{paper_id}", expect=204)
|
||||
except Exception as error: # noqa: BLE001 - cleanup must not mask the failure
|
||||
print(f"cleanup of paper {paper_id} failed: {error}", file=sys.stderr)
|
||||
for reference_id in created_references:
|
||||
try:
|
||||
call("DELETE", f"/references/{reference_id}", expect=204)
|
||||
print(f"cleaned up reference {reference_id}")
|
||||
except Exception as error: # noqa: BLE001
|
||||
print(f"cleanup of reference {reference_id} failed: {error}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user