govin/paper-doc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend: add the paper authoring schema, API, and smoke test

Browse Source 
A paper is a document written against a template. Its structure is never
copied into it: `paper` points at a template and the outline is read live on
every render, so switching `template_id` re-shapes the whole document in one
write.

Sentences are addressed by *position* rather than by a template row id:
`paper_sentence.paper_template_filed_sort` holds the sort of the placement the
sentence belongs to, and `sort` holds its place inside that paragraph. That
indirection is what makes a template switch non-destructive — a sentence that
remembers "position 7" lands on whatever the new template puts at position 7 —
and it is why a paragraph is any position either the template or the content
mentions: the structure survives with no content, and content whose position
the template does not define is still rendered, in order, under 未设定.

Citations are a table rather than a column, since one sentence may quote
several references. `quote` is required — a citation that does not say what it
quotes is refused with 422 — while `reference_id` is a plain nullable integer
with no foreign key, because the reference library does not exist yet.

Deleting a template a paper is written against is refused with 409 and a count,
matching how the field library refuses to drop a field still in use.

scripts/smoke_papers.py walks the whole loop — create, empty structure, write a
paragraph with citations, switch templates, keep unmatched positions, move a
paragraph, delete — in 40 checks, and cleans up after itself.
This commit is contained in:
govin
2026-09-18 17:29:12 +08:00
parent 0d0aa20be2
commit 06d7e922bd
13 changed files with 2173 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/crud/paper.py
+608
View File
@@ -0,0 +1,608 @@
"""Data access for papers: metadata, sentences, and the document they assemble into.
The heart of the module is :func:`build_document`. Nothing stores a paper's
final shape; it is recomputed on every read from two sources:
1. the **template** the paper points at — its placements, ordered by ``sort``,
which supply the headings and the empty structure;
2. the **sentences** — ordered by ``paper_template_filed_sort`` then ``sort``,
which supply the content.
A paragraph exists if either side mentions its position. That single rule buys
all three behaviours the feature needs:
* a paragraph with no content still renders (structure survives);
* a sentence whose position the template does not define still renders, under
an unset heading, in the right place in the order (nothing is lost when the
template changes);
* swapping the template changes nothing but the headings, because sentences are
addressed by position and never by a template row id.
Deleting is done in Python rather than with ``ON DELETE CASCADE`` because TiDB
parses foreign keys and then ignores them; the ORM's cascades are the only
thing that actually removes dependent rows.
"""
from collections import defaultdict
from collections.abc import Iterable, Sequence
from typing import Any
from sqlalchemy import func, or_, select
from sqlalchemy import update as sa_update
from sqlalchemy.orm import Session
from app.crud.filters import LIKE_ESCAPE, like_pattern
from app.models import (
Paper,
PaperSentence,
PaperSentenceReference,
PaperTemplate,
TemplateField,
)
from app.schemas.paper import (
CitationInput,
PaperCitationRead,
PaperCreate,
PaperDocumentRead,
PaperListItem,
PaperRead,
ParagraphRead,
ParagraphUpdate,
SentenceCreate,
SentenceRead,
SentenceUpdate,
)
# --- list / read -------------------------------------------------------------
def _conditions(
keyword: str | None,
status: str | None,
template_id: int | None,
) -> list:
"""Translate the list filters into SQLAlchemy predicates."""
conditions = []
if keyword:
pattern = like_pattern(keyword)
conditions.append(
or_(
Paper.title.like(pattern, escape=LIKE_ESCAPE),
Paper.author.like(pattern, escape=LIKE_ESCAPE),
Paper.keywords.like(pattern, escape=LIKE_ESCAPE),
)
)
if status:
conditions.append(Paper.status == status)
if template_id is not None:
conditions.append(Paper.template_id == template_id)
return conditions
def _sentence_count_column():
"""Correlated count of the paper's sentences."""
return (
select(func.count(PaperSentence.id))
.where(PaperSentence.paper_id == Paper.id)
.correlate(Paper)
.scalar_subquery()
)
def _paragraph_count_column():
"""Correlated count of the *distinct paragraphs* the paper has content in.
``DISTINCT`` rather than a plain count: this is "how many paragraphs have
been written", so ten sentences in one paragraph read as 1, not 10.
"""
return (
select(func.count(func.distinct(PaperSentence.paper_template_filed_sort)))
.where(PaperSentence.paper_id == Paper.id)
.correlate(Paper)
.scalar_subquery()
)
def _template_paragraph_count_column():
"""Correlated count of the paragraphs the paper's template defines.
The denominator of the progress read-out. ``DISTINCT`` for the same reason
ties are legal in a template: two placements may share a position, and that
is one paragraph, not two. A paper with no template counts 0.
"""
return (
select(func.count(func.distinct(TemplateField.sort)))
.where(TemplateField.template_id == Paper.template_id)
.correlate(Paper)
.scalar_subquery()
)
def _list_item(
paper: Paper,
*,
sentence_count: int,
paragraph_count: int,
template_paragraph_count: int,
) -> PaperListItem:
"""Assemble one table row from a paper and its three counts."""
return PaperListItem(
id=paper.id,
title=paper.title,
template_id=paper.template_id,
# ``Paper.template`` is eagerly joined, so this costs nothing here.
template_name=paper.template.name if paper.template is not None else None,
author=paper.author,
status=paper.status,
keywords=paper.keywords,
target_journal=paper.target_journal,
sentence_count=sentence_count,
paragraph_count=paragraph_count,
template_paragraph_count=template_paragraph_count,
created_at=paper.created_at,
updated_at=paper.updated_at,
)
def list_papers(
db: Session,
*,
keyword: str | None = None,
status: str | None = None,
template_id: int | None = None,
page: int = 1,
page_size: int = 20,
) -> tuple[list[PaperListItem], int]:
"""Return one page of papers, most recently edited first.
The three counts arrive as correlated subqueries rather than as loads of
the sentences themselves: a table row needs "3 / 12 段", not three hundred
sentence rows.
"""
conditions = _conditions(keyword, status, template_id)
total = db.scalar(select(func.count(Paper.id)).where(*conditions)) or 0
stmt = (
select(
Paper,
_sentence_count_column().label("sentence_count"),
_paragraph_count_column().label("paragraph_count"),
_template_paragraph_count_column().label("template_paragraph_count"),
)
.where(*conditions)
.order_by(Paper.updated_at.desc(), Paper.id.desc())
.offset((page - 1) * page_size)
.limit(page_size)
)
items = [
_list_item(
paper,
sentence_count=sentence_count,
paragraph_count=paragraph_count,
template_paragraph_count=template_paragraph_count,
)
for paper, sentence_count, paragraph_count, template_paragraph_count in (
db.execute(stmt).all()
)
]
return items, total
def get(db: Session, paper_id: int) -> Paper | None:
"""Return one paper with its sentences and template loaded, or ``None``.
Both relationships are configured for eager loading, so this is a fixed
number of queries rather than one per sentence.
"""
return db.get(Paper, paper_id)
def get_many(db: Session, paper_ids: Sequence[int]) -> list[Paper]:
"""Return every paper whose id is in ``paper_ids`` (missing ids ignored)."""
if not paper_ids:
return []
return list(db.scalars(select(Paper).where(Paper.id.in_(list(paper_ids)))).all())
def read(paper: Paper) -> PaperRead:
"""Build the detail payload for one already-loaded paper.
The counts are computed in Python from the eager-loaded relationships, so
they always agree with the document the same request renders — a SQL count
and a rendered list can never drift apart this way.
"""
item = _list_item(
paper,
sentence_count=len(paper.sentences),
paragraph_count=len({s.paper_template_filed_sort for s in paper.sentences}),
template_paragraph_count=(
len({placement.sort for placement in paper.template.items})
if paper.template is not None
else 0
),
)
return PaperRead(**item.model_dump(), abstract=paper.abstract)
# --- document assembly -------------------------------------------------------
def build_document(db: Session, paper: Paper) -> PaperDocumentRead:
"""Assemble the whole paper: structure, content, citations, warnings.
One pass over the placements and one over the sentences. Positions are
unioned, so a position only one of the two sides knows about still appears:
an empty paragraph from the template, an unmatched paragraph from the
content.
"""
warnings: list[str] = []
# --- 1. the template's side: which heading sits at which position -------
#
# Ties are legal in a template (that is deliberate upstream), so the first
# placement at a position — the relationship hands them over ordered by
# ``sort, id`` — is the one that renders the heading. The reader is told,
# because two fields sharing a position is far more often a mistake than a
# plan.
placement_by_sort: dict[int, TemplateField] = {}
tied: dict[int, list[str]] = defaultdict(list)
for item in paper.template.items if paper.template is not None else []:
if item.sort in placement_by_sort:
tied[item.sort].append(item.field.name)
else:
placement_by_sort[item.sort] = item
for sort, names in sorted(tied.items()):
head = placement_by_sort[sort].field.name
warnings.append(
f"模板在第 {sort} 段放了多个字段（{head}、{'、'.join(names)}），"
"正文按第一条显示。"
)
# --- 2. the content's side: sentences grouped by position ---------------
sentences_by_sort: dict[int, list[PaperSentence]] = defaultdict(list)
for sentence in paper.sentences:
sentences_by_sort[sentence.paper_template_filed_sort].append(sentence)
if paper.template is None and paper.sentences:
warnings.append("这篇论文还没有选择模板，段落没有对应标题，全部按未设定显示。")
if paper.template is not None and not paper.template.items:
warnings.append("所用模板还没有任何段落字段，论文暂时没有结构。")
# --- 3. the union, in ascending position order --------------------------
positions = sorted(set(placement_by_sort) | set(sentences_by_sort))
paragraphs: list[ParagraphRead] = []
citations: list[PaperCitationRead] = []
citation_index = 0
for position in positions:
placement = placement_by_sort.get(position)
field = placement.field if placement is not None else None
# Re-sorted rather than trusted: the relationship already orders rows,
# but ordering is a contract of this response, so it is stated here too.
rows = sorted(
sentences_by_sort.get(position, []),
key=lambda item: (item.sort, item.id),
)
sentences = [SentenceRead.model_validate(row) for row in rows]
paragraph = ParagraphRead(
paper_template_filed_sort=position,
template_field_id=placement.id if placement is not None else None,
field_id=field.id if field is not None else None,
name=field.name if field is not None else None,
level=field.level if field is not None else 1,
font_size=float(field.font_size) if field is not None else None,
font_color=field.font_color if field is not None else None,
matched=placement is not None,
sentences=sentences,
)
# Citations are numbered in reading order, so a marker in the text and
# its entry in the 参考文献 list cannot disagree.
for sentence in sentences:
for citation in sentence.citations:
citation_index += 1
citations.append(
PaperCitationRead(
index=citation_index,
id=citation.id,
reference_id=citation.reference_id,
quote=citation.quote,
sentence_id=sentence.id,
sentence_content=sentence.content,
paper_template_filed_sort=position,
paragraph_name=paragraph.name,
)
)
paragraphs.append(paragraph)
return PaperDocumentRead(
paper=read(paper),
paragraphs=paragraphs,
citations=citations,
warnings=warnings,
)
def find_paragraph(
document: PaperDocumentRead, field_sort: int
) -> ParagraphRead | None:
"""Pick one paragraph out of an assembled document."""
for paragraph in document.paragraphs:
if paragraph.paper_template_filed_sort == field_sort:
return paragraph
return None
def get_paragraph(db: Session, paper: Paper, field_sort: int) -> ParagraphRead | None:
"""Return one paragraph, assembled exactly as the document renders it.
Assembling the whole document to return one paragraph is not waste: it is
what guarantees the editor shows precisely what the reader will see, rather
than a second implementation of the same ordering rules.
"""
return find_paragraph(build_document(db, paper), field_sort)
# --- writes ------------------------------------------------------------------
def _touch(db: Session, paper: Paper) -> None:
"""Mark the paper as edited, on the database clock.
Writing content does not itself change any column of ``paper``, so the
``onupdate`` on ``updated_at`` would not fire and the list — which sorts by
it — would keep showing the creation time of a paper whose text changed an
hour ago. ``func.now()`` rather than Python's clock, so timestamps do not
depend on which host the API runs on.
"""
paper.updated_at = func.now() # type: ignore[assignment]
def _commit_touch(db: Session, paper: Paper) -> None:
"""Commit a content change and refresh the paper's timestamps.
The refresh is what turns the ``func.now()`` expression above back into a
real datetime for the response — MySQL and TiDB have no ``RETURNING``, so
the value only exists after a re-read.
"""
_touch(db, paper)
db.commit()
db.refresh(paper)
def _build_citations(items: Sequence[CitationInput]) -> list[PaperSentenceReference]:
"""Materialise the citation list of one sentence, numbering it from 1."""
return [
PaperSentenceReference(
reference_id=item.reference_id,
quote=item.quote,
sort=index,
)
for index, item in enumerate(items, start=1)
]
def _max_sort(db: Session, paper: Paper, field_sort: int) -> int:
"""The highest sentence sort currently used in one paragraph (0 if empty)."""
value = db.scalar(
select(func.coalesce(func.max(PaperSentence.sort), 0)).where(
PaperSentence.paper_id == paper.id,
PaperSentence.paper_template_filed_sort == field_sort,
)
)
return int(value or 0)
def create(db: Session, data: PaperCreate) -> Paper:
"""Insert a paper. No rows are created for its paragraphs.
The structure is read from the template on every render, so materialising
an empty sentence per paragraph would create rows that exist only to say
nothing — and that would then have to be kept in step with the template.
"""
paper = Paper(**data.model_dump())
db.add(paper)
db.commit()
db.refresh(paper)
return paper
def update(
db: Session,
paper: Paper,
*,
values: dict[str, Any],
template_changed: bool,
) -> Paper:
"""Apply a partial update.
``values`` holds only the keys the client actually sent — the route derives
it from ``model_fields_set``, which is the only way to tell "clear the
abstract" apart from "leave it alone"; both arrive as ``None``.
When the template changes, every sentence is re-stamped with the new
template id. The sentences themselves keep their positions, which is what
makes a switch non-destructive — and it keeps the pair
``(template_id, paper_template_filed_sort)`` an honest description of which
paragraph a sentence belongs to.
"""
for key, value in values.items():
setattr(paper, key, value)
if template_changed:
db.flush()
db.execute(
sa_update(PaperSentence)
.where(PaperSentence.paper_id == paper.id)
.values(template_id=paper.template_id)
)
_touch(db, paper)
db.commit()
db.refresh(paper)
return paper
def delete(db: Session, paper: Paper) -> None:
"""Delete a paper and, through the ORM cascade, all of its sentences."""
db.delete(paper)
db.commit()
def delete_many(db: Session, paper_ids: Sequence[int]) -> int:
"""Delete several papers, returning how many actually existed."""
papers = get_many(db, paper_ids)
for paper in papers:
db.delete(paper)
db.commit()
return len(papers)
# --- paragraphs and sentences ------------------------------------------------
def replace_paragraph(
db: Session,
paper: Paper,
field_sort: int,
payload: ParagraphUpdate,
) -> None:
"""Rewrite one paragraph's sentences, optionally moving the paragraph.
The editor sends the paragraph whole — every line it shows — so this is a
replacement, not a merge. ``target_sort`` is the manual form of what a
template switch does by itself: the sentences are written at another
position and appended after whatever that paragraph already holds, which is
why the numbering is computed from the target's current maximum rather than
from 1.
Blank lines are kept if the client sends them: an empty sentence is a
legitimate placeholder and renders as a blank line rather than vanishing.
"""
target = payload.target_sort if payload.target_sort is not None else field_sort
# Clear the source paragraph first, so that a move onto itself does not
# count its own rows when working out where the appended block starts.
existing = list(
db.scalars(
select(PaperSentence).where(
PaperSentence.paper_id == paper.id,
PaperSentence.paper_template_filed_sort == field_sort,
)
).all()
)
for row in existing:
db.delete(row)
db.flush()
base = _max_sort(db, paper, target)
for offset, item in enumerate(payload.sentences, start=1):
db.add(
PaperSentence(
paper_id=paper.id,
# Kept in step with the paper's template: the pair
# (template_id, position) is what names the paragraph.
template_id=paper.template_id,
paper_template_filed_sort=target,
sort=item.sort if item.sort is not None else base + offset,
content=item.content,
citations=_build_citations(item.citations),
)
)
_commit_touch(db, paper)
def append_sentence(db: Session, paper: Paper, data: SentenceCreate) -> PaperSentence:
"""Add one sentence to a paragraph, at the end unless told otherwise."""
sort = (
data.sort
if data.sort is not None
else _max_sort(db, paper, data.paper_template_filed_sort) + 1
)
sentence = PaperSentence(
paper_id=paper.id,
template_id=paper.template_id,
paper_template_filed_sort=data.paper_template_filed_sort,
sort=sort,
content=data.content,
citations=_build_citations(data.citations),
)
db.add(sentence)
_commit_touch(db, paper)
db.refresh(sentence)
return sentence
def update_sentence(
db: Session,
paper: Paper,
sentence: PaperSentence,
data: SentenceUpdate,
) -> PaperSentence:
"""Apply a partial update to one sentence.
``citations`` is a full replacement when present: the sentence's citation
list is short, and replacing it wholesale is the only way to be sure that a
deleted citation really is gone.
"""
if data.sort is not None:
sentence.sort = data.sort
if data.paper_template_filed_sort is not None:
sentence.paper_template_filed_sort = data.paper_template_filed_sort
if data.content is not None:
sentence.content = data.content
if data.citations is not None:
sentence.citations.clear()
db.flush()
sentence.citations.extend(_build_citations(data.citations))
sentence.template_id = paper.template_id
_commit_touch(db, paper)
db.refresh(sentence)
return sentence
def delete_sentence(db: Session, paper: Paper, sentence: PaperSentence) -> None:
"""Delete one sentence and its citations."""
db.delete(sentence)
_commit_touch(db, paper)
# --- template housekeeping ---------------------------------------------------
def _paper_counts_by_template(
db: Session, template_ids: Iterable[int]
) -> dict[int, int]:
"""How many papers use each template id."""
wanted = list({template_id for template_id in template_ids})
if not wanted:
return {}
stmt = (
select(Paper.template_id, func.count(Paper.id))
.where(Paper.template_id.in_(wanted))
.group_by(Paper.template_id)
)
return {template_id: count for template_id, count in db.execute(stmt).all()}
def template_paper_counts(
db: Session, template_ids: Sequence[int]
) -> dict[int, int]:
"""Public form of :func:`_paper_counts_by_template`.
Used by the template endpoints: deleting a template a paper is written
against would leave that paper with no structure at all, so it is refused
with the count that explains why.
"""
return _paper_counts_by_template(db, template_ids)