govin
06d7e922bd
A paper is a document written against a template. Its structure is never copied into it: `paper` points at a template and the outline is read live on every render, so switching `template_id` re-shapes the whole document in one write. Sentences are addressed by *position* rather than by a template row id: `paper_sentence.paper_template_filed_sort` holds the sort of the placement the sentence belongs to, and `sort` holds its place inside that paragraph. That indirection is what makes a template switch non-destructive — a sentence that remembers "position 7" lands on whatever the new template puts at position 7 — and it is why a paragraph is any position either the template or the content mentions: the structure survives with no content, and content whose position the template does not define is still rendered, in order, under 未设定. Citations are a table rather than a column, since one sentence may quote several references. `quote` is required — a citation that does not say what it quotes is refused with 422 — while `reference_id` is a plain nullable integer with no foreign key, because the reference library does not exist yet. Deleting a template a paper is written against is refused with 409 and a count, matching how the field library refuses to drop a field still in use. scripts/smoke_papers.py walks the whole loop — create, empty structure, write a paragraph with citations, switch templates, keep unmatched positions, move a paragraph, delete — in 40 checks, and cleans up after itself.
609 lines
21 KiB
Python
609 lines
21 KiB
Python
"""Data access for papers: metadata, sentences, and the document they assemble into.
|
|
|
|
The heart of the module is :func:`build_document`. Nothing stores a paper's
|
|
final shape; it is recomputed on every read from two sources:
|
|
|
|
1. the **template** the paper points at — its placements, ordered by ``sort``,
|
|
which supply the headings and the empty structure;
|
|
2. the **sentences** — ordered by ``paper_template_filed_sort`` then ``sort``,
|
|
which supply the content.
|
|
|
|
A paragraph exists if either side mentions its position. That single rule buys
|
|
all three behaviours the feature needs:
|
|
|
|
* a paragraph with no content still renders (structure survives);
|
|
* a sentence whose position the template does not define still renders, under
|
|
an unset heading, in the right place in the order (nothing is lost when the
|
|
template changes);
|
|
* swapping the template changes nothing but the headings, because sentences are
|
|
addressed by position and never by a template row id.
|
|
|
|
Deleting is done in Python rather than with ``ON DELETE CASCADE`` because TiDB
|
|
parses foreign keys and then ignores them; the ORM's cascades are the only
|
|
thing that actually removes dependent rows.
|
|
"""
|
|
|
|
from collections import defaultdict
|
|
from collections.abc import Iterable, Sequence
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, or_, select
|
|
from sqlalchemy import update as sa_update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.crud.filters import LIKE_ESCAPE, like_pattern
|
|
from app.models import (
|
|
Paper,
|
|
PaperSentence,
|
|
PaperSentenceReference,
|
|
PaperTemplate,
|
|
TemplateField,
|
|
)
|
|
from app.schemas.paper import (
|
|
CitationInput,
|
|
PaperCitationRead,
|
|
PaperCreate,
|
|
PaperDocumentRead,
|
|
PaperListItem,
|
|
PaperRead,
|
|
ParagraphRead,
|
|
ParagraphUpdate,
|
|
SentenceCreate,
|
|
SentenceRead,
|
|
SentenceUpdate,
|
|
)
|
|
|
|
|
|
# --- list / read -------------------------------------------------------------
|
|
|
|
|
|
def _conditions(
|
|
keyword: str | None,
|
|
status: str | None,
|
|
template_id: int | None,
|
|
) -> list:
|
|
"""Translate the list filters into SQLAlchemy predicates."""
|
|
conditions = []
|
|
if keyword:
|
|
pattern = like_pattern(keyword)
|
|
conditions.append(
|
|
or_(
|
|
Paper.title.like(pattern, escape=LIKE_ESCAPE),
|
|
Paper.author.like(pattern, escape=LIKE_ESCAPE),
|
|
Paper.keywords.like(pattern, escape=LIKE_ESCAPE),
|
|
)
|
|
)
|
|
if status:
|
|
conditions.append(Paper.status == status)
|
|
if template_id is not None:
|
|
conditions.append(Paper.template_id == template_id)
|
|
return conditions
|
|
|
|
|
|
def _sentence_count_column():
|
|
"""Correlated count of the paper's sentences."""
|
|
return (
|
|
select(func.count(PaperSentence.id))
|
|
.where(PaperSentence.paper_id == Paper.id)
|
|
.correlate(Paper)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _paragraph_count_column():
|
|
"""Correlated count of the *distinct paragraphs* the paper has content in.
|
|
|
|
``DISTINCT`` rather than a plain count: this is "how many paragraphs have
|
|
been written", so ten sentences in one paragraph read as 1, not 10.
|
|
"""
|
|
return (
|
|
select(func.count(func.distinct(PaperSentence.paper_template_filed_sort)))
|
|
.where(PaperSentence.paper_id == Paper.id)
|
|
.correlate(Paper)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _template_paragraph_count_column():
|
|
"""Correlated count of the paragraphs the paper's template defines.
|
|
|
|
The denominator of the progress read-out. ``DISTINCT`` for the same reason
|
|
ties are legal in a template: two placements may share a position, and that
|
|
is one paragraph, not two. A paper with no template counts 0.
|
|
"""
|
|
return (
|
|
select(func.count(func.distinct(TemplateField.sort)))
|
|
.where(TemplateField.template_id == Paper.template_id)
|
|
.correlate(Paper)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _list_item(
|
|
paper: Paper,
|
|
*,
|
|
sentence_count: int,
|
|
paragraph_count: int,
|
|
template_paragraph_count: int,
|
|
) -> PaperListItem:
|
|
"""Assemble one table row from a paper and its three counts."""
|
|
return PaperListItem(
|
|
id=paper.id,
|
|
title=paper.title,
|
|
template_id=paper.template_id,
|
|
# ``Paper.template`` is eagerly joined, so this costs nothing here.
|
|
template_name=paper.template.name if paper.template is not None else None,
|
|
author=paper.author,
|
|
status=paper.status,
|
|
keywords=paper.keywords,
|
|
target_journal=paper.target_journal,
|
|
sentence_count=sentence_count,
|
|
paragraph_count=paragraph_count,
|
|
template_paragraph_count=template_paragraph_count,
|
|
created_at=paper.created_at,
|
|
updated_at=paper.updated_at,
|
|
)
|
|
|
|
|
|
def list_papers(
|
|
db: Session,
|
|
*,
|
|
keyword: str | None = None,
|
|
status: str | None = None,
|
|
template_id: int | None = None,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
) -> tuple[list[PaperListItem], int]:
|
|
"""Return one page of papers, most recently edited first.
|
|
|
|
The three counts arrive as correlated subqueries rather than as loads of
|
|
the sentences themselves: a table row needs "3 / 12 段", not three hundred
|
|
sentence rows.
|
|
"""
|
|
conditions = _conditions(keyword, status, template_id)
|
|
|
|
total = db.scalar(select(func.count(Paper.id)).where(*conditions)) or 0
|
|
|
|
stmt = (
|
|
select(
|
|
Paper,
|
|
_sentence_count_column().label("sentence_count"),
|
|
_paragraph_count_column().label("paragraph_count"),
|
|
_template_paragraph_count_column().label("template_paragraph_count"),
|
|
)
|
|
.where(*conditions)
|
|
.order_by(Paper.updated_at.desc(), Paper.id.desc())
|
|
.offset((page - 1) * page_size)
|
|
.limit(page_size)
|
|
)
|
|
|
|
items = [
|
|
_list_item(
|
|
paper,
|
|
sentence_count=sentence_count,
|
|
paragraph_count=paragraph_count,
|
|
template_paragraph_count=template_paragraph_count,
|
|
)
|
|
for paper, sentence_count, paragraph_count, template_paragraph_count in (
|
|
db.execute(stmt).all()
|
|
)
|
|
]
|
|
return items, total
|
|
|
|
|
|
def get(db: Session, paper_id: int) -> Paper | None:
|
|
"""Return one paper with its sentences and template loaded, or ``None``.
|
|
|
|
Both relationships are configured for eager loading, so this is a fixed
|
|
number of queries rather than one per sentence.
|
|
"""
|
|
return db.get(Paper, paper_id)
|
|
|
|
|
|
def get_many(db: Session, paper_ids: Sequence[int]) -> list[Paper]:
|
|
"""Return every paper whose id is in ``paper_ids`` (missing ids ignored)."""
|
|
if not paper_ids:
|
|
return []
|
|
return list(db.scalars(select(Paper).where(Paper.id.in_(list(paper_ids)))).all())
|
|
|
|
|
|
def read(paper: Paper) -> PaperRead:
|
|
"""Build the detail payload for one already-loaded paper.
|
|
|
|
The counts are computed in Python from the eager-loaded relationships, so
|
|
they always agree with the document the same request renders — a SQL count
|
|
and a rendered list can never drift apart this way.
|
|
"""
|
|
item = _list_item(
|
|
paper,
|
|
sentence_count=len(paper.sentences),
|
|
paragraph_count=len({s.paper_template_filed_sort for s in paper.sentences}),
|
|
template_paragraph_count=(
|
|
len({placement.sort for placement in paper.template.items})
|
|
if paper.template is not None
|
|
else 0
|
|
),
|
|
)
|
|
return PaperRead(**item.model_dump(), abstract=paper.abstract)
|
|
|
|
|
|
# --- document assembly -------------------------------------------------------
|
|
|
|
|
|
def build_document(db: Session, paper: Paper) -> PaperDocumentRead:
|
|
"""Assemble the whole paper: structure, content, citations, warnings.
|
|
|
|
One pass over the placements and one over the sentences. Positions are
|
|
unioned, so a position only one of the two sides knows about still appears:
|
|
an empty paragraph from the template, an unmatched paragraph from the
|
|
content.
|
|
"""
|
|
warnings: list[str] = []
|
|
|
|
# --- 1. the template's side: which heading sits at which position -------
|
|
#
|
|
# Ties are legal in a template (that is deliberate upstream), so the first
|
|
# placement at a position — the relationship hands them over ordered by
|
|
# ``sort, id`` — is the one that renders the heading. The reader is told,
|
|
# because two fields sharing a position is far more often a mistake than a
|
|
# plan.
|
|
placement_by_sort: dict[int, TemplateField] = {}
|
|
tied: dict[int, list[str]] = defaultdict(list)
|
|
for item in paper.template.items if paper.template is not None else []:
|
|
if item.sort in placement_by_sort:
|
|
tied[item.sort].append(item.field.name)
|
|
else:
|
|
placement_by_sort[item.sort] = item
|
|
for sort, names in sorted(tied.items()):
|
|
head = placement_by_sort[sort].field.name
|
|
warnings.append(
|
|
f"模板在第 {sort} 段放了多个字段（{head}、{'、'.join(names)}），"
|
|
"正文按第一条显示。"
|
|
)
|
|
|
|
# --- 2. the content's side: sentences grouped by position ---------------
|
|
sentences_by_sort: dict[int, list[PaperSentence]] = defaultdict(list)
|
|
for sentence in paper.sentences:
|
|
sentences_by_sort[sentence.paper_template_filed_sort].append(sentence)
|
|
|
|
if paper.template is None and paper.sentences:
|
|
warnings.append("这篇论文还没有选择模板，段落没有对应标题，全部按未设定显示。")
|
|
if paper.template is not None and not paper.template.items:
|
|
warnings.append("所用模板还没有任何段落字段，论文暂时没有结构。")
|
|
|
|
# --- 3. the union, in ascending position order --------------------------
|
|
positions = sorted(set(placement_by_sort) | set(sentences_by_sort))
|
|
|
|
paragraphs: list[ParagraphRead] = []
|
|
citations: list[PaperCitationRead] = []
|
|
citation_index = 0
|
|
|
|
for position in positions:
|
|
placement = placement_by_sort.get(position)
|
|
field = placement.field if placement is not None else None
|
|
|
|
# Re-sorted rather than trusted: the relationship already orders rows,
|
|
# but ordering is a contract of this response, so it is stated here too.
|
|
rows = sorted(
|
|
sentences_by_sort.get(position, []),
|
|
key=lambda item: (item.sort, item.id),
|
|
)
|
|
sentences = [SentenceRead.model_validate(row) for row in rows]
|
|
|
|
paragraph = ParagraphRead(
|
|
paper_template_filed_sort=position,
|
|
template_field_id=placement.id if placement is not None else None,
|
|
field_id=field.id if field is not None else None,
|
|
name=field.name if field is not None else None,
|
|
level=field.level if field is not None else 1,
|
|
font_size=float(field.font_size) if field is not None else None,
|
|
font_color=field.font_color if field is not None else None,
|
|
matched=placement is not None,
|
|
sentences=sentences,
|
|
)
|
|
|
|
# Citations are numbered in reading order, so a marker in the text and
|
|
# its entry in the 参考文献 list cannot disagree.
|
|
for sentence in sentences:
|
|
for citation in sentence.citations:
|
|
citation_index += 1
|
|
citations.append(
|
|
PaperCitationRead(
|
|
index=citation_index,
|
|
id=citation.id,
|
|
reference_id=citation.reference_id,
|
|
quote=citation.quote,
|
|
sentence_id=sentence.id,
|
|
sentence_content=sentence.content,
|
|
paper_template_filed_sort=position,
|
|
paragraph_name=paragraph.name,
|
|
)
|
|
)
|
|
|
|
paragraphs.append(paragraph)
|
|
|
|
return PaperDocumentRead(
|
|
paper=read(paper),
|
|
paragraphs=paragraphs,
|
|
citations=citations,
|
|
warnings=warnings,
|
|
)
|
|
|
|
|
|
def find_paragraph(
|
|
document: PaperDocumentRead, field_sort: int
|
|
) -> ParagraphRead | None:
|
|
"""Pick one paragraph out of an assembled document."""
|
|
for paragraph in document.paragraphs:
|
|
if paragraph.paper_template_filed_sort == field_sort:
|
|
return paragraph
|
|
return None
|
|
|
|
|
|
def get_paragraph(db: Session, paper: Paper, field_sort: int) -> ParagraphRead | None:
|
|
"""Return one paragraph, assembled exactly as the document renders it.
|
|
|
|
Assembling the whole document to return one paragraph is not waste: it is
|
|
what guarantees the editor shows precisely what the reader will see, rather
|
|
than a second implementation of the same ordering rules.
|
|
"""
|
|
return find_paragraph(build_document(db, paper), field_sort)
|
|
|
|
|
|
# --- writes ------------------------------------------------------------------
|
|
|
|
|
|
def _touch(db: Session, paper: Paper) -> None:
|
|
"""Mark the paper as edited, on the database clock.
|
|
|
|
Writing content does not itself change any column of ``paper``, so the
|
|
``onupdate`` on ``updated_at`` would not fire and the list — which sorts by
|
|
it — would keep showing the creation time of a paper whose text changed an
|
|
hour ago. ``func.now()`` rather than Python's clock, so timestamps do not
|
|
depend on which host the API runs on.
|
|
"""
|
|
paper.updated_at = func.now() # type: ignore[assignment]
|
|
|
|
|
|
def _commit_touch(db: Session, paper: Paper) -> None:
|
|
"""Commit a content change and refresh the paper's timestamps.
|
|
|
|
The refresh is what turns the ``func.now()`` expression above back into a
|
|
real datetime for the response — MySQL and TiDB have no ``RETURNING``, so
|
|
the value only exists after a re-read.
|
|
"""
|
|
_touch(db, paper)
|
|
db.commit()
|
|
db.refresh(paper)
|
|
|
|
|
|
def _build_citations(items: Sequence[CitationInput]) -> list[PaperSentenceReference]:
|
|
"""Materialise the citation list of one sentence, numbering it from 1."""
|
|
return [
|
|
PaperSentenceReference(
|
|
reference_id=item.reference_id,
|
|
quote=item.quote,
|
|
sort=index,
|
|
)
|
|
for index, item in enumerate(items, start=1)
|
|
]
|
|
|
|
|
|
def _max_sort(db: Session, paper: Paper, field_sort: int) -> int:
|
|
"""The highest sentence sort currently used in one paragraph (0 if empty)."""
|
|
value = db.scalar(
|
|
select(func.coalesce(func.max(PaperSentence.sort), 0)).where(
|
|
PaperSentence.paper_id == paper.id,
|
|
PaperSentence.paper_template_filed_sort == field_sort,
|
|
)
|
|
)
|
|
return int(value or 0)
|
|
|
|
|
|
def create(db: Session, data: PaperCreate) -> Paper:
|
|
"""Insert a paper. No rows are created for its paragraphs.
|
|
|
|
The structure is read from the template on every render, so materialising
|
|
an empty sentence per paragraph would create rows that exist only to say
|
|
nothing — and that would then have to be kept in step with the template.
|
|
"""
|
|
paper = Paper(**data.model_dump())
|
|
db.add(paper)
|
|
db.commit()
|
|
db.refresh(paper)
|
|
return paper
|
|
|
|
|
|
def update(
|
|
db: Session,
|
|
paper: Paper,
|
|
*,
|
|
values: dict[str, Any],
|
|
template_changed: bool,
|
|
) -> Paper:
|
|
"""Apply a partial update.
|
|
|
|
``values`` holds only the keys the client actually sent — the route derives
|
|
it from ``model_fields_set``, which is the only way to tell "clear the
|
|
abstract" apart from "leave it alone"; both arrive as ``None``.
|
|
|
|
When the template changes, every sentence is re-stamped with the new
|
|
template id. The sentences themselves keep their positions, which is what
|
|
makes a switch non-destructive — and it keeps the pair
|
|
``(template_id, paper_template_filed_sort)`` an honest description of which
|
|
paragraph a sentence belongs to.
|
|
"""
|
|
for key, value in values.items():
|
|
setattr(paper, key, value)
|
|
|
|
if template_changed:
|
|
db.flush()
|
|
db.execute(
|
|
sa_update(PaperSentence)
|
|
.where(PaperSentence.paper_id == paper.id)
|
|
.values(template_id=paper.template_id)
|
|
)
|
|
|
|
_touch(db, paper)
|
|
db.commit()
|
|
db.refresh(paper)
|
|
return paper
|
|
|
|
|
|
def delete(db: Session, paper: Paper) -> None:
|
|
"""Delete a paper and, through the ORM cascade, all of its sentences."""
|
|
db.delete(paper)
|
|
db.commit()
|
|
|
|
|
|
def delete_many(db: Session, paper_ids: Sequence[int]) -> int:
|
|
"""Delete several papers, returning how many actually existed."""
|
|
papers = get_many(db, paper_ids)
|
|
for paper in papers:
|
|
db.delete(paper)
|
|
db.commit()
|
|
return len(papers)
|
|
|
|
|
|
# --- paragraphs and sentences ------------------------------------------------
|
|
|
|
|
|
def replace_paragraph(
|
|
db: Session,
|
|
paper: Paper,
|
|
field_sort: int,
|
|
payload: ParagraphUpdate,
|
|
) -> None:
|
|
"""Rewrite one paragraph's sentences, optionally moving the paragraph.
|
|
|
|
The editor sends the paragraph whole — every line it shows — so this is a
|
|
replacement, not a merge. ``target_sort`` is the manual form of what a
|
|
template switch does by itself: the sentences are written at another
|
|
position and appended after whatever that paragraph already holds, which is
|
|
why the numbering is computed from the target's current maximum rather than
|
|
from 1.
|
|
|
|
Blank lines are kept if the client sends them: an empty sentence is a
|
|
legitimate placeholder and renders as a blank line rather than vanishing.
|
|
"""
|
|
target = payload.target_sort if payload.target_sort is not None else field_sort
|
|
|
|
# Clear the source paragraph first, so that a move onto itself does not
|
|
# count its own rows when working out where the appended block starts.
|
|
existing = list(
|
|
db.scalars(
|
|
select(PaperSentence).where(
|
|
PaperSentence.paper_id == paper.id,
|
|
PaperSentence.paper_template_filed_sort == field_sort,
|
|
)
|
|
).all()
|
|
)
|
|
for row in existing:
|
|
db.delete(row)
|
|
db.flush()
|
|
|
|
base = _max_sort(db, paper, target)
|
|
|
|
for offset, item in enumerate(payload.sentences, start=1):
|
|
db.add(
|
|
PaperSentence(
|
|
paper_id=paper.id,
|
|
# Kept in step with the paper's template: the pair
|
|
# (template_id, position) is what names the paragraph.
|
|
template_id=paper.template_id,
|
|
paper_template_filed_sort=target,
|
|
sort=item.sort if item.sort is not None else base + offset,
|
|
content=item.content,
|
|
citations=_build_citations(item.citations),
|
|
)
|
|
)
|
|
|
|
_commit_touch(db, paper)
|
|
|
|
|
|
def append_sentence(db: Session, paper: Paper, data: SentenceCreate) -> PaperSentence:
|
|
"""Add one sentence to a paragraph, at the end unless told otherwise."""
|
|
sort = (
|
|
data.sort
|
|
if data.sort is not None
|
|
else _max_sort(db, paper, data.paper_template_filed_sort) + 1
|
|
)
|
|
sentence = PaperSentence(
|
|
paper_id=paper.id,
|
|
template_id=paper.template_id,
|
|
paper_template_filed_sort=data.paper_template_filed_sort,
|
|
sort=sort,
|
|
content=data.content,
|
|
citations=_build_citations(data.citations),
|
|
)
|
|
db.add(sentence)
|
|
_commit_touch(db, paper)
|
|
db.refresh(sentence)
|
|
return sentence
|
|
|
|
|
|
def update_sentence(
|
|
db: Session,
|
|
paper: Paper,
|
|
sentence: PaperSentence,
|
|
data: SentenceUpdate,
|
|
) -> PaperSentence:
|
|
"""Apply a partial update to one sentence.
|
|
|
|
``citations`` is a full replacement when present: the sentence's citation
|
|
list is short, and replacing it wholesale is the only way to be sure that a
|
|
deleted citation really is gone.
|
|
"""
|
|
if data.sort is not None:
|
|
sentence.sort = data.sort
|
|
if data.paper_template_filed_sort is not None:
|
|
sentence.paper_template_filed_sort = data.paper_template_filed_sort
|
|
if data.content is not None:
|
|
sentence.content = data.content
|
|
if data.citations is not None:
|
|
sentence.citations.clear()
|
|
db.flush()
|
|
sentence.citations.extend(_build_citations(data.citations))
|
|
|
|
sentence.template_id = paper.template_id
|
|
|
|
_commit_touch(db, paper)
|
|
db.refresh(sentence)
|
|
return sentence
|
|
|
|
|
|
def delete_sentence(db: Session, paper: Paper, sentence: PaperSentence) -> None:
|
|
"""Delete one sentence and its citations."""
|
|
db.delete(sentence)
|
|
_commit_touch(db, paper)
|
|
|
|
|
|
# --- template housekeeping ---------------------------------------------------
|
|
|
|
|
|
def _paper_counts_by_template(
|
|
db: Session, template_ids: Iterable[int]
|
|
) -> dict[int, int]:
|
|
"""How many papers use each template id."""
|
|
wanted = list({template_id for template_id in template_ids})
|
|
if not wanted:
|
|
return {}
|
|
stmt = (
|
|
select(Paper.template_id, func.count(Paper.id))
|
|
.where(Paper.template_id.in_(wanted))
|
|
.group_by(Paper.template_id)
|
|
)
|
|
return {template_id: count for template_id, count in db.execute(stmt).all()}
|
|
|
|
|
|
def template_paper_counts(
|
|
db: Session, template_ids: Sequence[int]
|
|
) -> dict[int, int]:
|
|
"""Public form of :func:`_paper_counts_by_template`.
|
|
|
|
Used by the template endpoints: deleting a template a paper is written
|
|
against would leave that paper with no structure at all, so it is refused
|
|
with the count that explains why.
|
|
"""
|
|
return _paper_counts_by_template(db, template_ids)
|