govin
45926af966
A template's abstract paragraph (`0 Abstract`) *is* the paper's abstract: it has a position in the document, the heading and typography the template gives it, and the same sentence-by-sentence editing as everything else. `paper.abstract` was a second home for that text — one the document never reads, so a paper could show two different abstracts and the column could drift from the body. The column is gone, from the table, the model, the schemas, the API payloads, the paper form, the list subtitle and the paper page. Nothing else changed. The text already written into it is not gone. Revision a83f5c21d7b6 writes each stored abstract into the paper's body first — one sentence per 。 at the paragraph carrying the abstract heading, appended after anything already there rather than replacing it. A template without such a heading keeps the text too, one position above its first paragraph, where the document renders it under 未设定. Verified against the one paper that had an abstract: 450 characters in, 450 out, identical including `|J| ≤ α · I⁻ᵝ`, split into six sentences in the `0 Abstract` paragraph, still with its own edit button. The smoke test no longer assumes an empty database: it records the paper total before it starts and compares against that, so it can run on a real one.
609 lines
21 KiB
Python
609 lines
21 KiB
Python
"""Data access for papers: metadata, sentences, and the document they assemble into.
|
|
|
|
The heart of the module is :func:`build_document`. Nothing stores a paper's
|
|
final shape; it is recomputed on every read from two sources:
|
|
|
|
1. the **template** the paper points at — its placements, ordered by ``sort``,
|
|
which supply the headings and the empty structure;
|
|
2. the **sentences** — ordered by ``paper_template_filed_sort`` then ``sort``,
|
|
which supply the content.
|
|
|
|
A paragraph exists if either side mentions its position. That single rule buys
|
|
all three behaviours the feature needs:
|
|
|
|
* a paragraph with no content still renders (structure survives);
|
|
* a sentence whose position the template does not define still renders, under
|
|
an unset heading, in the right place in the order (nothing is lost when the
|
|
template changes);
|
|
* swapping the template changes nothing but the headings, because sentences are
|
|
addressed by position and never by a template row id.
|
|
|
|
Deleting is done in Python rather than with ``ON DELETE CASCADE`` because TiDB
|
|
parses foreign keys and then ignores them; the ORM's cascades are the only
|
|
thing that actually removes dependent rows.
|
|
"""
|
|
|
|
from collections import defaultdict
|
|
from collections.abc import Iterable, Sequence
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, or_, select
|
|
from sqlalchemy import update as sa_update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.crud.filters import LIKE_ESCAPE, like_pattern
|
|
from app.models import (
|
|
Paper,
|
|
PaperSentence,
|
|
PaperSentenceReference,
|
|
Template,
|
|
TemplateField,
|
|
)
|
|
from app.schemas.paper import (
|
|
CitationInput,
|
|
PaperCitationRead,
|
|
PaperCreate,
|
|
PaperDocumentRead,
|
|
PaperListItem,
|
|
PaperRead,
|
|
ParagraphRead,
|
|
ParagraphUpdate,
|
|
SentenceCreate,
|
|
SentenceRead,
|
|
SentenceUpdate,
|
|
)
|
|
|
|
|
|
# --- list / read -------------------------------------------------------------
|
|
|
|
|
|
def _conditions(
|
|
keyword: str | None,
|
|
status: str | None,
|
|
template_id: int | None,
|
|
) -> list:
|
|
"""Translate the list filters into SQLAlchemy predicates."""
|
|
conditions = []
|
|
if keyword:
|
|
pattern = like_pattern(keyword)
|
|
conditions.append(
|
|
or_(
|
|
Paper.title.like(pattern, escape=LIKE_ESCAPE),
|
|
Paper.author.like(pattern, escape=LIKE_ESCAPE),
|
|
Paper.keywords.like(pattern, escape=LIKE_ESCAPE),
|
|
)
|
|
)
|
|
if status:
|
|
conditions.append(Paper.status == status)
|
|
if template_id is not None:
|
|
conditions.append(Paper.template_id == template_id)
|
|
return conditions
|
|
|
|
|
|
def _sentence_count_column():
|
|
"""Correlated count of the paper's sentences."""
|
|
return (
|
|
select(func.count(PaperSentence.id))
|
|
.where(PaperSentence.paper_id == Paper.id)
|
|
.correlate(Paper)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _paragraph_count_column():
|
|
"""Correlated count of the *distinct paragraphs* the paper has content in.
|
|
|
|
``DISTINCT`` rather than a plain count: this is "how many paragraphs have
|
|
been written", so ten sentences in one paragraph read as 1, not 10.
|
|
"""
|
|
return (
|
|
select(func.count(func.distinct(PaperSentence.paper_template_filed_sort)))
|
|
.where(PaperSentence.paper_id == Paper.id)
|
|
.correlate(Paper)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _template_paragraph_count_column():
|
|
"""Correlated count of the paragraphs the paper's template defines.
|
|
|
|
The denominator of the progress read-out. ``DISTINCT`` for the same reason
|
|
ties are legal in a template: two placements may share a position, and that
|
|
is one paragraph, not two. A paper with no template counts 0.
|
|
"""
|
|
return (
|
|
select(func.count(func.distinct(TemplateField.sort)))
|
|
.where(TemplateField.template_id == Paper.template_id)
|
|
.correlate(Paper)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _list_item(
|
|
paper: Paper,
|
|
*,
|
|
sentence_count: int,
|
|
paragraph_count: int,
|
|
template_paragraph_count: int,
|
|
) -> PaperListItem:
|
|
"""Assemble one table row from a paper and its three counts."""
|
|
return PaperListItem(
|
|
id=paper.id,
|
|
title=paper.title,
|
|
template_id=paper.template_id,
|
|
# ``Paper.template`` is eagerly joined, so this costs nothing here.
|
|
template_name=paper.template.name if paper.template is not None else None,
|
|
author=paper.author,
|
|
status=paper.status,
|
|
keywords=paper.keywords,
|
|
target_journal=paper.target_journal,
|
|
sentence_count=sentence_count,
|
|
paragraph_count=paragraph_count,
|
|
template_paragraph_count=template_paragraph_count,
|
|
created_at=paper.created_at,
|
|
updated_at=paper.updated_at,
|
|
)
|
|
|
|
|
|
def list_papers(
|
|
db: Session,
|
|
*,
|
|
keyword: str | None = None,
|
|
status: str | None = None,
|
|
template_id: int | None = None,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
) -> tuple[list[PaperListItem], int]:
|
|
"""Return one page of papers, most recently edited first.
|
|
|
|
The three counts arrive as correlated subqueries rather than as loads of
|
|
the sentences themselves: a table row needs "3 / 12 段", not three hundred
|
|
sentence rows.
|
|
"""
|
|
conditions = _conditions(keyword, status, template_id)
|
|
|
|
total = db.scalar(select(func.count(Paper.id)).where(*conditions)) or 0
|
|
|
|
stmt = (
|
|
select(
|
|
Paper,
|
|
_sentence_count_column().label("sentence_count"),
|
|
_paragraph_count_column().label("paragraph_count"),
|
|
_template_paragraph_count_column().label("template_paragraph_count"),
|
|
)
|
|
.where(*conditions)
|
|
.order_by(Paper.updated_at.desc(), Paper.id.desc())
|
|
.offset((page - 1) * page_size)
|
|
.limit(page_size)
|
|
)
|
|
|
|
items = [
|
|
_list_item(
|
|
paper,
|
|
sentence_count=sentence_count,
|
|
paragraph_count=paragraph_count,
|
|
template_paragraph_count=template_paragraph_count,
|
|
)
|
|
for paper, sentence_count, paragraph_count, template_paragraph_count in (
|
|
db.execute(stmt).all()
|
|
)
|
|
]
|
|
return items, total
|
|
|
|
|
|
def get(db: Session, paper_id: int) -> Paper | None:
|
|
"""Return one paper with its sentences and template loaded, or ``None``.
|
|
|
|
Both relationships are configured for eager loading, so this is a fixed
|
|
number of queries rather than one per sentence.
|
|
"""
|
|
return db.get(Paper, paper_id)
|
|
|
|
|
|
def get_many(db: Session, paper_ids: Sequence[int]) -> list[Paper]:
|
|
"""Return every paper whose id is in ``paper_ids`` (missing ids ignored)."""
|
|
if not paper_ids:
|
|
return []
|
|
return list(db.scalars(select(Paper).where(Paper.id.in_(list(paper_ids)))).all())
|
|
|
|
|
|
def read(paper: Paper) -> PaperRead:
|
|
"""Build the detail payload for one already-loaded paper.
|
|
|
|
The counts are computed in Python from the eager-loaded relationships, so
|
|
they always agree with the document the same request renders — a SQL count
|
|
and a rendered list can never drift apart this way.
|
|
"""
|
|
item = _list_item(
|
|
paper,
|
|
sentence_count=len(paper.sentences),
|
|
paragraph_count=len({s.paper_template_filed_sort for s in paper.sentences}),
|
|
template_paragraph_count=(
|
|
len({placement.sort for placement in paper.template.items})
|
|
if paper.template is not None
|
|
else 0
|
|
),
|
|
)
|
|
return PaperRead(**item.model_dump())
|
|
|
|
|
|
# --- document assembly -------------------------------------------------------
|
|
|
|
|
|
def build_document(db: Session, paper: Paper) -> PaperDocumentRead:
|
|
"""Assemble the whole paper: structure, content, citations, warnings.
|
|
|
|
One pass over the placements and one over the sentences. Positions are
|
|
unioned, so a position only one of the two sides knows about still appears:
|
|
an empty paragraph from the template, an unmatched paragraph from the
|
|
content.
|
|
"""
|
|
warnings: list[str] = []
|
|
|
|
# --- 1. the template's side: which heading sits at which position -------
|
|
#
|
|
# Ties are legal in a template (that is deliberate upstream), so the first
|
|
# placement at a position — the relationship hands them over ordered by
|
|
# ``sort, id`` — is the one that renders the heading. The reader is told,
|
|
# because two fields sharing a position is far more often a mistake than a
|
|
# plan.
|
|
placement_by_sort: dict[int, TemplateField] = {}
|
|
tied: dict[int, list[str]] = defaultdict(list)
|
|
for item in paper.template.items if paper.template is not None else []:
|
|
if item.sort in placement_by_sort:
|
|
tied[item.sort].append(item.field.name)
|
|
else:
|
|
placement_by_sort[item.sort] = item
|
|
for sort, names in sorted(tied.items()):
|
|
head = placement_by_sort[sort].field.name
|
|
warnings.append(
|
|
f"模板在第 {sort} 段放了多个字段（{head}、{'、'.join(names)}），"
|
|
"正文按第一条显示。"
|
|
)
|
|
|
|
# --- 2. the content's side: sentences grouped by position ---------------
|
|
sentences_by_sort: dict[int, list[PaperSentence]] = defaultdict(list)
|
|
for sentence in paper.sentences:
|
|
sentences_by_sort[sentence.paper_template_filed_sort].append(sentence)
|
|
|
|
if paper.template is None and paper.sentences:
|
|
warnings.append("这篇论文还没有选择模板，段落没有对应标题，全部按未设定显示。")
|
|
if paper.template is not None and not paper.template.items:
|
|
warnings.append("所用模板还没有任何段落字段，论文暂时没有结构。")
|
|
|
|
# --- 3. the union, in ascending position order --------------------------
|
|
positions = sorted(set(placement_by_sort) | set(sentences_by_sort))
|
|
|
|
paragraphs: list[ParagraphRead] = []
|
|
citations: list[PaperCitationRead] = []
|
|
citation_index = 0
|
|
|
|
for position in positions:
|
|
placement = placement_by_sort.get(position)
|
|
field = placement.field if placement is not None else None
|
|
|
|
# Re-sorted rather than trusted: the relationship already orders rows,
|
|
# but ordering is a contract of this response, so it is stated here too.
|
|
rows = sorted(
|
|
sentences_by_sort.get(position, []),
|
|
key=lambda item: (item.sort, item.id),
|
|
)
|
|
sentences = [SentenceRead.model_validate(row) for row in rows]
|
|
|
|
paragraph = ParagraphRead(
|
|
paper_template_filed_sort=position,
|
|
template_field_id=placement.id if placement is not None else None,
|
|
field_id=field.id if field is not None else None,
|
|
name=field.name if field is not None else None,
|
|
level=field.level if field is not None else 1,
|
|
font_size=float(field.font_size) if field is not None else None,
|
|
font_color=field.font_color if field is not None else None,
|
|
matched=placement is not None,
|
|
sentences=sentences,
|
|
)
|
|
|
|
# Citations are numbered in reading order, so a marker in the text and
|
|
# its entry in the 参考文献 list cannot disagree.
|
|
for sentence in sentences:
|
|
for citation in sentence.citations:
|
|
citation_index += 1
|
|
citations.append(
|
|
PaperCitationRead(
|
|
index=citation_index,
|
|
id=citation.id,
|
|
reference_id=citation.reference_id,
|
|
quote=citation.quote,
|
|
sentence_id=sentence.id,
|
|
sentence_content=sentence.content,
|
|
paper_template_filed_sort=position,
|
|
paragraph_name=paragraph.name,
|
|
)
|
|
)
|
|
|
|
paragraphs.append(paragraph)
|
|
|
|
return PaperDocumentRead(
|
|
paper=read(paper),
|
|
paragraphs=paragraphs,
|
|
citations=citations,
|
|
warnings=warnings,
|
|
)
|
|
|
|
|
|
def find_paragraph(
|
|
document: PaperDocumentRead, field_sort: int
|
|
) -> ParagraphRead | None:
|
|
"""Pick one paragraph out of an assembled document."""
|
|
for paragraph in document.paragraphs:
|
|
if paragraph.paper_template_filed_sort == field_sort:
|
|
return paragraph
|
|
return None
|
|
|
|
|
|
def get_paragraph(db: Session, paper: Paper, field_sort: int) -> ParagraphRead | None:
|
|
"""Return one paragraph, assembled exactly as the document renders it.
|
|
|
|
Assembling the whole document to return one paragraph is not waste: it is
|
|
what guarantees the editor shows precisely what the reader will see, rather
|
|
than a second implementation of the same ordering rules.
|
|
"""
|
|
return find_paragraph(build_document(db, paper), field_sort)
|
|
|
|
|
|
# --- writes ------------------------------------------------------------------
|
|
|
|
|
|
def _touch(db: Session, paper: Paper) -> None:
|
|
"""Mark the paper as edited, on the database clock.
|
|
|
|
Writing content does not itself change any column of ``paper``, so the
|
|
``onupdate`` on ``updated_at`` would not fire and the list — which sorts by
|
|
it — would keep showing the creation time of a paper whose text changed an
|
|
hour ago. ``func.now()`` rather than Python's clock, so timestamps do not
|
|
depend on which host the API runs on.
|
|
"""
|
|
paper.updated_at = func.now() # type: ignore[assignment]
|
|
|
|
|
|
def _commit_touch(db: Session, paper: Paper) -> None:
|
|
"""Commit a content change and refresh the paper's timestamps.
|
|
|
|
The refresh is what turns the ``func.now()`` expression above back into a
|
|
real datetime for the response — MySQL and TiDB have no ``RETURNING``, so
|
|
the value only exists after a re-read.
|
|
"""
|
|
_touch(db, paper)
|
|
db.commit()
|
|
db.refresh(paper)
|
|
|
|
|
|
def _build_citations(items: Sequence[CitationInput]) -> list[PaperSentenceReference]:
|
|
"""Materialise the citation list of one sentence, numbering it from 1."""
|
|
return [
|
|
PaperSentenceReference(
|
|
reference_id=item.reference_id,
|
|
quote=item.quote,
|
|
sort=index,
|
|
)
|
|
for index, item in enumerate(items, start=1)
|
|
]
|
|
|
|
|
|
def _max_sort(db: Session, paper: Paper, field_sort: int) -> int:
|
|
"""The highest sentence sort currently used in one paragraph (0 if empty)."""
|
|
value = db.scalar(
|
|
select(func.coalesce(func.max(PaperSentence.sort), 0)).where(
|
|
PaperSentence.paper_id == paper.id,
|
|
PaperSentence.paper_template_filed_sort == field_sort,
|
|
)
|
|
)
|
|
return int(value or 0)
|
|
|
|
|
|
def create(db: Session, data: PaperCreate) -> Paper:
|
|
"""Insert a paper. No rows are created for its paragraphs.
|
|
|
|
The structure is read from the template on every render, so materialising
|
|
an empty sentence per paragraph would create rows that exist only to say
|
|
nothing — and that would then have to be kept in step with the template.
|
|
"""
|
|
paper = Paper(**data.model_dump())
|
|
db.add(paper)
|
|
db.commit()
|
|
db.refresh(paper)
|
|
return paper
|
|
|
|
|
|
def update(
|
|
db: Session,
|
|
paper: Paper,
|
|
*,
|
|
values: dict[str, Any],
|
|
template_changed: bool,
|
|
) -> Paper:
|
|
"""Apply a partial update.
|
|
|
|
``values`` holds only the keys the client actually sent — the route derives
|
|
it from ``model_fields_set``, which is the only way to tell "clear the
|
|
author" apart from "leave it alone"; both arrive as ``None``.
|
|
|
|
When the template changes, every sentence is re-stamped with the new
|
|
template id. The sentences themselves keep their positions, which is what
|
|
makes a switch non-destructive — and it keeps the pair
|
|
``(template_id, paper_template_filed_sort)`` an honest description of which
|
|
paragraph a sentence belongs to.
|
|
"""
|
|
for key, value in values.items():
|
|
setattr(paper, key, value)
|
|
|
|
if template_changed:
|
|
db.flush()
|
|
db.execute(
|
|
sa_update(PaperSentence)
|
|
.where(PaperSentence.paper_id == paper.id)
|
|
.values(template_id=paper.template_id)
|
|
)
|
|
|
|
_touch(db, paper)
|
|
db.commit()
|
|
db.refresh(paper)
|
|
return paper
|
|
|
|
|
|
def delete(db: Session, paper: Paper) -> None:
|
|
"""Delete a paper and, through the ORM cascade, all of its sentences."""
|
|
db.delete(paper)
|
|
db.commit()
|
|
|
|
|
|
def delete_many(db: Session, paper_ids: Sequence[int]) -> int:
|
|
"""Delete several papers, returning how many actually existed."""
|
|
papers = get_many(db, paper_ids)
|
|
for paper in papers:
|
|
db.delete(paper)
|
|
db.commit()
|
|
return len(papers)
|
|
|
|
|
|
# --- paragraphs and sentences ------------------------------------------------
|
|
|
|
|
|
def replace_paragraph(
|
|
db: Session,
|
|
paper: Paper,
|
|
field_sort: int,
|
|
payload: ParagraphUpdate,
|
|
) -> None:
|
|
"""Rewrite one paragraph's sentences, optionally moving the paragraph.
|
|
|
|
The editor sends the paragraph whole — every line it shows — so this is a
|
|
replacement, not a merge. ``target_sort`` is the manual form of what a
|
|
template switch does by itself: the sentences are written at another
|
|
position and appended after whatever that paragraph already holds, which is
|
|
why the numbering is computed from the target's current maximum rather than
|
|
from 1.
|
|
|
|
Blank lines are kept if the client sends them: an empty sentence is a
|
|
legitimate placeholder and renders as a blank line rather than vanishing.
|
|
"""
|
|
target = payload.target_sort if payload.target_sort is not None else field_sort
|
|
|
|
# Clear the source paragraph first, so that a move onto itself does not
|
|
# count its own rows when working out where the appended block starts.
|
|
existing = list(
|
|
db.scalars(
|
|
select(PaperSentence).where(
|
|
PaperSentence.paper_id == paper.id,
|
|
PaperSentence.paper_template_filed_sort == field_sort,
|
|
)
|
|
).all()
|
|
)
|
|
for row in existing:
|
|
db.delete(row)
|
|
db.flush()
|
|
|
|
base = _max_sort(db, paper, target)
|
|
|
|
for offset, item in enumerate(payload.sentences, start=1):
|
|
db.add(
|
|
PaperSentence(
|
|
paper_id=paper.id,
|
|
# Kept in step with the paper's template: the pair
|
|
# (template_id, position) is what names the paragraph.
|
|
template_id=paper.template_id,
|
|
paper_template_filed_sort=target,
|
|
sort=item.sort if item.sort is not None else base + offset,
|
|
content=item.content,
|
|
citations=_build_citations(item.citations),
|
|
)
|
|
)
|
|
|
|
_commit_touch(db, paper)
|
|
|
|
|
|
def append_sentence(db: Session, paper: Paper, data: SentenceCreate) -> PaperSentence:
|
|
"""Add one sentence to a paragraph, at the end unless told otherwise."""
|
|
sort = (
|
|
data.sort
|
|
if data.sort is not None
|
|
else _max_sort(db, paper, data.paper_template_filed_sort) + 1
|
|
)
|
|
sentence = PaperSentence(
|
|
paper_id=paper.id,
|
|
template_id=paper.template_id,
|
|
paper_template_filed_sort=data.paper_template_filed_sort,
|
|
sort=sort,
|
|
content=data.content,
|
|
citations=_build_citations(data.citations),
|
|
)
|
|
db.add(sentence)
|
|
_commit_touch(db, paper)
|
|
db.refresh(sentence)
|
|
return sentence
|
|
|
|
|
|
def update_sentence(
|
|
db: Session,
|
|
paper: Paper,
|
|
sentence: PaperSentence,
|
|
data: SentenceUpdate,
|
|
) -> PaperSentence:
|
|
"""Apply a partial update to one sentence.
|
|
|
|
``citations`` is a full replacement when present: the sentence's citation
|
|
list is short, and replacing it wholesale is the only way to be sure that a
|
|
deleted citation really is gone.
|
|
"""
|
|
if data.sort is not None:
|
|
sentence.sort = data.sort
|
|
if data.paper_template_filed_sort is not None:
|
|
sentence.paper_template_filed_sort = data.paper_template_filed_sort
|
|
if data.content is not None:
|
|
sentence.content = data.content
|
|
if data.citations is not None:
|
|
sentence.citations.clear()
|
|
db.flush()
|
|
sentence.citations.extend(_build_citations(data.citations))
|
|
|
|
sentence.template_id = paper.template_id
|
|
|
|
_commit_touch(db, paper)
|
|
db.refresh(sentence)
|
|
return sentence
|
|
|
|
|
|
def delete_sentence(db: Session, paper: Paper, sentence: PaperSentence) -> None:
|
|
"""Delete one sentence and its citations."""
|
|
db.delete(sentence)
|
|
_commit_touch(db, paper)
|
|
|
|
|
|
# --- template housekeeping ---------------------------------------------------
|
|
|
|
|
|
def _paper_counts_by_template(
|
|
db: Session, template_ids: Iterable[int]
|
|
) -> dict[int, int]:
|
|
"""How many papers use each template id."""
|
|
wanted = list({template_id for template_id in template_ids})
|
|
if not wanted:
|
|
return {}
|
|
stmt = (
|
|
select(Paper.template_id, func.count(Paper.id))
|
|
.where(Paper.template_id.in_(wanted))
|
|
.group_by(Paper.template_id)
|
|
)
|
|
return {template_id: count for template_id, count in db.execute(stmt).all()}
|
|
|
|
|
|
def template_paper_counts(
|
|
db: Session, template_ids: Sequence[int]
|
|
) -> dict[int, int]:
|
|
"""Public form of :func:`_paper_counts_by_template`.
|
|
|
|
Used by the template endpoints: deleting a template a paper is written
|
|
against would leave that paper with no structure at all, so it is refused
|
|
with the count that explains why.
|
|
"""
|
|
return _paper_counts_by_template(db, template_ids)
|