Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file added CHANGELOG.md
Empty file.
23 changes: 21 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,31 @@ NB.

## Testing against ground truth

Ground truth dataset is stored in `data/ground_truth`, so run the following to obtain the accuracy of the model in `./reports`
Ground truth lives in `data/ground_truth` as `{"text": ..., "ground_truth": [{"itext",
"a", "b", "entity_id"}, ...]}`. `pelinker-link-files` scores against it automatically
whenever the input carries a `ground_truth` block:

```commandline
python run/testing/run_pel_test.py --text-path ./data/ground_truth/sample.0.gt.json --model-type biobert-stsb --layers-spec sent --extra-context
uv run pelinker-link-files -m models/pelinker.pubmedbert.run1 \
-o reports/gt_score.json data/ground_truth/sample.0.gt.json
```

The output JSON gains a `ground_truth_score` block:

- **Detection** — `precision` / `recall` / `f1` over character spans, matched by overlap
within a document. Unambiguous and comparable across model versions.
- **Entity accuracy** — over matched spans only, and only where the ids are comparable.
Since the KB-out work the linker predicts *minted cluster ids* (`kb::C0007`) while the
gold file carries *input KB* ids (`PEL.000032`), so `n_id_comparable` may be 0 and
`entity_accuracy` `null`. That means undefined, not zero — read the detection numbers.

Add `--kb-validation` for a `kb_lemma_validation` block: the rate at which a mention's
predicted entity agrees with the entity its own lemma resolves to in the KB. That is a
distant-supervision consistency check, not end-task accuracy.

Programmatic entry points: `pelinker.ground_truth.score_predictions_against_ground_truth`
and `pelinker.linker_kb_lemma.aggregate_kb_lemma_validation`.

## Serialize Model

"Train" a model on a corpus
Expand Down
131 changes: 131 additions & 0 deletions pelinker/analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import math
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace

import numpy as np
import pandas as pd
Expand All @@ -11,6 +12,7 @@
aggregate_grid_metrics,
solve_optimal_min_cluster_size_from_aggregated,
)
from pelinker.grid_export import per_combo_metrics_from_grid
from pelinker.reporting import (
AllScreenerCvResult,
BinaryClassifierMetrics,
Expand All @@ -31,6 +33,7 @@

from pelinker.config import (
ClusteringOptimizationConfig,
GridObjectiveSpec,
ManifoldOovScreenerConfig,
NegativeScreenerConfig,
)
Expand Down Expand Up @@ -223,6 +226,134 @@ def pooled_min_cluster_size_from_metrics_dfs(
return solved.chosen_min_cluster_size, solved.score_mean_at_chosen


def grid_solver_config(
optimization_config: ClusteringOptimizationConfig | None,
*,
grid_cluster_count_reward: float | None = None,
grid_n_entities: int | None = None,
grid_objective: GridObjectiveSpec | None = None,
optimization_method: str | None = None,
) -> ClusteringOptimizationConfig:
"""Merge optional solver overrides into ``optimization_config`` (or defaults)."""
base = optimization_config or ClusteringOptimizationConfig()
overrides: dict[str, object] = {}
if grid_cluster_count_reward is not None:
overrides["grid_cluster_count_reward"] = grid_cluster_count_reward
if grid_n_entities is not None:
overrides["grid_n_entities"] = grid_n_entities
if grid_objective is not None:
overrides["grid_objective"] = grid_objective
if optimization_method is not None:
overrides["optimization_method"] = optimization_method
return replace(base, **overrides) if overrides else base


def grid_solver_overrides_active(
*,
optimization_config: ClusteringOptimizationConfig | None,
grid_cluster_count_reward: float | None,
grid_n_entities: int | None,
grid_objective: GridObjectiveSpec | None,
optimization_method: str | None,
) -> bool:
"""True when any solver knob was supplied, i.e. a re-solve was explicitly requested."""
return any(
v is not None
for v in (
optimization_config,
grid_cluster_count_reward,
grid_n_entities,
grid_objective,
optimization_method,
)
)


def should_resolve_chosen_min_cluster_size(
*,
chosen_min_cluster_size: float | None,
optimization_config: ClusteringOptimizationConfig | None,
grid_cluster_count_reward: float | None,
grid_n_entities: int | None,
grid_objective: GridObjectiveSpec | None,
optimization_method: str | None,
) -> bool:
"""Re-solve only when no chosen value is on hand *and* solver knobs were supplied."""
if chosen_min_cluster_size is not None:
return False
return grid_solver_overrides_active(
optimization_config=optimization_config,
grid_cluster_count_reward=grid_cluster_count_reward,
grid_n_entities=grid_n_entities,
grid_objective=grid_objective,
optimization_method=optimization_method,
)


def solve_pooled_grid_from_metrics_list(
metrics_list: list[pd.DataFrame],
optimization_config: ClusteringOptimizationConfig | None = None,
*,
grid_cluster_count_reward: float | None = None,
grid_n_entities: int | None = None,
grid_objective: GridObjectiveSpec | None = None,
optimization_method: str | None = None,
) -> SmoothedGridOptimumResult:
"""Pooled grid solve on per-sample metric tables; returns full diagnostics."""
cfg = grid_solver_config(
optimization_config,
grid_cluster_count_reward=grid_cluster_count_reward,
grid_n_entities=grid_n_entities,
grid_objective=grid_objective,
optimization_method=optimization_method,
)
return pooled_grid_solve_from_metrics_dfs(metrics_list, cfg)


def solve_pooled_grid_by_combo_from_grid(
df_grid: pd.DataFrame,
optimization_config: ClusteringOptimizationConfig | None = None,
*,
grid_cluster_count_reward: float | None = None,
grid_n_entities: int | None = None,
grid_objective: GridObjectiveSpec | None = None,
optimization_method: str | None = None,
) -> dict[tuple[str, str], SmoothedGridOptimumResult]:
"""Pooled grid solve per (model, layer) from a grid export CSV frame."""
cfg = grid_solver_config(
optimization_config,
grid_cluster_count_reward=grid_cluster_count_reward,
grid_n_entities=grid_n_entities,
grid_objective=grid_objective,
optimization_method=optimization_method,
)
return {
combo: pooled_grid_solve_from_metrics_dfs(metrics_list, cfg)
for combo, metrics_list in per_combo_metrics_from_grid(df_grid).items()
}


def resolve_chosen_min_cluster_size_by_combo_from_grid(
df_grid: pd.DataFrame,
optimization_config: ClusteringOptimizationConfig | None = None,
*,
grid_cluster_count_reward: float | None = None,
grid_n_entities: int | None = None,
grid_objective: GridObjectiveSpec | None = None,
optimization_method: str | None = None,
) -> dict[tuple[str, str], int]:
"""Re-solve ``chosen_min_cluster_size`` per (model, layer) from a grid export CSV frame."""
solved = solve_pooled_grid_by_combo_from_grid(
df_grid,
optimization_config,
grid_cluster_count_reward=grid_cluster_count_reward,
grid_n_entities=grid_n_entities,
grid_objective=grid_objective,
optimization_method=optimization_method,
)
return {combo: result.chosen_min_cluster_size for combo, result in solved.items()}


def split_by_negative_label(
dfr: pd.DataFrame,
negative_label: str,
Expand Down
129 changes: 129 additions & 0 deletions pelinker/checkpoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""Shared machinery for resumable search checkpoints.

``model_selection``, ``dim_selection`` and ``scale_curve`` each keep their own
checkpoint dataclass, because their keys genuinely differ (a fused ``combination``, a
``(pca, umap)`` cell, a sample-size rung) and their on-disk field names must stay
stable for already-written state files. Everything that does *not* depend on those
names lives here: the version constant, the timestamp, the run fingerprint, the
atomic gzip-aware read/write, and the fingerprint fields all three searches share.
"""

from __future__ import annotations

import gzip
import json
import pathlib
from dataclasses import dataclass
from datetime import datetime, timezone
from hashlib import sha256
from typing import Any

from pelinker.io.json_files import is_gzip_file_path, load_json_path
from pelinker.onto import NEGATIVE_LABEL

CHECKPOINT_VERSION = 1


@dataclass
class FailureRecord:
"""One recorded failure, keyed by whatever the search iterates over."""

key: str
error: str
at: str


def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()


def compute_run_fingerprint(config: dict[str, Any]) -> str:
"""sha256 of the canonicalized run config; a change invalidates the checkpoint."""
blob = json.dumps(config, sort_keys=True, separators=(",", ":")).encode("utf-8")
return sha256(blob).hexdigest()


def require_checkpoint_version(data: dict[str, Any]) -> None:
if int(data.get("version", -1)) != CHECKPOINT_VERSION:
raise ValueError(
f"Unsupported checkpoint version: {data.get('version')!r}; "
f"expected {CHECKPOINT_VERSION}"
)


def read_checkpoint_json(path: pathlib.Path) -> dict[str, Any]:
"""Load a checkpoint payload (plain or gzipped) and assert it is an object."""
data = load_json_path(path)
if not isinstance(data, dict):
raise ValueError("checkpoint must be a JSON object")
return data


def write_checkpoint_json_atomic(path: pathlib.Path, payload: dict[str, Any]) -> None:
"""Write via ``*.tmp`` + ``replace`` so a crash never leaves a torn checkpoint."""
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
text = json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
if is_gzip_file_path(path):
with gzip.open(tmp, "wt", encoding="utf-8", newline="\n") as gz:
gz.write(text)
else:
tmp.write_text(text, encoding="utf-8")
tmp.replace(path)


def shared_fingerprint_fields(
*,
cluster_viz_method: str,
min_class_size: int,
seed: int,
pca_seed: int,
umap_seed: int | None,
clustering_sample_rows: int | None,
batch_size: int,
n_sample: int,
selected_labels_kb_path: pathlib.Path | None,
max_scale: int,
min_scale: int | None = None,
clustering_grid_step: int = 5,
negative_label: str = NEGATIVE_LABEL,
screener_kind: str = "lda",
drop_rare_entities: bool = False,
min_mentions_per_entity: int = 20,
max_mentions_per_entity: int | None = None,
max_mentions_negative: int | None = None,
mention_cap_seed: int = 13,
) -> dict[str, Any]:
"""The fingerprint fields common to every search; callers add their own on top.

``min_scale`` is resolved here to the same default
:meth:`~pelinker.config.ClusteringOptimizationConfig.resolved_min_scale` uses, so
an explicit ``min_scale`` equal to the default does not change the fingerprint.
"""
kb = None
if selected_labels_kb_path is not None:
kb = str(selected_labels_kb_path.expanduser().resolve())
resolved_min_scale = (
min_scale if min_scale is not None else max(1, min_class_size // 2)
)
return {
"batch_size": batch_size,
"clustering_grid_step": clustering_grid_step,
"clustering_sample_rows": clustering_sample_rows,
"cluster_viz_method": cluster_viz_method,
"drop_rare_entities": drop_rare_entities,
"max_mentions_negative": max_mentions_negative,
"max_mentions_per_entity": max_mentions_per_entity,
"max_scale": max_scale,
"mention_cap_seed": mention_cap_seed,
"min_class_size": min_class_size,
"min_mentions_per_entity": min_mentions_per_entity,
"min_scale": resolved_min_scale,
"n_sample": n_sample,
"negative_label": negative_label,
"pca_seed": pca_seed,
"screener_kind": screener_kind,
"seed": seed,
"selected_labels_kb_path": kb,
"umap_seed": umap_seed,
}
Loading