1"""Choosing the reference solution from the submitted solutions.
3Each submission may assert a partial order among its own solutions through
4``more_accurate_than``. Only orderings within a submission are trusted. A candidate
5reference R is scored by the fraction of asserted pairs (worse, better) from *other*
6submissions for which err(better, R) < err(worse, R) under the primary metric.
7Candidates are the maximal elements of each submission's order. Ties are broken by the
8smallest mean primary error to the other candidates; with no asserted pairs at all this
9tie-break is the whole rule, so the reference is then the medoid of the candidates.
10"""
11from __future__ import annotations
13from dataclasses import dataclass, field
17class OrderInfo:
18 # (worse, better, submission_key) triples over global solution keys "[source/]submission/solution"
19 pairs: list[tuple[str, str, str]] = field(default_factory=list)
20 # global key -> submission id
21 submission_of: dict[str, str] = field(default_factory=dict)
24def collect_order(submissions: list[dict]) -> OrderInfo:
25 info = OrderInfo()
26 for sub in submissions:
27 sid = sub.get("_key", sub["submission_id"])
28 for sol in sub["solutions"]:
29 key = f"{sid}/{sol['solution_id']}"
30 info.submission_of[key] = sid
31 for worse in sol.get("more_accurate_than", []):
32 info.pairs.append((f"{sid}/{worse}", key, sid))
33 return info
36def maximal_elements(info: OrderInfo) -> list[str]:
37 dominated = {worse for worse, _better, _sid in info.pairs}
38 return [k for k in info.submission_of if k not in dominated]
41def select_reference(
42 info: OrderInfo,
43 err: dict[str, dict[str, float]],
44 explicit: str | None = None,
45) -> dict:
46 """Return a report dict with keys: reference, candidates (list of dicts), rule."""
47 if explicit is not None:
48 return {"reference": explicit, "rule": "explicit", "candidates": []}
50 candidates = maximal_elements(info)
51 rows = []
52 for cand in candidates:
53 external = [p for p in info.pairs if p[2] != info.submission_of[cand]]
54 preserved = 0
55 violated = []
56 for worse, better, _sid in external:
57 if err[better][cand] < err[worse][cand]:
58 preserved += 1
59 else:
60 violated.append([worse, better])
61 others = [c for c in candidates if c != cand]
62 mean_err = sum(err[c][cand] for c in others) / len(others) if others else 0.0
63 siblings = [worse for worse, better, _sid in info.pairs if better == cand]
64 gap = min((err[w][cand] for w in siblings), default=None)
65 rows.append(
66 {
67 "candidate": cand,
68 "pairs_total": len(external),
69 "pairs_preserved": preserved,
70 "score": (preserved / len(external)) if external else None,
71 "convergence_gap": gap,
72 "mean_error_to_other_candidates": mean_err,
73 "violated_pairs": violated,
74 }
75 )
77 def sort_key(row):
78 score = row["score"] if row["score"] is not None else -1.0
79 gap = row["convergence_gap"] if row["convergence_gap"] is not None else float("inf")
80 return (-score, gap, row["mean_error_to_other_candidates"])
82 rows.sort(key=sort_key)
83 return {
84 "reference": rows[0]["candidate"] if rows else None,
85 "rule": "partial-order-agreement",
86 "candidates": rows,
87 }