"""Choosing the reference solution from the submitted solutions. Each submission may assert a partial order among its own solutions through ``more_accurate_than``. Only orderings within a submission are trusted. A candidate reference R is scored by the fraction of asserted pairs (worse, better) from *other* submissions for which err(better, R) < err(worse, R) under the primary metric. Candidates are the maximal elements of each submission's order. Ties are broken by the smallest mean primary error to the other candidates; with no asserted pairs at all this tie-break is the whole rule, so the reference is then the medoid of the candidates. """ from __future__ import annotations from dataclasses import dataclass, field @dataclass class OrderInfo: # (worse, better, submission_id) triples over global solution keys "submission/solution" pairs: list[tuple[str, str, str]] = field(default_factory=list) # global key -> submission id submission_of: dict[str, str] = field(default_factory=dict) def collect_order(submissions: list[dict]) -> OrderInfo: info = OrderInfo() for sub in submissions: sid = sub["submission_id"] for sol in sub["solutions"]: key = f"{sid}/{sol['solution_id']}" info.submission_of[key] = sid for worse in sol.get("more_accurate_than", []): info.pairs.append((f"{sid}/{worse}", key, sid)) return info def maximal_elements(info: OrderInfo) -> list[str]: dominated = {worse for worse, _better, _sid in info.pairs} return [k for k in info.submission_of if k not in dominated] def select_reference( info: OrderInfo, err: dict[str, dict[str, float]], explicit: str | None = None, ) -> dict: """Return a report dict with keys: reference, candidates (list of dicts), rule.""" if explicit is not None: return {"reference": explicit, "rule": "explicit", "candidates": []} candidates = maximal_elements(info) rows = [] for cand in candidates: external = [p for p in info.pairs if p[2] != info.submission_of[cand]] preserved = 0 violated = [] for worse, better, _sid in external: if err[better][cand] < err[worse][cand]: preserved += 1 else: violated.append([worse, better]) others = [c for c in candidates if c != cand] mean_err = sum(err[c][cand] for c in others) / len(others) if others else 0.0 siblings = [worse for worse, better, _sid in info.pairs if better == cand] gap = min((err[w][cand] for w in siblings), default=None) rows.append( { "candidate": cand, "pairs_total": len(external), "pairs_preserved": preserved, "score": (preserved / len(external)) if external else None, "convergence_gap": gap, "mean_error_to_other_candidates": mean_err, "violated_pairs": violated, } ) def sort_key(row): score = row["score"] if row["score"] is not None else -1.0 gap = row["convergence_gap"] if row["convergence_gap"] is not None else float("inf") return (-score, gap, row["mean_error_to_other_candidates"]) rows.sort(key=sort_key) return { "reference": rows[0]["candidate"] if rows else None, "rule": "partial-order-agreement", "candidates": rows, }