#!/usr/bin/env python3
"""
Re-verify every claim published at https://contextdebt.dev/report against upstream HEAD.

    curl -sO https://contextdebt.dev/claims.json
    curl -sO https://contextdebt.dev/verify.py
    python3 verify.py claims.json

Most claims name the exact file, so this fetches that one file over HTTPS and checks it —
the whole pass takes seconds and downloads a few hundred kilobytes. Only the handful of
claims that have to be searched across a whole repository fall back to a shallow clone,
which needs git; large files are skipped there, so git-lfs is not required.

Two kinds of result, and they mean opposite things:
  PRESENT / GONE_FROM_FILE / FILE_GONE / GONE   an answer about the claim
  CLONE_FAILED / ERROR:*                        an answer about this run — retry, conclude nothing
A claim that no longer holds is not automatically wrong. Upstream may have fixed it, which is
the outcome we want. It does mean the report is stale and we owe a dated correction.
"""
import json, subprocess, tempfile, shutil, os, sys, time, datetime
import urllib.request, urllib.error, urllib.parse

src = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), 'claims.json')
doc = json.load(open(src))
claims = doc['claims'] if isinstance(doc, dict) else doc

# Skip LFS payloads: every claim is a string in a text file, and pulling media would only
# make this slower and require git-lfs to be installed.
env = dict(os.environ, GIT_LFS_SKIP_SMUDGE='1', GIT_TERMINAL_PROMPT='0')

RAW = 'https://raw.githubusercontent.com'


def fetch_file(repo, path):
    """One file over HTTPS. Returns (status, body). Status is about the claim only when
    GitHub actually answered; anything else is about this run."""
    url = f"{RAW}/{repo}/HEAD/{urllib.parse.quote(path)}"
    req = urllib.request.Request(url, headers={'User-Agent': 'contextdebt-verify'})
    try:
        with urllib.request.urlopen(req, timeout=45) as r:
            return 'OK', r.read().decode('utf8', 'replace')
    except urllib.error.HTTPError as e:
        return ('FILE_GONE', '') if e.code == 404 else (f'ERROR:HTTP{e.code}', '')
    except Exception as e:
        return f'ERROR:{e.__class__.__name__}', ''


out = []
for c in claims:
    d = tempfile.mkdtemp()
    r = os.path.join(d, 'r')
    st = 'ERROR:unset'
    t0 = time.time()

    if c.get('file'):
        st, body = fetch_file(c['repo'], c['file'])
        if st == 'OK':
            st = 'PRESENT' if c['needle'] in body else 'GONE_FROM_FILE'
        if not st.startswith('ERROR'):
            shutil.rmtree(d, ignore_errors=True)
            secs = time.time() - t0
            out.append(dict(id=c['id'], repo=c['repo'], kind=c.get('kind'),
                            pr=c.get('pr') or '', status=st))
            print(f"{c['id']:<26}{c['repo']:<34}{st}", flush=True)
            continue
        # the fetch failed rather than answered — fall through to a clone

    try:
        cmd = ['git',
               '-c', 'filter.lfs.smudge=cat', '-c', 'filter.lfs.process=',
               '-c', 'filter.lfs.required=false',
               # git will wait forever on a stalled transfer; give up instead.
               '-c', 'http.lowSpeedLimit=1000', '-c', 'http.lowSpeedTime=45',
               'clone', '-q', '--depth', '1', '--single-branch']
        # When we know the file, fetch blobs on demand: a monorepo costs one file
        # instead of a few hundred megabytes. Claims that grep the whole tree need
        # the blobs anyway, so they clone normally.
        if c.get('file'):
            cmd += ['--filter=blob:none']
        cmd += [f"https://github.com/{c['repo']}", r]
        t0 = time.time()
        p = subprocess.run(cmd, timeout=600, env=env,
                           stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
        if p.returncode != 0 or not os.path.isdir(r):
            st = 'CLONE_FAILED'
        else:
            f = c.get('file') or ''
            if f:
                path = os.path.join(r, f)
                if not os.path.exists(path):
                    st = 'FILE_GONE'
                else:
                    body = open(path, encoding='utf8', errors='ignore').read()
                    st = 'PRESENT' if c['needle'] in body else 'GONE_FROM_FILE'
            else:
                g = subprocess.run(['grep', '-rlF', '--', c['needle'], r],
                                   capture_output=True, text=True)
                st = 'PRESENT' if g.stdout.strip() else 'GONE'
    except subprocess.TimeoutExpired:
        st = 'ERROR:Timeout'
    except Exception as e:
        st = f'ERROR:{e.__class__.__name__}'
    finally:
        shutil.rmtree(d, ignore_errors=True)
    secs = time.time() - t0 if 't0' in dir() else 0
    out.append(dict(id=c['id'], repo=c['repo'], kind=c.get('kind'), pr=c.get('pr') or '', status=st))
    slow = f"  {secs:.0f}s" if secs >= 20 else ""
    print(f"{c['id']:<26}{c['repo']:<34}{st}{slow}", flush=True)

stamp = datetime.datetime.now(datetime.timezone.utc).date().isoformat()
json.dump(dict(verified_on=stamp, results=out),
          open(os.path.join(os.path.dirname(os.path.abspath(src)), 'verified.json'), 'w'), indent=1)

held = [o for o in out if o['status'] == 'PRESENT']
moved = [o for o in out if o['status'] in ('GONE', 'GONE_FROM_FILE', 'FILE_GONE')]
broke = [o for o in out if o['status'].startswith(('CLONE_FAILED', 'ERROR'))]
print()
print(f"verified_on {stamp} (UTC) — {len(held)}/{len(out)} hold")
for o in moved:
    print("  MOVED — upstream changed this line:", o['id'], o['status'])
for o in broke:
    print("  NOT CHECKED — this run failed, not the claim:", o['id'], o['status'])
if broke:
    print("  Re-run before concluding anything about these.")