/**
 * Verification — US SLED Contact Directory.
 *
 *   node verify.mjs
 *
 * This asserts that the delivered, public-record dataset (dataset/) is complete,
 * correctly grouped, integrity-checksummed against its provenance manifest, and
 * — critically — that the four fields the source cannot supply (contact name,
 * title, email, domain) are EMPTY for every record (proving nothing was
 * fabricated). It also rebuilds the standalone site and confirms it embeds the
 * full directory. Verification is deterministic and offline; the proof that the
 * source is real and live lives in dataset/manifest.json + evidence/ (captured
 * by src/collect.mjs) and is reproducible via proof/REPRODUCE.md.
 *
 * Exit code is non-zero if any MUST_PASS check fails.
 */
import { createHash } from 'node:crypto';
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { JURISDICTIONS } from './src/fips.mjs';
import { COLUMNS, SEAM_FIELDS, SOURCE_NAME, groupByState } from './src/schema.mjs';

const DIR = dirname(fileURLToPath(import.meta.url));
const DATASET = join(DIR, 'dataset');
const EVIDENCE = join(DIR, 'evidence');
mkdirSync(join(DIR, 'evidence'), { recursive: true });

const sha256 = (s) => createHash('sha256').update(s).digest('hex');
const checks = [];
const check = (name, detail, passed) => checks.push({ name, detail, passed: !!passed });
const readJSON = (p) => JSON.parse(readFileSync(p, 'utf8'));

/* ---------- load delivered artifacts ---------- */
const contactsRaw = readFileSync(join(DATASET, 'contacts.json'), 'utf8');
const contacts = JSON.parse(contactsRaw);
const csvRaw = readFileSync(join(DATASET, 'contacts.csv'), 'utf8');
const manifest = readJSON(join(DATASET, 'manifest.json'));
const summary = readJSON(join(DATASET, 'summary.json'));
const fetchLog = readJSON(join(EVIDENCE, 'fetch-log.json'));

const expectedStates = new Set(JURISDICTIONS.map((j) => j.usps));
const grouped = groupByState(contacts);

/* ---------- 1. completeness ---------- */
check('Every record loads and totals agree across dataset + summary + manifest',
  `contacts=${contacts.length}, summary=${summary.totalOrganizations}, manifest=${manifest.totalOrganizations}`,
  contacts.length > 0 && contacts.length === summary.totalOrganizations && contacts.length === manifest.totalOrganizations);

check('All 50 states + DC are present (51 jurisdictions), no extras',
  `states=${grouped.size}, expected=${expectedStates.size}`,
  grouped.size === expectedStates.size && [...grouped.keys()].every((s) => expectedStates.has(s)));

const missingStates = [...expectedStates].filter((s) => !grouped.has(s) || grouped.get(s).length === 0);
check('Every jurisdiction has at least one organization', `emptyStates=${missingStates.length}`, missingStates.length === 0);

/* ---------- 2. real fields populated ---------- */
const REAL_REQUIRED = ['state', 'organizationName', 'organizationType', 'phone', 'source', 'sourceUrl', 'datasetYear'];
const incompleteReal = contacts.filter((r) => REAL_REQUIRED.some((f) => r[f] === '' || r[f] == null));
check('Every record has the real required fields (name, type, phone, state, provenance)',
  `incomplete=${incompleteReal.length}/${contacts.length}`, incompleteReal.length === 0);

const withPhone = contacts.filter((r) => r.phone && /\d/.test(r.phone)).length;
const phonePct = (withPhone / contacts.length) * 100;
check('Phone-number coverage is at least 99%', `${phonePct.toFixed(1)}% (${withPhone}/${contacts.length})`, phonePct >= 99);

/* ---------- 3. NO FABRICATION: seam fields empty everywhere ---------- */
const fabricated = contacts.filter((r) => SEAM_FIELDS.some((f) => String(r[f] || '').trim() !== ''));
check('DISCLOSED SEAMS are empty for every record (no fabricated name/title/email/domain)',
  `seamFields=[${SEAM_FIELDS.join(', ')}], nonEmpty=${fabricated.length}`, fabricated.length === 0);

/* ---------- 4. provenance integrity ---------- */
const officialHost = 'https://educationdata.urban.org/';
const badSource = contacts.filter((r) => r.source !== SOURCE_NAME || !String(r.sourceUrl).startsWith(officialHost));
check('Every record carries the official source name + a source URL on the NCES/Urban host',
  `badSource=${badSource.length}`, badSource.length === 0);

const jsonShaOk = sha256(contactsRaw) === manifest.contactsJsonSha256;
const csvShaOk = sha256(csvRaw) === manifest.contactsCsvSha256;
check('dataset/contacts.json matches the sha256 recorded in the provenance manifest', `match=${jsonShaOk}`, jsonShaOk);
check('dataset/contacts.csv matches the sha256 recorded in the provenance manifest', `match=${csvShaOk}`, csvShaOk);

const allOk200 = fetchLog.requests.length === expectedStates.size && fetchLog.requests.every((r) => r.httpStatus === 200);
check('Fetch log shows one HTTP 200 request per jurisdiction (51/51)',
  `requests=${fetchLog.requests.length}, non200=${fetchLog.requests.filter((r) => r.httpStatus !== 200).length}`, allOk200);

/* ---------- 5. grouping + CSV consistency ---------- */
const groupSum = [...grouped.values()].reduce((a, l) => a + l.length, 0);
check('Per-state grouping partitions every record exactly once', `sum=${groupSum} == ${contacts.length}`, groupSum === contacts.length);

let perStateFilesOk = true; let perStateDetail = '';
for (const j of JURISDICTIONS) {
  const p = join(DATASET, 'by-state', `${j.usps}.json`);
  if (!existsSync(p)) { perStateFilesOk = false; perStateDetail = `missing ${j.usps}.json`; break; }
  const recs = readJSON(p);
  if (recs.length !== (grouped.get(j.usps) || []).length) { perStateFilesOk = false; perStateDetail = `${j.usps} count mismatch`; break; }
}
check('A per-state JSON file exists for all 51 jurisdictions with matching counts', perStateDetail || 'all match', perStateFilesOk);

const csvLines = csvRaw.replace(/\n$/, '').split('\n');
const csvHeaderOk = csvLines[0] === COLUMNS.join(',');
const csvRowsOk = csvLines.length - 1 === contacts.length;
check('CSV header equals the canonical schema and row count equals record count',
  `header=${csvHeaderOk}, rows=${csvLines.length - 1}/${contacts.length}`, csvHeaderOk && csvRowsOk);

/* ---------- 6. determinism ---------- */
const a = JSON.stringify(groupByState(contacts).get('CA').map((r) => r.organizationName));
const b = JSON.stringify(groupByState(contacts).get('CA').map((r) => r.organizationName));
const sortedGlobally = contacts.every((r, i) => i === 0 || contacts[i - 1].state < r.state ||
  (contacts[i - 1].state === r.state && contacts[i - 1].organizationName.localeCompare(r.organizationName) <= 0));
check('Grouping is deterministic and the dataset is canonically sorted (state, then name)',
  `stable=${a === b}, globallySorted=${sortedGlobally}`, a === b && sortedGlobally);

/* ---------- 7. retained raw evidence ---------- */
let rawEvidenceOk = true; let rawDetail = '';
for (const st of ['AK', 'DC']) {
  const p = join(EVIDENCE, `raw-sample-${st}.json`);
  if (!existsSync(p)) { rawEvidenceOk = false; rawDetail = `missing raw-sample-${st}.json`; break; }
  const raw = readJSON(p);
  if (raw.length !== (grouped.get(st) || []).length) { rawEvidenceOk = false; rawDetail = `${st} raw count mismatch (${raw.length} vs ${grouped.get(st).length})`; break; }
}
check('Untouched raw API responses are retained as evidence and match the delivered counts', rawDetail || 'AK + DC match', rawEvidenceOk);

/* ---------- 8. site builds + embeds the directory ---------- */
let siteBuilt = false, siteEmbeds = false, siteBytes = 0;
try {
  execFileSync('node', ['build-site.mjs'], { cwd: DIR, stdio: 'pipe' });
  const sitePath = join(DIR, 'public', 'index.html');
  if (existsSync(sitePath)) {
    const html = readFileSync(sitePath, 'utf8');
    siteBytes = Buffer.byteLength(html);
    siteBuilt = true;
    siteEmbeds = html.includes(String(contacts.length)) && /Grouped by state/i.test(html) && html.includes('Anchorage School District');
  }
} catch { siteBuilt = false; }
check('Standalone site builds to public/index.html', `built=${siteBuilt}, ${(siteBytes / 1024).toFixed(0)} KB`, siteBuilt);
check('Site embeds the full directory (record count + state grouping + a known org)', `embeds=${siteEmbeds}`, siteEmbeds);

/* ============================== report ============================== */
const passed = checks.filter((c) => c.passed).length;
const total = checks.length;
const allPass = passed === total;

const disclosedSeams = [
  `DISCLOSED SEAM — no named contact: the source (${SOURCE_NAME}) does not publish a named individual per organization, so the "contactName" column is empty for all ${contacts.length} records. It is never fabricated.`,
  'DISCLOSED SEAM — no title: there is no per-person job title in the source; the "contactTitle" column is empty for every record.',
  'DISCLOSED SEAM — no email: the source publishes no email addresses; the "email" column is empty for every record. Emails are not invented.',
  'DISCLOSED SEAM — no web domain: the source publishes no website/domain field; the "domain" column is empty for every record.',
  'SCOPE SEAM — education sector only: "SLED" here is realized as the Education layer (public K-12 school districts / LEAs). State agencies, county/municipal/local governments, and higher-ed institutions are NOT included in this build (see design/CAPABILITY_GAP.json capability seams).',
  `DATASET VINTAGE: data is the NCES CCD LEA Directory for school year ${manifest.datasetYear}; it reflects that collection cycle, not today.`,
];

const report = {
  outcome: 'us-sled-contact-directory',
  title: 'US SLED (Education) Organization Contact Directory — all 50 states + DC, grouped by state',
  strictness: 'IRS_AUDITOR',
  generatedAt: new Date().toISOString(),
  proofStatus: 'completeness + grouping + provenance-integrity + no-fabrication of a real public-record SLED directory; live source captured with per-request provenance',
  passed,
  total,
  passRate: passed / total,
  status: allPass ? 'PASS' : 'FAIL',
  evaluatedOn: `NCES CCD LEA Directory public-record dataset ${manifest.datasetYear} (dataset/contacts.json), offline snapshot with provenance manifest + retained raw evidence`,
  manualIntervention: false,
  disclosedSeams,
  metrics: {
    totalOrganizations: contacts.length,
    jurisdictions: grouped.size,
    statesPlusDc: expectedStates.size,
    phoneCoveragePct: Math.round(phonePct * 10) / 10,
    realColumns: COLUMNS.length - SEAM_FIELDS.length,
    seamColumns: SEAM_FIELDS.length,
    fabricatedValues: fabricated.length,
    unsupportedSourceRecords: badSource.length,
    datasetYear: manifest.datasetYear,
    contactsJsonBytes: Buffer.byteLength(contactsRaw),
    contactsCsvBytes: Buffer.byteLength(csvRaw),
  },
  checks,
};

writeFileSync(join(DIR, 'verification-report.json'), JSON.stringify(report, null, 2) + '\n');
mkdirSync(join(DIR, 'evidence'), { recursive: true });
writeFileSync(join(DIR, 'evidence', 'verification-results.json'), JSON.stringify(report, null, 2) + '\n');

const md = [
  '# Verification Report — US SLED Contact Directory',
  '',
  `> Real public-record data from the **${SOURCE_NAME}**. Verification asserts the directory is complete, correctly grouped by state, integrity-checksummed to its provenance, and that the four fields the source cannot supply (contact name, title, email, domain) are **empty for every record** — never fabricated.`,
  '',
  `**Strictness:** IRS_AUDITOR  |  **Status:** ${report.status} ${passed}/${total} (${Math.round(report.passRate * 100)}%)  |  **Generated:** ${report.generatedAt}`,
  '',
  '## Disclosed seams & limitations',
  '',
  ...disclosedSeams.map((s) => `- ${s}`),
  '',
  '## Directory metrics',
  '',
  '| Metric | Value |',
  '| --- | --- |',
  ...Object.entries(report.metrics).map(([k, v]) => `| ${k} | ${v} |`),
  '',
  '## Checks',
  '',
  '| Check | Detail | Result |',
  '| --- | --- | --- |',
  ...checks.map((c) => `| ${c.name} | ${c.detail} | ${c.passed ? 'PASS' : 'FAIL'} |`),
  '',
].join('\n');
writeFileSync(join(DIR, 'verification-report.md'), md + '\n');

console.log(`[verify] ${report.status}: ${passed}/${total} checks (${Math.round(report.passRate * 100)}%).`);
for (const c of checks) console.log(`  ${c.passed ? 'PASS' : 'FAIL'}  ${c.name}  [${c.detail}]`);
if (!allPass) process.exitCode = 1;
