/**
 * Collector — builds the SLED contact directory from public-record data.
 *
 *   node src/collect.mjs            # latest available year, all 50 states + DC
 *   node src/collect.mjs --year 2023
 *
 * Source: NCES Common Core of Data (CCD) "LEA Directory", served as JSON by the
 * Urban Institute Education Data Portal (a public, key-free API that mirrors the
 * official US Department of Education NCES dataset). Every public school
 * district / education agency in each state is a SLED organization.
 *
 * For each jurisdiction the collector:
 *   - fetches the directory (following pagination),
 *   - records the exact request URL, HTTP status, row count, a sha256 of the
 *     raw response body, and a timestamp (provenance),
 *   - keeps the full raw JSON for two sample states as retained evidence,
 *   - normalizes rows into the canonical contact schema (src/schema.mjs).
 *
 * Outputs (all under dataset/ so the Proof Layer checksums them):
 *   dataset/contacts.json         every record, all states
 *   dataset/contacts.csv          same, CSV
 *   dataset/by-state/<ST>.json    per-state records
 *   dataset/by-state/<ST>.csv     per-state CSV
 *   dataset/summary.json          per-state counts + totals
 *   dataset/manifest.json         provenance: source, per-state URL/status/sha256
 * Retained raw evidence:
 *   evidence/raw-sample-<ST>.json full untouched API response for sample states
 *   evidence/fetch-log.json       machine log of every request
 */
import { createHash } from 'node:crypto';
import { writeFileSync, mkdirSync, rmSync, existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { JURISDICTIONS } from './fips.mjs';
import { toContact, toCsv, groupByState, COLUMNS, SOURCE_NAME, SEAM_FIELDS } from './schema.mjs';

const DIR = join(dirname(fileURLToPath(import.meta.url)), '..');
const DATASET = join(DIR, 'dataset');
const BYSTATE = join(DATASET, 'by-state');
const EVIDENCE = join(DIR, 'evidence');

const argv = process.argv.slice(2);
const argYear = (() => { const i = argv.indexOf('--year'); return i >= 0 ? Number(argv[i + 1]) : null; })();
const YEAR_CANDIDATES = argYear ? [argYear] : [2023, 2022, 2021];
const API = 'https://educationdata.urban.org/api/v1/school-districts/ccd/directory';
const SAMPLE_STATES = new Set(['AK', 'DC']); // raw responses retained as evidence
const sha256 = (s) => createHash('sha256').update(s).digest('hex');
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function getJson(url, tries = 4) {
  let lastErr;
  for (let attempt = 1; attempt <= tries; attempt++) {
    try {
      const res = await fetch(url, { headers: { Accept: 'application/json', 'User-Agent': 'forge-sled-collector/1.0 (+public-record CCD directory)' } });
      const text = await res.text();
      if (!res.ok) throw new Error(`HTTP ${res.status}`);
      return { status: res.status, text, json: JSON.parse(text) };
    } catch (e) {
      lastErr = e;
      await sleep(500 * attempt);
    }
  }
  throw lastErr;
}

/** Resolve the newest year that actually returns data (probe with DC=11). */
async function resolveYear() {
  for (const y of YEAR_CANDIDATES) {
    try {
      const { json } = await getJson(`${API}/${y}/?fips=11`);
      if (json && json.count > 0) return y;
    } catch { /* try next */ }
  }
  throw new Error('No CCD directory year returned data: ' + YEAR_CANDIDATES.join(','));
}

async function fetchJurisdiction(j, year) {
  const retrievedAt = new Date().toISOString();
  const firstUrl = `${API}/${year}/?fips=${j.fips}`;
  let url = firstUrl;
  const rawRows = [];
  let pages = 0;
  let firstBody = null;
  let status = 0;
  while (url) {
    const { status: s, text, json } = await getJson(url);
    status = s;
    if (firstBody == null) firstBody = text;
    pages += 1;
    if (Array.isArray(json.results)) rawRows.push(...json.results);
    url = json.next || null;
    if (url) await sleep(120);
  }
  const records = rawRows.map((r) => toContact(r, j, firstUrl, year, retrievedAt));
  return {
    jurisdiction: j,
    sourceUrl: firstUrl,
    httpStatus: status,
    pages,
    rawCount: rawRows.length,
    records,
    rawBodySha256: firstBody ? sha256(firstBody) : null,
    rawSample: SAMPLE_STATES.has(j.usps) ? rawRows : null,
    retrievedAt,
  };
}

async function main() {
  const startedAt = new Date().toISOString();
  rmSync(DATASET, { recursive: true, force: true });
  mkdirSync(BYSTATE, { recursive: true });
  mkdirSync(EVIDENCE, { recursive: true });

  const year = await resolveYear();
  console.log(`[collect] source: ${SOURCE_NAME}`);
  console.log(`[collect] dataset year: ${year}; jurisdictions: ${JURISDICTIONS.length}`);

  const all = [];
  const fetchLog = [];
  const perState = [];

  for (const j of JURISDICTIONS) {
    process.stdout.write(`  ${j.usps} (fips ${j.fips}) ... `);
    const r = await fetchJurisdiction(j, year);
    all.push(...r.records);
    perState.push({ state: j.usps, stateName: j.name, fips: j.fips, organizations: r.records.length });
    fetchLog.push({
      state: j.usps, fips: j.fips, sourceUrl: r.sourceUrl, httpStatus: r.httpStatus,
      pages: r.pages, records: r.rawCount, rawBodySha256: r.rawBodySha256, retrievedAt: r.retrievedAt,
    });
    if (r.rawSample) writeFileSync(join(EVIDENCE, `raw-sample-${j.usps}.json`), JSON.stringify(r.rawSample, null, 2) + '\n');
    console.log(`${r.records.length} orgs (HTTP ${r.httpStatus}, ${r.pages} page(s))`);
    await sleep(120);
  }

  // Deterministic global ordering: state, then organization name.
  all.sort((a, b) => (a.state === b.state ? a.organizationName.localeCompare(b.organizationName) : a.state.localeCompare(b.state)));

  // Whole-dataset files.
  writeFileSync(join(DATASET, 'contacts.json'), JSON.stringify(all, null, 2) + '\n');
  writeFileSync(join(DATASET, 'contacts.csv'), toCsv(all));

  // Per-state files.
  const grouped = groupByState(all);
  for (const [st, recs] of grouped) {
    writeFileSync(join(BYSTATE, `${st}.json`), JSON.stringify(recs, null, 2) + '\n');
    writeFileSync(join(BYSTATE, `${st}.csv`), toCsv(recs));
  }

  perState.sort((a, b) => a.state.localeCompare(b.state));
  const withPhone = all.filter((r) => r.phone).length;
  const summary = {
    source: SOURCE_NAME,
    datasetYear: year,
    generatedAt: startedAt,
    finishedAt: new Date().toISOString(),
    jurisdictions: JURISDICTIONS.length,
    totalOrganizations: all.length,
    organizationsWithPhone: withPhone,
    phoneCoveragePct: Math.round((withPhone / all.length) * 1000) / 10,
    seamFields: SEAM_FIELDS,
    perState,
  };
  writeFileSync(join(DATASET, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');

  const manifest = {
    source: SOURCE_NAME,
    sourceHomepage: 'https://educationdata.urban.org/documentation/school-districts.html',
    officialSource: 'US Department of Education, National Center for Education Statistics (NCES), Common Core of Data (CCD)',
    api: API,
    datasetYear: year,
    license: 'US federal government public-record data (NCES CCD); Urban Institute Education Data Portal terms apply.',
    schemaColumns: COLUMNS,
    seamFields: SEAM_FIELDS,
    seamNote: 'contactName, contactTitle, email, and domain are NOT published by this source. They are intentionally empty (DISCLOSED SEAM), never fabricated.',
    generatedAt: startedAt,
    finishedAt: new Date().toISOString(),
    jurisdictions: JURISDICTIONS.length,
    totalOrganizations: all.length,
    contactsJsonSha256: sha256(JSON.stringify(all, null, 2) + '\n'),
    contactsCsvSha256: sha256(toCsv(all)),
    fetchLog,
  };
  writeFileSync(join(DATASET, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
  writeFileSync(join(EVIDENCE, 'fetch-log.json'), JSON.stringify({ source: SOURCE_NAME, datasetYear: year, requests: fetchLog }, null, 2) + '\n');

  console.log(`[collect] DONE: ${all.length} organizations across ${JURISDICTIONS.length} jurisdictions (year ${year}).`);
  console.log(`[collect] phone coverage: ${summary.phoneCoveragePct}%`);
  console.log(`[collect] wrote dataset/contacts.{json,csv}, dataset/by-state/*, dataset/{summary,manifest}.json`);
}

main().catch((e) => { console.error('[collect] FAILED:', e.message); process.exit(1); });
