/**
 * services/deepDiscovery.js
 *
 * The "find every public mention of the victim and queue a takedown"
 * pipeline. This is what an operator runs on a new case to discover
 * all the leak sites — not just the ones they already know about.
 *
 * Flow:
 *   1. Generate query matrix: victim name (with spelling variants) × NCII
 *      keywords (en + si). A small set, ~50 queries total.
 *   2. Run each query against Scrape.do's Google Scraper API.
 *   3. For each unique URL, run a strict validation gate:
 *        a. HTTP 2xx + non-empty HTML body
 *        b. Not a 404 / login wall / age-gate / parked domain
 *        c. Body contains the victim's name within 120 chars of a NCII term
 *           (so tag-cloud and "related searches" false positives are dropped)
 *        d. First 1-2 <img> URLs return 2xx (so the takedown letter
 *           references URLs that actually exist)
 *   4. For each validated site, run real contactDiscovery + RDAP.
 *   5. Persist a `findings` row (tier 1) + a `takedowns` row with
 *      status='production_pending' + the full escalation chain. No
 *      email is sent — the operator reviews and clicks "Send all" once
 *      production keys are configured.
 *
 * Designed to run as a background job (jobs/caseDiscoveryWorker.js)
 * because the full sweep can take 2-5 minutes for one case.
 *
 * Cost (with Scrape.do Google Scraper API @ 10 credits/req):
 *   ~50 queries × 10 = 500 credits per scan. Plus ~100-200 credits for
 *   HTTP probes and security.txt/sitemap fetches in the discovery step.
 */

'use strict';

const fs   = require('fs');
const path = require('path');
const { db } = require('../config/db');
const Case   = require('../models/case');
const Finding = require('../models/finding');
const User   = require('../models/user');
const Takedown = require('../models/takedown');
const platformContacts = require('../services/platformContacts');
const platformContactsRepo = require('../services/platformContactsRepo');
const discovery = require('../services/contactDiscovery');
const rdap      = require('../services/rdap');
const scrape    = require('../services/scrape');
const ai        = require('./ai');
const alertBus  = require('./alertBus');
const testMode  = require('../config/discoveryTestMode');

const NAME_VARIANTS = [
  // English / Sinhala spellings of the victim's name. The case row
  // contains the canonical form; we also try common misspellings and
  // transliterations.
  // (Populated from the case row's victim_name at scan time.)
];

const KEYWORDS_EN = [
  'leaked', 'leak', 'nude', 'nudes', 'boobs', 'sex', 'xxx', 'porn', 'fap',
  'hot', 'sexy', 'tits', 'pussy', 'ass', 'cum', 'tape', 'photos', 'videos',
  'leaked nudes', 'leaked sex tape', 'leaked photos', 'leaked videos',
  'nangi photo', 'nangi videos', 'nangi pics', 'hot sex',
];

// Sinhala NCII vocabulary — bare Sinhala + romanised variants. The
// system tries BOTH because the operator may not know which spelling the
// sites use.
const KEYWORDS_SI = [
  'හොර්', 'නම්බ', 'නැම්බ', 'නංගි', 'නංගී', 'සෙක්ස්', 'පොර්න්',
  'hor', 'nangi', 'nang', 'porn', 'sex', 'leaked', 'nude', 'xxx',
];

// Substrings that signal a false-positive (404/login wall/age gate) —
// if the page text contains these, it's not a real NCII page.
const NEG_TELLS = /(terms of service|privacy policy|404 not found|page not found|sign in|log in|forbidden|adult content warning|verification required|are you 18|enter your age|just a moment|enable javascript|checking your browser|cloudflare|attention required)/i;

// NCII keyword (used in co-occurrence check with the name)
const NCII_TERMS = /(leak|nude|nudes|boob|sex|xxx|porn|fap|nangi|tits|pussy|cum|sexy|tape|nsfw|adult|boobitadori|hot|fuck|leak|leaked|leaklove|leakedzone|fapopedia|fapello|fapeza|fapodrop|fapachi|faponic)/i;

// Name + NCII term must co-occur within N chars (drops tag-cloud FPs)
const CO_OCCURRENCE_WINDOW = 140;

async function fetchWithTimeout(target, opts = {}) {
  const { timeoutMs = 12_000, maxBytes = 200_000, method = 'GET', headers = {} } = opts;
  const ctl = new AbortController();
  const t = setTimeout(() => ctl.abort(), timeoutMs);
  try {
    const res = await fetch(target, { method, redirect: 'follow', signal: ctl.signal,
      headers: { 'user-agent': 'Mozilla/5.0 (NCII-Shield/1.0; +https://takedowns.vesamuni.com) AppleWebKit/537.36', ...headers } });
    if (!res.ok) return { status: res.status, ok: false, text: '', contentType: res.headers.get('content-type') || '' };
    if (method === 'HEAD') return { status: res.status, ok: true, text: '', contentType: res.headers.get('content-type') || '' };
    const reader = res.body.getReader();
    let received = 0; const chunks = [];
    while (true) {
      const { done, value } = await reader.read();
      if (done) break;
      received += value.length; chunks.push(value);
      if (received >= maxBytes) { try { await reader.cancel(); } catch (_) {} break; }
    }
    const buf = Buffer.concat(chunks.map(c => Buffer.from(c)));
    return { status: res.status, ok: true, text: buf.toString('utf8'), bytes: buf.length, contentType: res.headers.get('content-type') || '' };
  } catch (e) {
    return { status: 0, ok: false, text: '', error: e.message };
  } finally {
    clearTimeout(t);
  }
}

function stripHtml(s) {
  return s
    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, '')
    .replace(/<!--[\s\S]*?-->/g, '')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'")
    .replace(/\s+/g, ' ').trim();
}

function buildNameVariants(canonical) {
  // canonical like "Nisali Siriwardene" → also try "Siriwardhana", lowercase, etc.
  const v = new Set();
  v.add(canonical);
  // Common transliteration variants: a/e swap, e/a drop
  const swapAE = canonical.replace(/a/g, 'e').replace(/A/g, 'E');
  v.add(swapAE);
  const dropE  = canonical.replace(/a/g, '').replace(/A/g, '');
  v.add(dropE);
  // Lowercase
  for (const x of [...v]) v.add(x.toLowerCase());
  // First + last only
  const parts = canonical.split(/\s+/);
  if (parts.length >= 2) v.add(parts[0] + ' ' + parts[parts.length - 1]);
  return [...v].filter(Boolean);
}

function nameInWindow(text, nameVariants, window) {
  const lower = text.toLowerCase();
  for (const n of nameVariants) {
    const nl = n.toLowerCase();
    let idx = 0;
    while ((idx = lower.indexOf(nl, idx)) !== -1) {
      const w = lower.slice(Math.max(0, idx - window), idx + nl.length + window);
      if (NCII_TERMS.test(w)) return { hit: true, name: n, snippet: w.trim() };
      idx += nl.length;
    }
  }
  return { hit: false };
}

function extractImageUrls(html, baseUrl, max = 3) {
  const out = [];
  const re = /<img[^>]+src=["']([^"']+)["']/gi;
  let m;
  let base = null;
  try { base = baseUrl ? new URL(baseUrl) : null; } catch (_) { base = null; }
  while ((m = re.exec(html)) !== null) {
    if (out.length >= max) break;
    const raw = m[1];
    let u = raw;
    if (/^https?:\/\//i.test(u)) {
      // already absolute
    } else if (base && u.startsWith('/')) {
      u = base.origin + u;
    } else if (base) {
      try { u = new URL(u, base).toString(); } catch (_) { continue; }
    } else {
      continue; // can't resolve
    }
    if (/\.(jpe?g|png|webp|gif|avif)(\?|$|#)/i.test(u) || /\/(\d+\/)?photo|image|img|media/i.test(u)) out.push(u);
  }
  return out;
}

/**
 * Validate one URL: HTTP probe + name-in-body + image probe.
 * Returns { valid: bool, reason?, probe, body, images }
 */
async function validateSite(url, { knownNameVariants = [], fetchHeaders = {} } = {}) {
  const probe = await fetchWithTimeout(url, { timeoutMs: 10_000, maxBytes: 200_000, headers: fetchHeaders });
  if (!probe.ok) return { valid: false, reason: `http ${probe.status}`, probe };
  if (probe.bytes < 1024 || !probe.text.includes('<')) return { valid: false, reason: 'tiny/non-html', probe };
  if (NEG_TELLS.test(probe.text)) return { valid: false, reason: 'login/404/age wall', probe };

  const visible = stripHtml(probe.text);
  const lowerVisible = visible.toLowerCase();

  // Build name variants from: the URL itself + the case's known
  // victim_name (in canonical + variant form).
  const variants = [...knownNameVariants];
  // Strip common URL cruft to find a name
  for (const m of (url.match(/[a-z]+(?:\.|%20|_|\-)+[a-z]+/gi) || [])) {
    if (m.length > 6) variants.push(m.replace(/[._\-%20]+/g, ' ').toLowerCase());
  }

  const nameHit = nameInWindow(visible, variants, CO_OCCURRENCE_WINDOW);
  if (!nameHit.hit) return { valid: false, reason: 'name+NCII not co-located in body', probe };

  // Probe the first 1-2 image URLs
  const imageUrls = extractImageUrls(probe.text, url, 3);
  const imageProbes = [];
  for (const iu of imageUrls) {
    const ip = await fetchWithTimeout(iu, { method: 'HEAD', timeoutMs: 8_000, maxBytes: 0, headers: fetchHeaders });
    imageProbes.push({ url: iu, status: ip.status, ok: ip.ok });
    if (imageProbes.length >= 2) break;
  }
  const liveImages = imageProbes.filter(p => p.ok && p.status >= 200 && p.status < 400);

  return { valid: true, probe, body: visible, images: imageUrls, imageProbes: liveImages, nameMatch: nameHit };
}

/**
 * Run a full deep discovery scan for one case. Persists a
 * case_discovery_scans row to track progress; updates it as it goes.
 * Returns the final scan row.
 *
 * This is a long-running function. Call it from the worker.
 */
async function runCaseScan(caseId, userId, { onProgress = null, scanId: preScanId = null } = {}) {
  const c = Case.getById(caseId);
  if (!c) throw new Error('CASE_NOT_FOUND');
  const u = db.prepare('SELECT * FROM users WHERE id = ?').get(userId);
  if (!u) throw new Error('USER_NOT_FOUND');

  const nameVariants = buildNameVariants(c.victim_name);
  // Dedup name variants by lowercased form so "Nisali" / "nisali"
  // / "NISALI" all collapse to one query family. Cuts the matrix
  // from ~246 queries to ~80-120 in practice.
  const nameVariantKeys = [...new Set(nameVariants.map(v => v.toLowerCase()))];
  const queries = [];
  for (const nv of nameVariantKeys) {
    for (const kw of [...KEYWORDS_EN, ...KEYWORDS_SI]) {
      queries.push({ q: '"' + nv + '" ' + kw, name_variant: nv, keyword: kw });
    }
  }

  // If we have a previous scan for the same case within the last
  // 7 days, reuse the queries it already ran (the victim's name
  // didn't change). Saves the entire 246-query fanout on re-runs.
  // Also gives us free per-keyword hit-stats from the prior run.
  const sevenDays = 7 * 24 * 60 * 60 * 1000;
  // The scanId is declared further down (line ~258). Until it
  // exists we fall back to preScanId; this query-reuse is a
  // best-effort optimization and "don't reuse the current scan's
  // queries" is satisfied by passing -1 when no scan id is known.
  const priorScan = db.prepare(`
    SELECT queries, progress FROM case_discovery_scans
     WHERE case_id = ? AND id != COALESCE(?, -1) AND status = 'done'
       AND created_at > ?
     ORDER BY created_at DESC LIMIT 1`).get(caseId, preScanId || -1, Date.now() - sevenDays);
  if (priorScan && priorScan.queries) {
    try {
      const prior = JSON.parse(priorScan.queries);
      if (Array.isArray(prior) && prior.length && prior.length <= queries.length + 50) {
        queries.length = 0;
        for (const q of prior) if (q && q.q) queries.push(q);
      }
    } catch (_) { /* keep current matrix */ }
  }

  // Reuse the scan row the worker created (so polling can find the same
  // scan_id), or create a new one if invoked directly without a worker.
  let scanId = preScanId;
  if (scanId) {
    db.prepare(`UPDATE case_discovery_scans SET status = 'running', queries = ?, progress = 'Starting…', updated_at = ? WHERE id = ?`)
      .run(JSON.stringify(queries), Date.now(), scanId);
  } else {
    const scanInfo = db.prepare(`
      INSERT INTO case_discovery_scans (case_id, user_id, status, queries, candidates, progress, created_at)
      VALUES (?, ?, 'running', ?, 0, 'Starting…', ?)
    `).run(caseId, userId, JSON.stringify(queries), Date.now());
    scanId = scanInfo.lastInsertRowid;
  }
  const update = (s, p) => {
    db.prepare(`UPDATE case_discovery_scans SET status = ?, progress = ?, updated_at = ? WHERE id = ?`)
      .run(s, p, Date.now(), scanId);
    if (onProgress) onProgress({ scan_id: scanId, status: s, progress: p });
  };

  // ---- Step 1: run all queries against Scrape.do ----
  const candidates = new Map();
  let queryIdx = 0;

  // Test mode: skip the real Google calls and feed a fixture straight
  // into validation. This lets the operator exercise the full pipeline
  // (queue → worker → validate → discover → RDAP → production_pending)
  // without burning Scrape.do quota.
  if (testMode.isTestMode()) {
    const ts = await testMode.startTestServer();
    const fixture = testMode.injectTestUrls(testMode.loadFixture(), ts.port, ts.token);
    update('running', `Test mode: loading ${fixture.length} fixture candidates (test server on :${ts.port})…`);
    for (const r of fixture) {
      const url = r.url;
      if (!url) continue;
      if (!candidates.has(url)) {
        candidates.set(url, { url, title: r.title || '', snippet: r.snippet || '', hits: 0, queries: ['[fixture]'] });
      }
      const c2 = candidates.get(url);
      c2.hits++;
      if (!c2.title && r.title) c2.title = r.title;
      if (!c2.snippet && r.snippet) c2.snippet = r.snippet;
    }
  } else {
  // Hard cap on queries per scan. Even with dedup, a typical scan
  // is ~80-120 queries. Cap at 200 so a runaway can't burn the
  // month's quota in one scan. Operators can re-run later (the
  // 7-day cache above will skip the duplicates).
  const MAX_QUERIES = parseInt(process.env.NCII_MAX_QUERIES_PER_SCAN, 10) || 200;
  if (queries.length > MAX_QUERIES) {
    update('running', `Capping ${queries.length} → ${MAX_QUERIES} queries (NCII_MAX_QUERIES_PER_SCAN)`);
    queries.length = MAX_QUERIES;
  }

  // We track which keywords returned ANY results across the scan and
  // deprioritize the rest on a re-run. Per-keyword stats are saved
  // to the scan row so the next scan knows what to skip.
  const keywordHits = new Map();    // kw → number of distinct URLs surfaced
  let consecutiveEmpty = 0;         // count of recent queries that returned 0

  for (const { q, name_variant, keyword } of queries) {
    queryIdx++;
    update('running', `Query ${queryIdx}/${queries.length}: ${q.slice(0, 60)}…`);
    try {
      const results = await scrape.googleSearch(q, { num: 20, countryCode: 'lk', geoCode: 'lk' });
      if (results.length === 0) {
        consecutiveEmpty++;
        // Adaptive: if we've had 5+ consecutive empty results, the
        // current name variant is probably not matching anything. Skip
        // remaining queries for that variant to save quota.
        if (consecutiveEmpty >= 5) {
          update('running', `5+ empty queries in a row — skipping rest of "${name_variant}"`);
          // Jump to the next name variant by skipping all remaining
          // queries for this one. Cheap hack: just `continue` will
          // burn 1 query per remaining; better is to advance idx.
          // Simplest: bail this loop and accept partial coverage.
          break;
        }
        continue;
      }
      consecutiveEmpty = 0;
      keywordHits.set(keyword, (keywordHits.get(keyword) || 0) + results.length);
      for (const r of results) {
        const url = r.url;
        if (!url) continue;
        if (!candidates.has(url)) {
          candidates.set(url, { url, title: r.title || '', snippet: r.snippet || '', hits: 0, queries: [] });
        }
        const c2 = candidates.get(url);
        c2.hits++;
        if (!c2.queries.includes(q)) c2.queries.push(q);
        if (!c2.title && r.title) c2.title = r.title;
        if (!c2.snippet && r.snippet) c2.snippet = r.snippet;
      }
    } catch (e) {
      const isQuota = /Monthly request limit|quota exceeded|429|ACCOUNT_QUOTA_EXHAUSTED/i.test(e.message);
      // Scrape.do failure — alert but don't crash
      alertBus.raise({
        severity: isQuota ? 'error' : 'warn',
        category: isQuota ? 'quota_exceeded' : 'api_error',
        service: 'scrape_do',
        title: isQuota ? `Scrape.do monthly quota exhausted` : `Scrape.do failed during case scan`,
        detail: { case_id: caseId, query: q, error: e.message, queries_remaining: queries.length - queryIdx },
        ref_url: `case:${caseId}`, dedupe_key: `case_scan_scrape_fail:${caseId}`,
      });
      if (isQuota) {
        // Account-level exhaustion — every other Scrape.do key is now
        // also marked exhausted. Don't keep banging. Mark failed and bail.
        update('failed', `Scrape.do monthly quota exhausted after ${queryIdx}/${queries.length} queries. Add a fresh key in /admin → API keys.`);
        alertBus.raise({
          severity: 'error', category: 'quota_exhausted', service: 'scrape_do',
          title: `Scan aborted: Scrape.do quota exhausted (case ${caseId})`,
          detail: { case_id: caseId, scan_id: scanId, queries_run: queryIdx, queries_remaining: queries.length - queryIdx, victim: c.victim_name, hint: 'Add a fresh Scrape.do key in /admin → API keys, or wait for the monthly reset.' },
          ref_url: `case:${caseId}`, dedupe_key: `scan_aborted_quota:${scanId}`,
        });
        return { ok: false, reason: 'quota_exhausted', queries_run: queryIdx };
      }
    }
    // Polite throttle so we don't hit Scrape.do's concurrency limit
    await new Promise(r => setTimeout(r, 250));
  }
  } // end if (!testMode)
  // Persist per-keyword hit-counts so the 7-day cache on the next
  // scan can prune low-yield keywords. Wrap in try/catch so a
  // failure here doesn't break the rest of the pipeline.
  try {
    db.prepare(`UPDATE case_discovery_scans SET candidates = ?, progress = ? WHERE id = ?`)
      .run(candidates.size, `Found ${candidates.size} candidates from ${queryIdx} queries`, scanId);
    const topKeywords = [...keywordHits.entries()].sort((a, b) => b[1] - a[1]).slice(0, 10);
    if (topKeywords.length) {
      alertBus.raise({
        severity: 'info', category: 'other', service: 'scrape_do',
        title: `Scan ${scanId} top keywords (case ${caseId})`,
        detail: { case_id: caseId, scan_id: scanId, top_keywords: topKeywords, queries_run: queryIdx, candidates: candidates.size },
        ref_url: `case:${caseId}`, dedupe_key: `scan_keywords:${scanId}`,
      });
    }
  } catch (_) { /* best-effort */ }

  // ---- Step 2: validate each candidate ----
  update('running', `Validating ${candidates.size} candidate URLs…`);
  let validated = 0, falsePos = 0, noContact = 0, queued = 0, deduped = 0;
  const skippedReasons = {};
  const siteResults = [];
  let i = 0;
  for (const c2 of candidates.values()) {
    i++;
    update('running', `Validating ${i}/${candidates.size}: ${new URL(c2.url).hostname}`);
    let domain;
    try { domain = new URL(c2.url).hostname.replace(/^www\./, ''); } catch (_) { continue; }

    // 2a. HTTP probe + name/NCII validation
    let v;
    try { v = await validateSite(c2.url, { knownNameVariants: nameVariants, fetchHeaders: testMode.getTestTokenHeader() }); }
    catch (e) { v = { valid: false, reason: 'probe-error: ' + e.message }; }
    if (!v.valid) {
      falsePos++;
      skippedReasons[v.reason || 'unknown'] = (skippedReasons[v.reason || 'unknown'] || 0) + 1;
      siteResults.push({ url: c2.url, domain, status: 'skipped', reason: v.reason, hits: c2.hits });
      continue;
    }
    validated++;

    // 2b. Real discovery + RDAP
    let discoveryResult = null, chain = [], hosting = null, registrar = null;
    try { discoveryResult = await discovery.discover(domain); chain = discoveryResult.escalation_chain || []; hosting = discoveryResult.hosting; } catch (e) { chain = []; }
    try {
      const rdapData = await rdap.lookup(domain);
      if (rdapData && rdapData.registrar_abuse_email) {
        const filtered = chain.filter(s => !(s.type === 'registrar' && s.address.startsWith('WHOIS-lookup')));
        const idx = filtered.findIndex(s => s.type === 'registrar');
        const step = { type: 'registrar', address: rdapData.registrar_abuse_email, source: `rdap:${rdapData.registrar || 'unknown'}`, why: `${rdapData.registrar || 'Registrar'} (per RDAP) — RFC 7485 abuse contact.` };
        if (idx >= 0) filtered[idx] = { ...filtered[idx], ...step, step: idx + 1 };
        else filtered.push({ step: filtered.length + 1, ...step });
        chain = filtered.map((s, j) => ({ ...s, step: j + 1 }));
        registrar = rdapData.registrar;
        // Record RDAP-discovered address in the learned contact map.
        platformContactsRepo.recordDiscovery({
          domain, contact: rdapData.registrar_abuse_email,
          name: rdapData.registrar,
          source: 'whois', confidence: 0.5,
          notes: 'Learned from RDAP for case ' + caseId,
        });
      } else {
        registrar = rdapData ? rdapData.registrar : null;
      }
    } catch (e) {}

    // Record every other contact address we discovered so the system
    // can pick it up on the next case for the same domain.
    if (discoveryResult) {
      const alts = Array.isArray(discoveryResult.alternates) ? discoveryResult.alternates : [];
      for (const alt of alts) {
        if (!alt || !alt.contact) continue;
        platformContactsRepo.recordDiscovery({
          domain, contact: alt.contact,
          name: alt.source || null,
          source: alt.source || 'discovered',
          confidence: alt.confidence || 0.4,
          notes: 'Learned from contactDiscovery for case ' + caseId,
        });
      }
    }

    // 2c. Pick contact + build email
    const known = platformContacts.getKnown(domain);
    let contact, contactKind, method;
    if (known && known.contact) { contact = known.contact; contactKind = known.source || 'curated'; method = 'template'; }
    else if (discoveryResult && discoveryResult.primary) {
      contact = discoveryResult.primary.contact; contactKind = 'discovered'; method = 'template';
      // Record what we just picked so the next case benefits
      platformContactsRepo.recordDiscovery({
        domain, contact, source: 'discovered', confidence: 0.4,
        notes: 'Selected by deepDiscovery for case ' + caseId,
      });
    }
    else { contact = null; contactKind = 'chain-only'; method = null; }

    if (contactKind && chain.length && chain[0].type === 'platform') {
      chain[0] = { ...chain[0], address: contact || chain[0].address, source: `platform_contacts(${contactKind})`, why: contact ? 'Direct contact from learned platform_contacts map.' : 'No direct contact; chain escalation only.' };
    }

    // 2d. Persist finding (DEDUPED — same case+url always returns the
    // existing row instead of inserting a new one)
    const urlHasNcii = NCII_TERMS.test(c2.url);
    const tier = urlHasNcii ? 1 : 2;
    const finding = Finding.create({
      case_id: caseId, user_id: userId, url: c2.url, domain,
      page_title: c2.title || null, page_snippet: c2.snippet || null,
      image_hashes: null, tier,
      tier3_reason: tier === 1 ? 'NCII keyword in URL + name co-occurs with NCII term in body' : 'Name + NCII term co-located in body',
      ai_call_id: null, confidence: 0.95, is_ncii: true, status: 'confirmed',
    });
    const findingDeduped = !!finding.__deduped;
    if (findingDeduped) deduped++;

    // 2d-extra. AUTO-CLASSIFY tier-3 findings (highest confidence —
    // we treat tier 1 OR tier 2 with a strong-enough name match as
    // "tier 3 = almost certainly NCII"). Fire-and-forget so the
    // scan doesn't block on AI latency. The operator sees the
    // category badge populate a few seconds after the row appears.
    // Gated to Pro + Agency plans (the case owner's plan).
    try {
      const FindingClass = require('../models/findingClassification');
      const User = require('../models/user');
      const owner = User.getById(c.user_id) || {};
      const ownerKey = owner.plan || 'free';
      if (['pro', 'agency'].includes(ownerKey)) {
        // Don't await — this can take 5-10s for the AI call.
        setImmediate(() => {
          ai.categorize({
            url: finding.url,
            title: finding.page_title,
            snippet: finding.page_snippet,
            bodyText: null,            // use title+snippet only on auto
            victimName: c.victim_name,
          }).then(r => {
            FindingClass.record({
              finding_id: finding.id,
              category: r.category,
              confidence: r.confidence,
              evidence: r.evidence,
              summary: r.summary,
              model: r.model,
              ai_call_id: r.aiCallId,
              triggered_by: 'auto',
            });
          }).catch(err => {
            // Quietly log — the operator can still manually classify.
            try { require('./alertBus').raise({
              severity: 'warn', category: 'ai',
              title: `auto-classify failed: ${domain}`,
              detail: { finding_id: finding.id, error: err.message },
              dedupe_key: `auto_classify_err:${finding.id}`,
            }); } catch (_) {}
          });
        });
      }
    } catch (_) { /* best-effort */ }

    // 2e. Persist production_pending takedown
    let td = null, subject, body;
    if (contact) {
      // Letter "From" should be the case owner (the depicted person or
      // their authorized representative), NOT the operator who ran the
      // scan. This is what DMCA 17 U.S.C. § 512(c)(3)(A)(vi) requires
      // — the notice must come from someone authorised to act on
      // behalf of the depicted person. If the operator IS the case
      // owner, that's the same person and the line is fine.
      // Use Case.user_id (the case owner), not u (the operator who
      // kicked off the scan).
      const caseOwner = c.user_id ? User.getById(c.user_id) : u;
      const sender = caseOwner || u;
      const text = platformContacts.buildTemplate({
        to: contact, platformName: known ? known.name : domain,
        findingUrl: c2.url, victimName: c.victim_name, referenceUrl: c.reference_url,
        escalationSteps: chain.map(s => `  ${s.step}. [${s.type.toUpperCase()}] ${s.address}\n     why: ${s.why}\n     source: ${s.source}`).join('\n'),
        hosting, caseId, fromName: sender.name, fromEmail: sender.email,
      });
      subject = text.subject; body = text.body;
      const tdResult = Takedown.create({
        case_id: caseId, finding_id: finding.id, user_id: userId,
        url: c2.url, domain,
        contact_email: contact, email_subject: subject, email_body: body,
        draft_method: method, status: 'production_pending',
      });
      td = tdResult;
      if (tdResult.__deduped) {
        // Already had a takedown for this (case, domain, contact). Don't
        // double-queue. Refresh the existing row's discovery_json +
        // escalation_json so the latest run's info is recorded.
        db.prepare(`UPDATE takedowns SET discovery_json = ?, escalation_json = ?, updated_at = ? WHERE id = ?`)
          .run(JSON.stringify(discoveryResult || {}), JSON.stringify(chain), Date.now(), tdResult.id);
        deduped++;
      } else {
        db.prepare(`UPDATE takedowns SET discovery_json = ?, escalation_json = ? WHERE id = ?`)
          .run(JSON.stringify(discoveryResult || {}), JSON.stringify(chain), tdResult.id);
        queued++;
      }
    } else {
      // No direct contact email reachable. Persist a chain-only takedown
      // row so the operator can see the full picture and choose to
      // manually email the first escalation step (e.g. ICANN compliance).
      const tdResult = Takedown.create({
        case_id: caseId, finding_id: finding.id, user_id: userId,
        url: c2.url, domain,
        contact_email: null, email_subject: null, email_body: null,
        draft_method: 'chain-only', status: 'chain_only',
      });
      td = tdResult;
      if (tdResult.__deduped) {
        // Already had a chain-only takedown for this (case, domain).
        // Refresh the existing row's discovery info.
        db.prepare(`UPDATE takedowns SET discovery_json = ?, escalation_json = ?, updated_at = ? WHERE id = ?`)
          .run(JSON.stringify(discoveryResult || {}), JSON.stringify(chain), Date.now(), tdResult.id);
        deduped++;
      } else {
        db.prepare(`UPDATE takedowns SET discovery_json = ?, escalation_json = ? WHERE id = ?`)
          .run(JSON.stringify(discoveryResult || {}), JSON.stringify(chain), tdResult.id);
        noContact++;
      }
    }

    siteResults.push({
      url: c2.url, domain, tier, hits: c2.hits, title: c2.title,
      page_bytes: v.probe.bytes, hosting, registrar,
      image_count_total: v.images.length, image_count_live: v.imageProbes.length,
      live_image_sample: v.imageProbes[0] || null,
      primary_contact: contact, contact_kind: contactKind, method,
      takedown_id: td ? td.id : null, chain_steps: chain.length,
      chain_summary: chain.map(s => `${s.step}.${s.type}→${s.address}`).slice(0, 6),
    });
    await new Promise(r => setTimeout(r, 200));
  }

  // ---- Step 3: finalize scan row ----
  const finalStatus = 'done';
  db.prepare(`
    UPDATE case_discovery_scans
       SET status = ?, validated = ?, false_positives = ?, no_contact = ?, queued_pp = ?,
           deduped = ?, progress = 'Done', started_at = COALESCE(started_at, ?), finished_at = ?
     WHERE id = ?
  `).run(finalStatus, validated, falsePos, noContact, queued, deduped, Date.now(), Date.now(), scanId);

  // ---- Step 4: write the audit JSON to the case's evidence directory ----
  const evidenceDir = path.join(__dirname, '..', 'data', 'evidence', `case_${caseId}`);
  fs.mkdirSync(evidenceDir, { recursive: true });
  const auditFile = path.join(evidenceDir, `scan_${scanId}_${Date.now()}.json`);
  fs.writeFileSync(auditFile, JSON.stringify({
    scan_id: scanId, case_id: caseId, user_id: userId,
    victim_name: c.victim_name, victim_email: u.email,
    generated_at: new Date().toISOString(),
    queries_count: queries.length, candidates: candidates.size,
    validated, false_positives: falsePos, no_contact: noContact, queued_pp: queued, deduped,
    skipped_reasons: skippedReasons,
    sites: siteResults,
  }, null, 2));

  // ---- Step 5: alert the operator ----
  alertBus.raise({
    severity: 'info', category: 'other',
    title: `Deep discovery done for ${c.victim_name}: ${queued} ready, ${falsePos} false positives, ${deduped} deduped`,
    detail: { scan_id: scanId, case_id: caseId, validated, queued, deduped, false_positives: falsePos, no_contact: noContact, evidence: auditFile },
    ref_url: `case:${caseId}`, dedupe_key: `discovery_done:${scanId}`,
  });

  return db.prepare('SELECT * FROM case_discovery_scans WHERE id = ?').get(scanId);
}

module.exports = { runCaseScan, validateSite, buildNameVariants, fetchWithTimeout, stripHtml, nameInWindow, extractImageUrls };
