/**
 * services/contactDiscovery.js
 *
 * Discover real abuse / DMCA contacts for a domain by scraping:
 *   1. https://<domain>/.well-known/security.txt     (RFC 9116)
 *   2. https://<domain>/sitemap.xml                  (DMCA / legal pages often listed)
 *   3. https://<domain>/robots.txt                   (sitemaps referenced)
 *   4. https://<domain>/dmca, /legal, /takedown, /info, /2257 (common paths)
 *   5. The home page footer (regex for email addresses)
 *   6. HackerOne directory (security.txt often points here)
 *
 * Returns: {
 *   primary:  { contact, source, confidence }  // best DMCA contact
 *   secondary: [...]
 *   form_url:  string | null
 *   hosting:   'cloudflare' | 'amazon' | 'google' | ...   (from headers)
 *   registrar: string | null
 *   escalation_chain: [ { type, address, why } ]
 * }
 */

'use strict';

const { URL } = require('url');
const keyRotator = require('./keyRotator');

const COMMON_LEGAL_PATHS = [
  '/dmca', '/DMCA', '/legal/dmca', '/legal/takedown', '/information/dmca',
  '/2257', '/compliance', '/legal', '/takedown', '/report',
  '/help', '/support', '/contact', '/contacts', '/policies', '/info',
];

// We always check the /contacts/ path FIRST because that's where most
// user-generated-content sites put their abuse contact, and the page
// usually has a Cloudflare-obfuscated email we have to decode.
// Each path is tried with AND without a trailing slash — many
// WordPress / nginx / Cloudflare setups return 404 on /contacts
// but 200 on /contacts/ (or vice versa).
const PRIORITY_PATHS = [
  '/contacts', '/contacts/',
  '/contact', '/contact/',
  '/dmca', '/dmca/', '/legal/dmca', '/legal/dmca/',
  '/report', '/report/',
  '/about', '/about/',
  '/terms', '/terms/',
  '/.well-known/security.txt',
];

const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
const ROLE_BOX = /^(abuse|dmca|legal|trust|safety|notice|privacy|support|help|info|contact|2257|removals?|compliance|copyright|ip|intellectual|trustandsafety)@/i;

// Obvious placeholders / examples that we should never treat as a real contact.
const PLACEHOLDER_RE = /^(you|yourname|example|test|foo|bar|user|username|me|email|address|admin)@/i;
// Common CDN/registrar patterns we *do* want to recognise.
const SERVICE_PATTERNS = [
  /^abuse@(?:cloudflare|amazonaws|google|microsoft|vercel|fastly|googleusercontent|gmail|outlook|apple|meta|facebook|instagram|twitter|x)\./i,
];

/**
 * Decode a Cloudflare __cf_email__ data attribute. The hex string is
 * `XXYYZZ...` where XX is the XOR key (in decimal-as-2-hex-digits) and
 * the rest are pairs of hex bytes that get XOR'd with XX to recover
 * the plaintext email. (Reference: Cloudflare's
 * email-decode.min.js — reverse-engineered from public docs.)
 */
function decodeCloudflareEmail(hex) {
  if (!hex || hex.length < 4) return '';
  const key = parseInt(hex.substr(0, 2), 16);
  let out = '';
  for (let i = 2; i < hex.length; i += 2) {
    const code = parseInt(hex.substr(i, 2), 16) ^ key;
    out += String.fromCharCode(code);
  }
  return out;
}

/**
 * Extract every email-shaped string from an HTML blob, including
 * Cloudflare-obfuscated ones. Returns deduped array.
 */
function extractEmailsFromHtml(html) {
  if (!html) return [];
  const out = new Set();
  // Plain mailto:
  const mailtoRe = /mailto:([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})/gi;
  let m;
  while ((m = mailtoRe.exec(html)) !== null) out.add(m[1].toLowerCase());
  // Cloudflare __cf_email__ data-cfemail="HEX"
  const cfRe = /data-cfemail="([a-f0-9]+)"/gi;
  while ((m = cfRe.exec(html)) !== null) {
    const decoded = decodeCloudflareEmail(m[1]);
    if (decoded && decoded.includes('@')) out.add(decoded.toLowerCase());
  }
  // Bare emails in text (less reliable but useful as a last resort)
  const bareRe = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
  while ((m = bareRe.exec(html)) !== null) out.add(m[0].toLowerCase());
  // gen_mail('lhs', 'rhs') pattern (EroMe, older PHP sites) — splits an
  // email into two string literals and concatenates with @ at runtime.
  // e.g.  gen_mail('contact', 'erome.com')  →  contact@erome.com
  const genMailRe = /gen_mail\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)/g;
  while ((m = genMailRe.exec(html)) !== null) {
    const lhs = m[1].trim(), rhs = m[2].trim();
    if (lhs && rhs && !lhs.includes('@') && !rhs.includes('@')) {
      out.add((lhs + '@' + rhs).toLowerCase());
    }
  }
  return [...out].filter(e => !PLACEHOLDER_RE.test(e));
}

/**
 * Fetch a URL via Scrape.do proxy (so we get past anti-bot, get rendered HTML).
 * Falls back to direct fetch if Scrape.do has no key.
 */
async function fetchViaScrape(targetUrl) {
  if (!keyRotator.getKey('scrape_do')) {
    return await directFetch(targetUrl);
  }
  return keyRotator.runWithRotation('scrape_do', async (key) => {
    // Scrape.do's basic proxy API: pass the target URL via the `url` query
    // param, double-encoded. We use a manual URLSearchParams to avoid the
    // `URL` class re-encoding our already-encoded value.
    const u = new URL('https://api.scrape.do/');
    u.searchParams.set('token', key.key_value);
    u.searchParams.append('url', targetUrl);
    const res = await fetch(u.toString());
    if (!res.ok) {
      const t = await res.text();
      throw new Error(`scrape ${res.status}: ${t.slice(0, 200)}`);
    }
    return { text: await res.text(), status: res.status, headers: res.headers };
  }).then(r => r.result);
}

/**
 * Run a structured Google search via Scrape.do's Google Scraper API.
 * Returns parsed JSON results (or an array of { url, title, snippet } on
 * best-effort parsing). Costs 10 credits per request.
 */
async function googleSearchStructured(query, { num = 20, countryCode = 'us', gl = 'us', hl = 'en' } = {}) {
  if (!keyRotator.getKey('scrape_do')) {
    throw new Error('SCRAPE_DO_KEY_REQUIRED');
  }
  return keyRotator.runWithRotation('scrape_do', async (key) => {
    const u = new URL('https://api.scrape.do/plugin/google/search');
    u.searchParams.set('token', key.key_value);
    u.searchParams.set('q', query);
    u.searchParams.set('num', String(num));
    u.searchParams.set('gl', gl);
    u.searchParams.set('hl', hl);
    const res = await fetch(u.toString());
    if (!res.ok) {
      const t = await res.text();
      throw new Error(`scrape google ${res.status}: ${t.slice(0, 200)}`);
    }
    return await res.json();
  }).then(r => r.result);
}

async function directFetch(targetUrl) {
  const res = await fetch(targetUrl, {
    redirect: 'follow',
    headers: { 'user-agent': 'NCII-Shield-Discovery/1.0 (+contact-discovery)' },
  });
  return { text: await res.text(), status: res.status, headers: res.headers };
}

function extractEmails(text) {
  // New extractor is Cloudflare-aware (handles __cf_email__ obfuscation)
  return extractEmailsFromHtml(text);
}

/**
 * Probe a domain's most likely contact pages in priority order and
 * return the BEST published contact email + source. Used by the
 * live pipeline to find the platform's real contact (vs. the
 * registrar's fallback). Returns { contact, source, confidence, page }
 * or null.
 *
 * Strategy: walk PRIORITY_PATHS in order. For each page:
 *   - extract every email (mailto: + Cloudflare-obfuscated)
 *   - score them by role prefix (dmca@ > abuse@ > legal@ > ... > gmail)
 *   - if we get a non-gmail, non-personal address, return immediately
 *   - else keep looking and take the best at the end
 */
async function findPlatformContact(domain) {
  domain = String(domain || '').toLowerCase().replace(/^www\./, '').trim();
  if (!domain) return null;
  const candidates = [];   // { email, source, confidence, page }
  for (const path of PRIORITY_PATHS) {
    let r;
    try { r = await safeFetch(`https://${domain}${path}`); }
    catch (_) { continue; }
    if (!r || r.status !== 200) continue;
    const emails = extractEmailsFromHtml(r.text);
    for (const e of emails) {
      const kind = classifyEmail(e);
      let conf = 0;
      if (kind === 'dmca')         conf = 0.95;
      else if (kind === 'abuse')   conf = 0.9;
      else if (kind === 'legal')   conf = 0.85;
      else if (kind === 'service') conf = 0.7;  // abuse@cloudflare.com etc.
      else if (kind === 'personal') conf = 0.5; // gmail — sketchy but published
      else conf = 0.4;
      candidates.push({ email: e, source: `path:${path}`, confidence: conf, page: path });
    }
  }
  if (!candidates.length) return null;
  // Pick the best (highest confidence). If multiple are tied, prefer
  // the one that appeared on the most-common /contacts path.
  candidates.sort((a, b) => b.confidence - a.confidence || a.page.localeCompare(b.page));
  return candidates[0];
}

function rankEmails(emails) {
  // Higher score = more likely a real abuse contact
  const scored = emails.map(e => {
    const lower = e.toLowerCase();
    let score = 0;
    if (lower.startsWith('dmca@'))         score = 100;
    else if (lower.startsWith('abuse@'))   score = 95;
    else if (lower.startsWith('legal@'))   score = 90;
    else if (lower.startsWith('trust@'))   score = 85;
    else if (lower.startsWith('safety@'))  score = 85;
    else if (lower.startsWith('notice@'))  score = 80;
    else if (lower.startsWith('removals@'))score = 80;
    else if (lower.startsWith('2257@'))    score = 75;
    else if (lower.startsWith('compliance@')) score = 75;
    else if (lower.startsWith('support@')) score = 30;
    else if (lower.startsWith('help@'))    score = 25;
    else if (lower.startsWith('info@'))    score = 20;
    else if (lower.startsWith('contact@')) score = 20;
    else if (lower.startsWith('privacy@')) score = 15;
    else if (lower.startsWith('noreply@') || lower.startsWith('no-reply@')) score = 0;
    else if (ROLE_BOX.test(lower))         score = 50;
    return { email: e, score };
  });
  return scored.sort((a, b) => b.score - a.score);
}

function detectHosting(headers) {
  if (!headers) return null;
  // Look for common CDN signatures in response headers
  const all = Object.entries(headers).map(([k, v]) => `${k}:${v}`).join('\n').toLowerCase();
  if (all.includes('cf-ray') || all.includes('cloudflare'))    return 'cloudflare';
  if (all.includes('cf-cache-status'))                         return 'cloudflare';
  if (all.includes('x-amz-cf-id') || all.includes('amazonaws')) return 'amazon';
  if (all.includes('x-amz-request-id'))                        return 'amazon';
  if (all.includes('x-goog') || all.includes('google'))       return 'google';
  if (all.includes('x-azure'))                                  return 'azure';
  if (all.includes('x-vercel-id'))                              return 'vercel';
  if (all.includes('x-fastly'))                                 return 'fastly';
  if (all.includes('server: litespeed') || all.includes('x-litespeed')) return 'litespeed';
  if (all.includes('server: openresty') || all.includes('nginx') || all.includes('server: nginx')) return 'nginx';
  if (all.includes('x-powered-by') && all.includes('php'))     return 'shared-hosting';
  if (all.includes('x-shopify'))                                return 'shopify';
  if (all.includes('x-sucuri'))                                 return 'sucuri';
  if (all.includes('server: akamaighost') || all.includes('x-akamai')) return 'akamai';
  if (all.includes('server: bigip') || all.includes('x-cnection')) return 'f5';
  return null;
}

// Reverse DNS on the resolved IP can also tell us. We can't easily do that
// synchronously here, but a few well-known IP ranges (CN region) are good
// proxies for "Cloudflare-fronted" already.

function hostingAbuseAddress(hosting) {
  // Documented abuse contacts for major hosts/CDNs.
  const map = {
    cloudflare:      'abuse@cloudflare.com',
    amazon:          'abuse@amazonaws.com',
    google:          'abuse@google.com',
    azure:           'abuse@microsoft.com',
    vercel:          'abuse@vercel.com',
    fastly:          'abuse@fastly.com',
    akamai:          'abuse@akamai.com',
    shopify:         'abuse@shopify.com',
    sucuri:          'abuse@sucuri.net',
    litespeed:       'support@litespeedtech.com',
    nginx:           null,           // nginx is just a web server, not a host
    'shared-hosting':null,           // we'd need to identify the host; can't generically escalate
    f5:              'abuse@f5.com',
  };
  return map[hosting] || null;
}

// Trust & Safety / content-policy escalation contacts at major infrastructure
// providers. These can act on violations even when the platform ignores us,
// because they have contractual terms with the operator.
function hostingTrustSafety(hosting) {
  const map = {
    cloudflare:      { email: 'trust@cloudflare.com',
                       why:   'Cloudflare Trust & Safety can de-list content that violates their TOS or US law; they have a 24-hour SLA for image-based abuse.' },
    amazon:          { email: 'trust-safe@amazon.com',
                       why:   'AWS Trust & Safety can suspend EC2/S3/CloudFront origin accounts serving illegal content.' },
    google:          { email: 'abuse@google.com',
                       why:   'Google abuse team can remove content indexed in Search, de-list YouTube, and act on Workspace abuse.' },
    azure:           { email: 'abuse@microsoft.com',
                       why:   'Microsoft Digital Crimes Unit handles Azure-hosted illegal content.' },
    vercel:          { email: 'trust@vercel.com',
                       why:   'Vercel Trust & Safety can suspend deployments serving illegal content.' },
    fastly:          { email: 'trust@fastly.com',
                       why:   'Fastly Trust & Safety can de-list edge content.' },
    akamai:          { email: 'abuse@akamai.com',
                       why:   'Akamai Trust & Safety can block edge delivery.' },
    shopify:         { email: 'trust-safety@shopify.com',
                       why:   'Shopify Trust & Safety can take down stores serving illegal content.' },
    sucuri:          { email: 'abuse@sucuri.net',
                       why:   'Sucuri can revoke WAF/proxy for sites serving illegal content.' },
    litespeed:       { email: 'support@litespeedtech.com',
                       why:   'LiteSpeed (web server) can be configured to block at the host; their support forwards to the originating web host.' },
    f5:              { email: 'abuse@f5.com',
                       why:   'F5 can be configured to block at the edge; their abuse team can act on WAF customers.' },
  };
  return map[hosting] || null;
}

/**
 * Main entry. Discovers contacts for `domain` and returns a structured plan.
 * NEVER throws — always returns a result with at least the fallback chain.
 */
async function discover(domain) {
  domain = String(domain || '').toLowerCase().replace(/^www\./, '').trim();
  if (!domain) return emptyResult('');

  const result = {
    domain,
    primary:    null,    // { contact, source, confidence, kind }
    secondary:  [],
    form_url:   null,
    security_txt: null,
    hosting:    null,
    registrar:  null,
    escalation_chain: [],
    errors:     [],
    discovered_at: Date.now(),
  };

  // 0. PLATFORM-FIRST contact probe. The most likely place the
  //    operator will find a real abuse email is the site's own
  //    /contacts, /dmca, or /report page. This handles Cloudflare
  //    email obfuscation (decoded via data-cfemail) and ranks by
  //    role prefix (dmca@ > abuse@ > legal@ > trust@ > personal).
  try {
    const platformContact = await findPlatformContact(domain);
    if (platformContact) {
      const kind = classifyEmail(platformContact.email);
      // Boost the confidence — the email is published on the
      // platform's own page, not a registrar.
      const conf = Math.max(platformContact.confidence, kind === 'dmca' ? 0.9 : kind === 'abuse' ? 0.85 : 0.7);
      result.secondary.unshift({
        contact: platformContact.email,
        source: platformContact.source,
        confidence: conf,
        kind,
      });
    }
  } catch (_) { /* best-effort */ }

  // 0b. CLOUDFARE-FRONTED FALLBACK. If the site is behind Cloudflare
  //     and the platform itself didn't publish a contact, the
  //     next-step abuse address is `abuse@cloudflare.com` — that's
  //     their actual abuse channel (not the /abuse form). Cloudflare
  //     forwards DMCA reports to the underlying host. This is the
  //     correct first step in the chain when the platform has no
  //     direct contact.
  if (result.hosting && /cloudflare/i.test(result.hosting)) {
    result.secondary.push({
      contact: 'abuse@cloudflare.com',
      source: 'hosting:cloudflare',
      confidence: 0.8,
      kind: 'service',
    });
  }

  // 1. security.txt
  try {
    const platformContact = await findPlatformContact(domain);
    if (platformContact) {
      const kind = classifyEmail(platformContact.email);
      // Boost the confidence — the email is published on the
      // platform's own page, not a registrar.
      const conf = Math.max(platformContact.confidence, kind === 'dmca' ? 0.9 : kind === 'abuse' ? 0.85 : 0.7);
      result.secondary.unshift({
        contact: platformContact.email,
        source: platformContact.source,
        confidence: conf,
        kind,
      });
    }
  } catch (_) { /* best-effort */ }

  // 1. security.txt
  try {
    const sec = await safeFetch(`https://${domain}/.well-known/security.txt`);
    if (sec && sec.status === 200) {
      result.security_txt = parseSecurityTxt(sec.text);
      // Pull any Contact lines that look like emails
      for (const c of (result.security_txt.contacts || [])) {
        if (c.includes('@')) {
          result.secondary.push({ contact: c, source: 'security.txt', confidence: 90, kind: classifyEmail(c) });
        }
      }
    }
  } catch (e) { result.errors.push(`security.txt: ${e.message}`); }

  // 2. common legal pages
  for (const p of COMMON_LEGAL_PATHS) {
    try {
      const r = await safeFetch(`https://${domain}${p}`);
      if (r && r.status === 200) {
        result.hosting = result.hosting || detectHosting(r.headers);
        const emails = extractEmails(r.text);
        for (const e of emails) {
          result.secondary.push({ contact: e, source: `path:${p}`, confidence: p.includes('dmca') || p.includes('legal') ? 85 : 60, kind: classifyEmail(e) });
        }
        // Find form URL
        const formMatch = r.text.match(/<form[^>]+action=["']([^"']+)["']/i);
        if (formMatch) result.form_url = formMatch[1];
        // Maybe a security disclosure program
        const h1 = r.text.match(/https?:\/\/(?:www\.)?hackerone\.com\/[a-zA-Z0-9_-]+/);
        if (h1 && !result.security_txt) result.security_txt = { contacts: [h1[0]], expires: null };
      }
    } catch (e) { /* ignore path errors */ }
  }

  // 3. sitemap
  try {
    const sm = await safeFetch(`https://${domain}/sitemap.xml`);
    if (sm && sm.status === 200) {
      const urls = [...sm.text.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m => m[1]);
      const interesting = urls.filter(u => /dmca|legal|takedown|2257|compliance|trust|report/i.test(u)).slice(0, 5);
      for (const u of interesting) {
        try {
          const page = await safeFetch(u);
          if (page && page.status === 200) {
            const emails = extractEmails(page.text);
            for (const e of emails) {
              result.secondary.push({ contact: e, source: `sitemap:${new URL(u).pathname}`, confidence: 80, kind: classifyEmail(e) });
            }
          }
        } catch (_) {}
      }
    }
  } catch (e) { result.errors.push(`sitemap: ${e.message}`); }

  // 4. home page (footer)
  try {
    const home = await safeFetch(`https://${domain}/`);
    if (home && home.status === 200) {
      result.hosting = result.hosting || detectHosting(home.headers);
      const emails = extractEmails(home.text);
      for (const e of emails) {
        result.secondary.push({ contact: e, source: 'homepage', confidence: 30, kind: classifyEmail(e) });
      }
    }
  } catch (e) { result.errors.push(`home: ${e.message}`); }

  // 4b. ALWAYS also do a direct fetch for hosting detection (Scrape.do may
  //     strip cf-ray etc.). Hosting detection is too critical to miss.
  try {
    const direct = await fetch(`https://${domain}/`, {
      method: 'HEAD',
      redirect: 'follow',
      headers: { 'user-agent': 'Mozilla/5.0 (compatible; NCII-Shield/1.0)' },
    });
    const h = {};
    direct.headers.forEach((v, k) => { h[k] = v; });
    const detected = detectHosting(h);
    if (detected) result.hosting = detected;
  } catch (e) { result.errors.push(`host-probe: ${e.message}`); }

  // 5. robots.txt (might list sitemaps with legal pages)
  try {
    const robots = await safeFetch(`https://${domain}/robots.txt`);
    if (robots && robots.status === 200) {
      const sitemaps = [...robots.text.matchAll(/Sitemap:\s*(\S+)/gi)].map(m => m[1]);
      for (const s of sitemaps.slice(0, 2)) {
        try {
          const r = await safeFetch(s);
          if (r && r.status === 200) {
            const urls = [...r.text.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m => m[1]);
            const legal = urls.filter(u => /dmca|legal|takedown|2257|compliance/i.test(u)).slice(0, 3);
            for (const u of legal) {
              try {
                const page = await safeFetch(u);
                if (page && page.status === 200) {
                  for (const e of extractEmails(page.text)) {
                    result.secondary.push({ contact: e, source: `robots-sitemap:${new URL(u).pathname}`, confidence: 75, kind: classifyEmail(e) });
                  }
                }
              } catch (_) {}
            }
          }
        } catch (_) {}
      }
    }
  } catch (e) { /* ignore */ }

  // de-dupe, keep highest confidence per contact
  const map = new Map();
  for (const c of result.secondary) {
    const key = c.contact.toLowerCase();
    if (!map.has(key) || map.get(key).confidence < c.confidence) map.set(key, c);
  }
  result.secondary = [...map.values()].sort((a, b) => b.confidence - a.confidence);

  // Pick primary: best kind=dmca, else abuse, else any high-confidence
  result.primary = result.secondary.find(c => c.kind === 'dmca' && c.confidence >= 70)
                || result.secondary.find(c => c.kind === 'abuse' && c.confidence >= 70)
                || result.secondary.find(c => c.kind === 'legal' && c.confidence >= 70)
                || result.secondary[0]
                || null;

  // 6. Build escalation chain
  result.escalation_chain = buildEscalationChain(result, domain);

  return result;
}

function buildEscalationChain(result, domain) {
  const chain = [];
  let step = 0;

  // Step 1: platform's own direct abuse / DMCA contact
  if (result.primary) {
    step++;
    chain.push({
      step,
      type: 'platform',
      address: result.primary.contact,
      source: result.primary.source,
      why: `Direct abuse contact for ${domain}, ranked by reliability of the address role (dmca@ > abuse@ > legal@).`,
    });
  }

  // Step 2: platform's own security disclosure program (HackerOne etc.)
  if (result.security_txt && result.security_txt.contacts) {
    for (const c of result.security_txt.contacts) {
      if (c.startsWith('http')) {
        step++;
        chain.push({
          step,
          type: 'platform-disclosure',
          address: c,
          source: 'security.txt',
          why: `${domain}'s own vulnerability / content disclosure program. Submit as a "non-consensual intimate imagery" report — they route to legal/trust teams.`,
        });
      } else if (c.includes('@') && !chain.some(s => s.address === c)) {
        step++;
        chain.push({
          step,
          type: 'platform-disclosure-email',
          address: c,
          source: 'security.txt',
          why: `Contact from ${domain}'s security.txt.`,
        });
      }
    }
  }

  // Step 3: hosting provider's standard abuse channel
  if (result.hosting) {
    const addr = hostingAbuseAddress(result.hosting);
    if (addr) {
      step++;
      chain.push({
        step,
        type: 'hosting-abuse',
        address: addr,
        source: `detected: ${result.hosting}`,
        why: `${result.hosting} is detected as the network / CDN fronting ${domain}. Their abuse team can suspend service for the offending property under their AUP.`,
      });
    }
  }

  // Step 4: hosting provider's Trust & Safety team (separate from generic abuse)
  if (result.hosting) {
    const ts = hostingTrustSafety(result.hosting);
    if (ts && !chain.some(s => s.address === ts.email)) {
      step++;
      chain.push({
        step,
        type: 'hosting-trust-safety',
        address: ts.email,
        source: `detected: ${result.hosting}`,
        why: ts.why,
      });
    }
  }

  // Step 5: registrar-level abuse (per RFC 7485)
  step++;
  chain.push({
    step,
    type: 'registrar',
    address: `WHOIS-lookup: https://lookup.icann.org/en/lookup?q=${encodeURIComponent(domain)}`,
    source: 'rfc7485',
    why: `If ${domain}'s operator and host ignore us, we file a complaint with the registrar listed in the public WHOIS. Registrars are contractually required to investigate and can suspend the domain.`,
    action: 'lookup-and-email',
  });

  // Step 6: ICANN compliance (last-resort, only if registrar doesn't act)
  step++;
  chain.push({
    step,
    type: 'icann',
    address: 'compliance@icann.org',
    source: 'icann',
    why: `ICANN Contractual Compliance can be invoked if the registrar fails to act. This is a public-record complaint that affects the registrar's accreditation.`,
  });

  return chain;
}

function classifyEmail(email) {
  const lower = email.toLowerCase();
  if (lower.startsWith('dmca@'))         return 'dmca';
  if (lower.startsWith('abuse@'))        return 'abuse';
  if (lower.startsWith('legal@'))        return 'legal';
  if (lower.startsWith('trust@'))        return 'trust';
  if (lower.startsWith('safety@'))       return 'trust';
  if (lower.startsWith('compliance@'))   return 'compliance';
  if (lower.startsWith('removals@'))     return 'removals';
  if (lower.startsWith('2257@'))         return '2257';
  if (lower.startsWith('privacy@'))      return 'privacy';
  if (lower.startsWith('noreply@') || lower.startsWith('no-reply@')) return 'noreply';
  return 'generic';
}

function parseSecurityTxt(text) {
  const out = { contacts: [], expires: null, preferred_languages: [], raw: text };
  for (const line of text.split('\n')) {
    const m = line.match(/^\s*([A-Za-z-]+)\s*:\s*(.+?)\s*$/);
    if (!m) continue;
    const [, key, val] = m;
    if (key.toLowerCase() === 'contact') out.contacts.push(val);
    if (key.toLowerCase() === 'expires') out.expires = val;
    if (key.toLowerCase() === 'preferred-languages') out.preferred_languages.push(val);
  }
  return out;
}

async function safeFetch(url) {
  try {
    return await fetchViaScrape(url);
  } catch (_) {
    try { return await directFetch(url); }
    catch (e) { return null; }
  }
}

function emptyResult(domain) {
  return {
    domain,
    primary: null, secondary: [], form_url: null,
    security_txt: null, hosting: null, registrar: null,
    escalation_chain: buildEscalationChain({ primary: null, hosting: null }, domain),
    errors: ['no input'],
    discovered_at: Date.now(),
  };
}

module.exports = {
  discover, findPlatformContact, buildEscalationChain,
  classifyEmail, hostingAbuseAddress, detectHosting, rankEmails,
  decodeCloudflareEmail, extractEmailsFromHtml, PRIORITY_PATHS,
};
