/**
 * services/scrape.js
 * Multi-backend Google search with free fallbacks.
 *
 *   1. Scrape.do (paid, primary)  — structured Google Scraper API
 *   2. Scrape.do (HTML proxy)     — when structured fails
 *   3. Direct Google (no auth)     — when Scrape.do is exhausted/401
 *   4. DuckDuckGo HTML (no auth)   — when Google also blocks us
 *
 * Returns the same {url, title, snippet}[] shape regardless of which
 * backend produced the results. The fallback chain only fires for
 * non-zero-result queries; one OK response short-circuits the rest.
 */

'use strict';

const keyRotator = require('./keyRotator');
const alertBus   = require('./alertBus');

const SCRAPEDO_URL = 'https://api.scrape.do/';

async function googleSearch(query, { num = 20, countryCode = 'us', geoCode = 'us' } = {}) {
  // 1. Scrape.do structured
  try {
    const { result, key } = await keyRotator.runWithRotation('scrape_do', async (k) => {
      const u = new URL('https://api.scrape.do/plugin/google/search');
      u.searchParams.set('token', k.key_value);
      u.searchParams.set('q', query);
      u.searchParams.set('num', String(num));
      u.searchParams.set('gl', countryCode);
      u.searchParams.set('hl', 'en');
      const res = await fetch(u.toString());
      if (!res.ok) {
        const t = await res.text();
        throw new Error(`scrape google ${res.status}: ${t.slice(0, 200)}`);
      }
      return res.json();
    });
    const parsed = parseGoogleStructured(result);
    if (parsed.length) return parsed;
    // Empty result — fall through to the next backend so the user
    // gets something instead of an empty array.
  } catch (e) {
    console.warn('[scrape] structured Google failed:', e.message);
  }

  // 2. Scrape.do HTML proxy
  try {
    const { result, key } = await keyRotator.runWithRotation('scrape_do', async (k) => {
      const u = new URL('https://api.scrape.do/');
      u.searchParams.set('token', k.key_value);
      u.searchParams.append('url', `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${num}&gl=${countryCode}`);
      const res = await fetch(u.toString());
      if (!res.ok) {
        const t = await res.text();
        throw new Error(`scrape_do ${res.status}: ${t.slice(0, 200)}`);
      }
      return res.text();
    });
    const parsed = parseGoogleResults(result);
    if (parsed.length) return parsed;
  } catch (e) {
    console.warn('[scrape] Scrape.do HTML proxy failed:', e.message);
  }

  // 3. Direct Google (no auth). This is the FREE backend — works
  //    when Scrape.do is exhausted or no key is configured. May be
  //    rate-limited; we keep a polite UA.
  try {
    const direct = await fetch(
      `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${num}&hl=en&gl=${countryCode}`,
      { headers: { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36', 'accept-language': 'en-US,en;q=0.9' } }
    );
    if (direct.ok) {
      const text = await direct.text();
      const parsed = parseGoogleResults(text);
      if (parsed.length) return parsed;
    }
  } catch (e) {
    console.warn('[scrape] direct Google failed:', e.message);
  }

  // 4. DuckDuckGo HTML (no auth, last-resort).
  try {
    const ddg = await fetch(
      `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
      { headers: { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36' } }
    );
    if (ddg.ok) {
      const text = await ddg.text();
      const parsed = parseDuckDuckGoResults(text);
      if (parsed.length) return parsed;
    }
  } catch (e) {
    console.warn('[scrape] DuckDuckGo failed:', e.message);
  }

  return [];
}

function parseGoogleStructured(data) {
  if (!data || typeof data !== 'object') return [];
  const arr = data.organic_results
          || data.local_results
          || data.results
          || data.knowledge_graph
          || [];
  if (!Array.isArray(arr)) return [];
  return arr.map(r => ({
    url:     r.link || r.url || r.original_url || '',
    title:   r.title || r.name || '',
    snippet: r.snippet || r.description || r.snippet_highlighted_words?.join(' ') || '',
  })).filter(r => r.url);
}

function parseGoogleResults(text) {
  try {
    const json = JSON.parse(text);
    if (Array.isArray(json.organic_results)) {
      return json.organic_results.map(r => ({
        url: r.link || r.url,
        title: r.title || '',
        snippet: r.snippet || r.description || '',
      }));
    }
  } catch (_) { /* fall through */ }

  const out = [];
  const anchorRe = /<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
  let m;
  while ((m = anchorRe.exec(text)) !== null) {
    const url = m[1];
    if (/google\.com|googleusercontent|gstatic/.test(url)) continue;
    const title = stripTags(m[2]);
    if (!title) continue;
    out.push({ url, title, snippet: '' });
    if (out.length >= 20) break;
  }
  return out;
}

/**
 * DuckDuckGo HTML parser. DDG's HTML endpoint returns results inside
 * <a class="result__a" href="…"> with the URL in a uddg= query
 * parameter (their redirect shim). We extract that.
 */
function parseDuckDuckGoResults(text) {
  const out = [];
  const re = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
  let m;
  while ((m = re.exec(text)) !== null) {
    let url = m[1];
    // DDG wraps outbound URLs in a redirect; resolve the real target.
    if (url.startsWith('//duckduckgo.com/l/?uddg=')) {
      try { url = decodeURIComponent(url.split('uddg=')[1].split('&')[0]); } catch (_) {}
    } else if (url.startsWith('/l/?uddg=')) {
      try { url = 'https://duckduckgo.com' + url; } catch (_) {}
    }
    if (!/^https?:\/\//.test(url)) continue;
    const title = stripTags(m[2]);
    if (!title) continue;
    out.push({ url, title, snippet: '' });
    if (out.length >= 20) break;
  }
  return out;
}

function stripTags(s) {
  return String(s).replace(/<[^>]+>/g, '').replace(/&[a-z]+;/gi, ' ').trim();
}

module.exports = { googleSearch, parseGoogleResults, parseDuckDuckGoResults, parseGoogleStructured };
