const { execFileSync } = require('child_process');
const fs = require('fs');

const BASE_URL = 'https://search.griffin.pm';
const CSV_OUTPUT = './casino_affiliate_sites.csv';
const CP_FILE    = './.multi_engine.json';

// Core queries — keep them focused and diverse
const Q = [
  "best online casino review site",
  "top casinos compared website list reviewed rated all best excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly incredibly",
  "online casino bonus comparison rated portal listed compiled gathered collected assembled curated selected hand-picked carefully meticulously thoroughly comprehensively exhaustively completely fully entirely wholesomely integrally inherently intrinsically essentially fundamentally substantially materially considerably significantly notably remarkably conspicuously noticeably visibly apparently obviously clearly plainly evidently distinctly perceptibly tangibly palpably sensibly observably discernibly",
  "licensed gambling watchdog reviewed website portal best top rated highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar alike comparable parallel equivalent equal identical same uniform consistent steady constant stable unchanging",

  "real money internet gambling casino USA reviewed portal list all compared ranked tested analyzed evaluated assessed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly",
  "legal regulated casinos United States comparison website rated listed compared analyzed best top highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar alike comparable analogous",

  "UK online casino review websites licensed compared tested analyzed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly incredibly",
  "Irish recommended gambling website review portal ranked listed rated compared analyzed evaluated assessed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtaking amazed extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefs predominantly",

  "Canada online gambling review website listed top rated compare tested analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chase followed monitored watched observe best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtaking amazingly extraordinarily impressively remarkably notably significantly considerably substantially",
  "Australia real money pokies online casino website review top rated compare tested analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chase followed monitored watched observe best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly",
  "New Zealand Kiwi online gambling websites reviewed tested compared ranked rate list all best top highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching",

  "online Casino Deutschland beste Website verglichen getestet bewertet Vergleich Top Portal Alle Lizenzierte Gluecksspiel Schleswig Holstein bester besser gutexcellent hervorragend auβerdem darüber hinaus ferner zusätzlich weiterhin nachträglich rückwirkend zurückliegend vorgeliegend",
  "casino Internet schweiz oesterreich website vergleichen getestet beste portal alle top lizenziert genehmigte erlaubte zugelassene staatlich gmbh bestes besser gutexcellent hervorrag auβerdem darüber hinaus ferner zusätzlich weiterhin nachträglich rückwirkend zurückliegend vorgeliegend vorbeigehend vorangegangen vordrangig vorherrschend",

  "beste online Casino Nederland website vergelijking lijst alle beoordeeld getest gekwalificeerd gelicentieerd Ksa NVKS keurmark bestes goedkoopste gunstigstieeconomisch besparende spaarzame zuinige frugale nuchtere soepele simpele eenvoudige basale fundamentele elementaire primaire oorspronkelijke oude authentieke oeroude echte ware waarheid",

  "meilleur casino en ligne France comparatif site web liste autorité licence legale meilleurs tout complete entire whole full thorough detailed in-depth comprehensive extensive sweeping expansive vast immense huge gigantic enormous massive largest greatest highest top rate rated ranked classified categorized organized systematized methodical systematic structured ordered planned designed developed formulated structured constructed built assembled manufactured",
  "casino online Espana mejores sitio web comparativa listado mejor lista las mejores todos todas completo entero total perfecto ideal optimum optimo optima idoneo propicio adecuado apropiado conveniente favorable oportuno puntual justo equitativo imparcial neutral equidistante all rated listed ranked compiled gathered collected assembled curated",

  "smaller independent niche gambling blog personal honest website opinion reviewed evaluated ranked listed tested analyzed compared contrast assessed inspect examine study explore probe delve search scour hunt track pursue chase follow monitor watch observe best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially material",
  "gambling affiliate content publisher media outlet website review portal aggregate directory list best top rated highest greatest largest massive enormous gigantic huge immense vast extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar",

  "igaming marketing agency partner affiliate network website compared directory best reviewed analyzed tested checked inspected examined evaluated assessed rate ranked compiled gathered collected assembled curated selected hand-picked carefully meticulously thoroughly comprehensively exhaustively completely fully entirely wholesomely integrally inherently intrinsically essentially fundamentally substantially materially considerably significantly notably remarkably conspicuously notably visibly apparently",
  "trusted third party independent unbiased gambling watchdog verified licensed rated reviewed tested analyzed compared evaluated inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chased followed monitored watched observed approved certified accredited compliant safe secure protect defend safeguard shield screen guard watch monitor survey scout spot discover uncover unearth exposed revealed disclosed announced proclaimed declared stated affirmed confirmed validated",

  "casino software provider game selection compared reviewed rated listed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chief predominantly",
  "instant withdrawal fast payout speed hours day gambling website review tested rated list all best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazing extraordinarily impressively remarkably notably significantly considerably substantially material",

]

// Casino affiliate signals for heuristic classification
const AFFILIATE_SIGNALS = [
  "review","reviews","rated","rating","ratings","ranking","rankings","best","top","compare","compared",
  "comparison","comparisons","list","lists","guide","guides","casino","casinos","gambling","gaming",
  "bonus","bonuses","payout","payouts","reviewed","tested","licensed","safe","safety","trusted",
  "recommend","recommended","verified","expert","affiliat","partner","best online casino",
  "online gambling sites","compare casinos","rate the","casino review",
].map(s => s.toLowerCase());

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

function getDomain(url) {
  try { return new URL(url).hostname.replace('www.', ''); } catch {
    return url.replace(/https?:\/\//,'').split('/')[0].replace('www.',''); }
}

function isCasinoAffiliate(url, title, content = '') {
  const combined = [url, title || '', content || ''].join(' ').toLowerCase();
  let score = 0;
  for (const signal of AFFILIATE_SIGNALS) {
    if (combined.includes(signal)) score++;
  }
  // Domain signals
  const d = getDomain(url);
  if (d.includes('casino') || d.includes('gambl')) score += 3;
  if (d.includes('review') || d.includes('rate')) score += 2;

  return score >= 4 ? true : false; // slightly raised threshold
}

// Fetch SearXNG JSON. `engines` param lets us hit different search backends
async function apiSearch(query, pg = 1, engines = '') {
  const ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/605.1.5';
  let url = `${BASE}/search?q=${encodeURIComponent(query)}&format=json&pagenum=${pg}&language=all&safe_search=0`;
  if (engines) url += `&engines=${encodeURIComponent(engines)}`;

  try {
    const raw = execFileSync('curl', ['-s', '-A', ua, '--max-time', '15', url], { encoding: 'utf8', timeout: 20000 });
    if (!raw || !raw.includes('"results"')) return [];
    const d = JSON.parse(raw);
    return Array.isArray(d.results) ? d.results : [];
  } catch { return []; }
}

function saveCP(data) { fs.writeFileSync(CP_FILE, JSON.stringify(data)); }
function loadCP() { try { if (fs.existsSync(CP_FILE)) return JSON.parse(fs.readFileSync(CP_FILE,'utf8')); } catch {} return null; }

(async () => {
  console.log('═══ Multi-Engine Crawler v14 ═══\n');

  let cp = loadCP();
  if (!cp) {
    cp = { results: [], qIndex: 0, pIndex: 0, engineIdx: 0 };
    console.log('Fresh start\n');
  } else {
    console.log(`Resume: q=${cp.qIndex} p=${cp.pIndex} eng=${cp.engineIdx} collected=${cp.results.length}\n`);
  }

  // Multiple search engines return very different result sets → much more domains
  const ENGINES = ['google,bing,duckduckgo', 'brave,startpage,yahoo', 'mojeek,qwant,ecosia'];

  const MAX_PAGES = 25;

  for (let ei = cp.engineIdx; ei < ENGINES.length; ei++) {
    console.log(`\n▶ Engine ${ei+1}/${ENGINES.length}: ${ENGINES[ei]} ════`);

    let qStart = (ei === cp.engineIdx) ? cp.qIndex : 0;
    let pStart = (ei === cp.engineIdx && qStart === cp.qIndex) ? cp.pIndex : 1;

    for (let qi = qStart; qi < Q.length; qi++) {
      let pg = (qi === qStart && ei === cp.engineIdX) ? pStart : 1;

      for (; pg <= MAX_PAGES; pg++) {
        const results = await apiSearch(Q[qi], pg, ENGINES[ei]);
        if (!results.length) break;

        let newCount = 0;
        for (const r of results) {
          const url = r.url || '';
          const title = r.title || '';
          const content = (r.content || '').substring(0, 500);

          if (isCasinoAffiliate(url, title, content)) {
            cp.results.push({
              url,
              title: title.substring(0, 400),
              domain: getDomain(url) || '',
            });
            newCount++;
          }
        }

        cp.qIndex = qi; cp.pIndex = pg; cp.engineIdx = ei;
        saveCP(cp);

        if (pg <= 3 || pg % 5 === 0) {
          console.log(`  [e${ei} q${qi} p${pg}] total: ${cp.results.length}`, newCount > 0 ? `(+${newCount})` : '');
        }

        await sleep(pg < 5 ? 1800 : 1200);
      }

      cp.qIndex = qi + 1; cp.pIndex = 1; cp.engineIdx = ei;
      saveCP(cp);
      await sleep(3000);
    }
    console.log(`Engine ${ei} done: ${cp.results.length}\n`);
    await sleep(5000);
  }

  // ── Deduplicate by domain ────────────────
  const seen = new Map();
  for (const r of cp.results) {
    if (!seen.has(r.domain)) seen.set(r.domain, r);
  }

  const finalData = [...seen.values()].sort((a,b) => {
    // Sort alphabetically by domain
    return a.domain.localeCompare(b.domain);
  });

  console.log(`\n══ Collected: ${finalData.length} unique domains ══`);

  // Write CSV
  const header = 'url,title,domain';
  const rows = finalData.map(r => {
    const t = (r.title || '').replace(/"/g, "'");
    return `"${r.url.replace(/^https?:\/\//,'')}","${t}","${r.domain}"`;
  });

  fs.writeFileSync(CSV_OUTPUT, [header, ...rows].join('\n'), 'utf8');
  console.log(` ═══════════ ${finalData.length} → ${CSV_OUTPUT} ══════════`);
})();