const { execFile } = require('child_process');
const fs = require('fs');
const util = require('util');

const execFileP = (cmd, args) => new Promise((resolve, reject) => {
  const proc = execFile(cmd, args, { timeout: 15000 }, (err, stdout) => {
    if (err) return resolve({ err, stdout });
    resolve({ err: null, stdout });
  });
});

const BASE = 'https://search.griffin.pm';
const CSV_FILE = './casino_affiliate_sites.csv';

// ── 120+ queries across regions, niches, languages ───────────────
const Q = [
  // EN global broad
  "best online casino review site",
  "top casino reviews ranked website comparison",
  "online casino bonus comparison portal rated all",
  "compare rate casinos directory platform reviewed best",
  "independent casino review trusted expert tested website",
  "safest licensed online casino watchdog reviewed list",
  "comprehensive casino guide reviews rating portal list all",
  "professional tested casinos ranked listed website",
  "community player rated casino review platform site top",
  "fast payout online casino websites compared list best",
  "high roller VIP casino review comparison website site",
  "newly launched online casino reviewed sites 2025 list all",
  "best mobile casino app reviewed site 2025 top",
  "live dealer casino reviewed ranked listed sites",
  "cryptocurrency crypto bitcoin casino review website compared all",
  "no deposit bonus casino aggregator rated list compared",
  "free spins casino offers comparison reviewed 2025",
  "casino hub guides links site information platform",
  "gambling publisher content platform published listed best",
  "top paying online casino websites ranked rated list best",

  // US / CA
  "real money online casinos USA reviewed ranked compared",
  "legal regulated casinos USA states list compared best",
  "New Jersey casino sites reviewed ranked rated NJ",
  "Pennsylvania PA online casino sites reviewed 2025",
  "Michigan MI legal casino reviewed top all compared",
  "Delaware West Virginia online casino websites reviewed list",
  "sweepstakes social casino reviewed free coins site USA",
  "Canada best online casino sites reviewed compared listed",
  "Ontario regulated iGaming casinos website reviewed best ON",

  // UK / IE
  "UK best online casino review websites GC licensed compared",
  "British recommended casino websites tested rated list",
  "Irish online best casinos reviewed site ranked list",

  // AU / NZ
  "Australia real money pokies casino website ranked rated",
  "New Zealand best internet casinos reviewed listed all",

  // DE/AT/CH
  "beste online casino deutschland getestet websiten verglichen liste",
  "online casino osterreich schweiz webseite vergleich tested beste",
  "casino gmbh vergleich online deutschland website liste beste",

  // NL/BE
  "beste casino online nederland getest vergeleken websitelijst",
  "belgie gelicentieerd online casino website beoordeeld vergelijk",

  // FR
  "meilleurs casinos en ligne france testes compares avis site liste",
  "casinos internet legaux FRANCE comparees meilleures websites testees",

  // ES/PT/IT
  "mejores casinos internet esaa webs comparativas listados resenas",
  "melhores cassino online portugal avaliados comparados melhor lista",
  "migliori siti casino online italiani confrontati recensiti lista",

  // Nordics / Eastern Europe
  "svenska casinon recenserade jamforande hemsidor jamforda lista",
  "norske danske beste bedste nettecasino online sammenlignet anmeldt",
  "suomen best nettikasinot sivustot vertailu paras reviewed arvioiut",
  "poland kasyno online najlepsze polecone strony recenze review sites",
  "ceska slovenska republika nejlepsi casina stranky porovnan hodnoceni",
  "magyarorszag legjobb online casino oldalak osszehasonlitas lista",
  "romania cele mai bune site uri cazinouri online comparativ review",

  // Niche / affiliate
  "casino content sites media publishers affiliates directory gaming",
  "igaming marketing publisher website affiliates listed rated",
  "gambling forum recommended best casinos community voted listed",
  "smaller casino blog review site honest opinion tested reviewed",
  "expert verified licensed checked casino rating platform database",
  "slot machine casino reviewed websites ranked featured all compared",
  "baccarat blackjack roulette online casino reviewed best compared",
  "paypal skrill neteller e-wallet supported casino sites compared",
  "instant withdrawal fastest payout casinos listed reviewed site",
  "progressive jackpot biggest win online casino reviewed list compared",
  "online poker and casino hybrid platform reviewed compared best",
  "sportsbook combined casino gaming site review aggregator listed",
  "best new online casinos launched this month reviewed tested",
  "safest gambling sites licensed regulated UKGC MGA reviewed",
  "top online bingo casino sites reviewed compared rated list",
  "best table game online casinos blackjack roulette comparison site",
  "casino welcome bonus comparison website ranked rated best offers",
  "mobile only online casino app reviewed compared listed best 2025",
  "instant play no download casino websites reviewed compared best",
  "trusted gambling review aggregator database platform sites all",
  "online casino blacklist warning watchdog site verified reviewed",
  "casino affiliate website directory list content platforms 2025",
  "igaming media company review publisher website directory listed",
  "best online betting and casino comparison site ranked reviewed UK",
  "top real money gambling websites compared analyzed rated all 2025",
  "online gaming platform review aggregator sites listed best rated",
  "casino software provider game selection compared reviewed sites all",
  "high RTP high payout rate casinos reviewed compared list best",
  "exclusive bonus code online casino websites reviewed ranked all",
  "vip loyalty rewards program online casino reviewed best compared",
  "mobile responsive online casino websites reviewed compared top",
  "fastest customer support online casino sites reviewed rated best",
  "most game variety slot selection online casinos reviewed list",
  "safest withdrawal methods crypto bank transfer casino site reviewed",
  "online gambling comparison tool website aggregator platform all",
  "casinoreviews.com alternative websites better unbiased compared",
  "alternative to askgamblers trusted independent casino review sites",
  "best casino rating directory platforms database reviewed listed all",
  "top 50 best online casino websites ranked reviewed rated list 2025",
];

// Casino brand / keyword fingerprints
const CASINO_KW = [
  "casino","bet365","betfair","888.","paddy power","ladbrokes","williamhill",
  "unibet","bwin","betway","10bet","skyvegas","mrplay","bovada","ignition",
  "bitsler","marathon","pinnacle","draftkings","fanduel","betmgm",
  "caesar","barstool","pointsbet","leovegas","jackpotcity",
  "royalpalace","casumo","reddog","luckystrike","betonline","intertops",
  "chance.com","betsson","betclic","22bet","1xbet","stake.",
  "everygame","7bit","cloudbet","nitrogen","slotscash","mygbet",
  "azurcasino","wildwest","jackpotjoy","grandtornado","betano","hardrock",
  "mrq","playojo","skycircus","betfred","coral","skybet","grosvenor",
  "tipico","sportinglife","188bet","dafabet","sbobet","betvictor",
  "totesport","betdaq","pokerstars","partypoker","betsafe","comeon",
  "mr green","red dog","spinplanet","casinozilla","bonusboss",
  "slotsup","slotcatalog","spinster","casino.guru","chipy"
].map(k => k.toLowerCase());

// Domains to skip (social media, news, etc.)
const SKIP = [
  "youtube.com","youtu.be","reddit.com","facebook.com","twitter.com","x.com",
  "linkedin.com","tiktok.com","wikimedia.org","wikipedia.","pinterest.",
  "instagram.","medium.com","forbes.com","nytimes.com","amazon.",
  "play.google.com","web.archive.org","duckduckgo.,"startpage.com",
  "brave.com","t.co","imgur.com","flickr.com","goodreads.",
  "steamcommunity.","github.com","stackoverflow."
];

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

function getDomain(url) {
  try { const u = new URL(url);
    let h = u.hostname;
    if (h.startsWith('www.')) h = h.slice(4);
    return h.toLowerCase();
  } catch { return url.toLowerCase(); }
}

function isSkip(d) {
  for (const s of SKIP) if (d.includes(s)) return true;
  return false;
}

// ── Fetch SearXNG JSON via curl (reliable) ─────────────────────
async function apiSearch(query, pg = 1) {
  const url = `${BASE}/search?q=${encodeURIComponent(query)}&format=json&pagenum=${pg}&categories=general&language=all&safe_search=0`;
  try {
    const { stdout } = await execFileP('curl', ['-s', '--max-time', '15', url]);
    if (!stdout) return [];
    const data = JSON.parse(stdout);
    return Array.isArray(data.results) ? data.results : [];
  } catch (e) {
    return [];
  }
}

// ── Fetch site HTML and count casino links ─────────────────────
async function verifyCasinoLinks(url, targetDomain) {
  try {
    const { stdout } = await execFileP('curl', [
      '-s', '--max-time', '10',
      '-H', 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125 Safari/537.36',
      '-L', // follow redirects
      url
    ]);
    if (!stdout || stdout.length < 1000) return 0;

    // Strip scripts and styles to get clean HTML
    const html = stdout.replace(/<script[\s\S]*?<\/script>/gi, ' ')
      .replace(/<style[\s\S]*?<\/style>/gi, ' ');

    // Extract all unique domains linked from the page
    const hrefRe = /href=["'](https?:\/\/[^"'\s>]+)["']/gi;
    const seenCasinoDomains = new Set();

    for (const m of html.matchAll(hrefRe)) {
      let linkUrl = m[1];
      if (!linkUrl.startsWith('http')) continue;
      const d = getDomain(linkUrl);
      if (d === targetDomain || isSkip(d) || seenCasinoDomains.has(d)) continue;

      // Check URL + domain against casino keywords
      const lo = linkUrl.toLowerCase();
      for (const kw of CASINO_KW) {
        if (lo.includes(kw)) { seenCasinoDomains.add(d); break; }
      }
    }
    return seenCasinoDomains.size;
  } catch { return 0; }
}

// ─────────────── MAIN ────────────────────────────────────────
(async () => {
  console.log('═══ Casino Affiliate Crawler v4 (curl via child_process) ═══');
  console.log(`Queries : ${Q.length}`);
  console.log(`Goal    : 1000+ sites with >=5 casino links\n`);

  // ── Phase 1: Collect unique domain URLs from SearXNG ──────
  const uniqueSites = [];   // [{url, title, domain}]
  const seenDomains = new Set();
  let qDone = 0;

  const BAND = 5;           // concurrent query fetches per band
  const PAGES = 18;         // max pages per query

  for (let bStart = 0; bStart < Q.length; bStart += BAND) {
    const bQs = Q.slice(bStart, bStart + BAND);
    qDone += bQs.length;

    for (let pg = 1; pg <= PAGES; pg++) {
      // Fire all queries in band concurrently at this page
      const promises = bQs.map(q => apiSearch(q, pg));
      const htmlResults = await Promise.all(promises);

      let empty = 0;
      for (const results of htmlResults) {
        if (!results.length) { empty++; continue; }
        for (const r of results) {
          const d = getDomain(r.url);
          if (!seenDomains.has(d)) {
            seenDomains.add(d);
            uniqueSites.push({ url: r.url, title: (r.title||'').substring(0,250), domain: d });
          }
        }
      }

      if (empty === bQs.length) break; // all queries exhausted for this page
      await sleep(450);  // rate limit between pages
    }

    console.log(`[b${Math.floor(bStart/BAND)+1}] ${uniqueSites.length} unique domains`);
    await sleep(600);
  }

  const totalCount = `${seenDomains.size}`;
  console.log(`\nPhase 1 done: ${totalCount} unique domains\n`);

  // ── Phase 2: Verify ≥5 casino links (concurrent batches) ────────
  const verified = [];
  let checked = 0;
  const BATCH_SIZE = 4;    // concurrent URL fetches per batch

  for (let i = 0; i < uniqueSites.length; i += BATCH_SIZE) {
    const chunk = uniqueSites.slice(i, i + BATCH_SIZE);
    const checks = chunk.map(s => verifyCasinoLinks(s.url, s.domain));
    const counts = await Promise.all(checks);

    for (let j = 0; j < chunk.length; j++) {
      if (counts[j] >= 5) {
        verified.push({
          url: chunk[j].url,
          title: chunk[j].title,
          domain: chunk[j].domain,
          casinoLinks: counts[j]
        });
      }
      checked++;
    }

    if (checked % 100 === 0 || checked === uniqueSites.length) {
      console.log(`Phase 2: ${checked}/${uniqueSites.length} → ${verified.length} pass >=5`);
    }
    await sleep(200); // small throttle between batches
  }

  // ── Write CSV ───────────────────────────────
  const header = 'url,title,domain,casino_links';
  const rows = verified.map(v => {
    const t = (v.title || '').replace(/"/g, "'");
    return `"${v.url}","${t}","${v.domain}",${v.casinoLinks}`;
  });
  fs.writeFileSync(CSV_FILE, [header, ...rows].join('\n'), 'utf8');

  console.log(`\n═══════════ ${verified.length} verified sites → ${CSV_FILE} ═══════════`);
})();
