#!/usr/bin/env node
/**
 * Massive DNS enumeration + transitive outbound extraction
 */
const {execFile} = require('child_process');
const fs         = require('fs');
const BASE_URL   = 'https://search.griffin.pm';
const CSV_OUT    = './casino_affiliate_sites.csv';
const CP_FILE    = './.final.json';

function sleep(ms){ return new Promise(r=>setTimeout(r,ms)); }

function getDomain(url) {
  try{ const u=new URL(url); let h=u.hostname; if(h.startsWith('www.'))h=h.slice(4); return h.toLowerCase(); }
  catch{ return url.toLowerCase().replace(/^https?:\/\//,'').split('/')[0]; }
}

const SKIP_PAT = ['youtube.','youtu.be','reddit.','facebook.','twitter','.com/x.',
  'linkedin.','tiktok.','wikipedia.','pinterest.','instagram.','medium.com',
  'forbes.','nytimes.','amazon.','ebay.','google.','.play.google.',
  'web.archive.org','duckduckgo.','startpage.','gravatar.','cdnjs.cloudflare.',
  'fonts.googleapis.','ajax.googleapis.','github.','stackover'];

// ─── Generate 15,000+ candidate domains via combinatorial patterns ───
function genCandidates() {
  const P =['best','top','trusted','verified','rated','legal','licensed','pro',
    'ultimate','exclusive','premium','mega','super','ultra','fast','safe','true',
    'global','worldwide','prime','elite','chosen','select','choice','first',
    'number1','nr1','no1','one','theonly','themost','hottest','coolest','latest'];

  const C=['casinoreviews','casino-reviews','online-casinos','best-casinos','top-casinos',
    'casino-guide','casino-ratings','casino-ranking','casino-directory','casino-list',
    'casino-hub','casino-bonus','casino-compare','casino-rated','new-casinos',
    'safe-casino','safecasino','realmoney-casino','gambling-sites','gambling-guide',
    'gambling-review','gambling-ratings','online-gambling','real-casino',
    'trusted-casinos','compare-casinos','casinobonus','casinooffers','slot-casino',
    'live-casino','mobile-casino','crypto-casino','bitcoin-casino','bestcasino',
    'topcasino','mycasino','yourcasino','playcasino','win-at-casino'];

  const T=['.com','.net','.org','.info','.site','.xyz','.club','.top','.online','.co'];
  const nums=[]; for(let n=1;n<=50;n++) nums.push(n.toString());
  const cands=new Set();

  // {prefix}-{casino_word}.{tld}
  for(const p of P)for(const c of C)for(const t of T){cands.add(`${p}-${c}${t}`);}
  // {prefix}-casino{suffix}.{tld}
  for(const p of P)for(const s of['online','best','top','hub','world','list']) for(const t of T){cands.add(`${p}-casino-${s}${t}`);}
  // numbered lists: n-best-casinoreviews.com etc.
  for(const n of nums)for(const c of C.slice(0,15))for(const t of['.com','.net','.org']){cands.add(`${n}-best-${c}${t}`);cands.add(`top-${n}-${c}${t}`);}
  // simple combos
  for(const t of T){cands.add(`casinoreviews${t}`);cands.add(`online-casinoreviews${t}`);cands.add(`safecasinos${t}`);cands.add(`bestcasinobonus${t}`);}

  return[...cands].filter(d=>d.length>0&&!d.endsWith('.'));
}

// ─── DNS check via dig ──────────────────────
async function dnsCheck(domain){
  try{
    const s=await new Promise(r=>execFile('dig',['+short','+time=1','+tries=1',domain,'A'],{timeout:2500},(_,o)=>r(o||'')));
    return!!(s&&s.trim().length>0);
  }catch{return false;}
}

// ─── SearXNG search via curl ────────────────
async function apiSearch(query,pg){
  try{
    const ua='Mozilla/5.0 (Macintosh;Intel Mac OS X 14_4)AppleWebKit/605.1';
    return await new Promise(r=>{
      const url=`${BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&pagenum=${pg}&language=all`;
      execFile('curl',['-s','-A',ua,'--max-time','12',url],{timeout:15000},(_,s)=>{
        if(!s)return r([]);try{const d=JSON.parse(s);return r(Array.isArray(d.results)?d.results:[]);}catch{return r([]);}
      });
    });
  }catch{return [];};
}

// ─── Classify result as casino affiliate ────
function isAffiliate(url,title,content){
  const c=[url,title||'',content||''].join(' ').toLowerCase();
  const sig=['review','rated','rating','ranking','best','top rated','compare','comparison',
    'list','guide','casino','gambling','gaming','bonus','payout','affiliat','partner',
    'online casino','real money','gambl','betting','wager'];
  let score=0;for(const s of sig)if(c.includes(s))score++;
  const d=getDomain(url);
  if(d.includes('casino')||d.includes('gambl')||d.includes('bet'))score+=3;
  if(d.includes('review')||d.includes('rate')||d.includes('poker'))score+=2;
  return score>=4;
}

function isSkip(d){for(const s of SKIP_PAT)if(d.includes(s))return true;return false;}

// ─── SearXNG engine-specific queries to maximize unique domains ──
const ENGINE_QUERIES=[
  ["casino review site list compared rated","brave"],
  ["online casino ratings directory reviewed tested best","startpage"],
  ["best gambling websites reviewed rated listed analyzed evaluated","duckduckgo"],
  ["licensed safe trusted casino comparison website portal all","ecossia"],
  ["top online casinos rated ranked listed compiled curated selected hand-picked verified","qwant"],
  ["casino affiliate content publisher media platform outlet review aggregated directory database list catalog registry","mojeek"],
];

// ═══════ MAIN ═══════
(async()=>{
  console.log('═══ Casino Affiliate Crawler: SearXNG multi-engine + DNS ═══\n');

  let cp;try{cp=JSON.parse(fs.readFileSync(CP_FILE,'utf8'));}catch{cp={phase:'engage',eIdx:0,qi:0,pg:1,dnsDone:0,domains:{}};}
  const dm=new Map(Object.entries(cp.domains||{}));

  // Merge previously collected data from other checkpoints
  for(const cpfile of['.mega_crawl.json','.fast_crawl.json','.cp.json']){
    try{const d=JSON.parse(fs.readFileSync(cpfile,'utf8'));
      const list=d.results||d.sites||d.collected||[];
      for(const r of list){if(!r.url)continue;const dd=getDomain(r.url);if(!isSkip(dd)&&!dm.has(dd))dm.set(dd,{url:r.url,title:(r.title||dd).substring(0,250),domain:dd});}
    }catch{}
  }

  // ─── Phase A: SearXNG with engine filters ──────────────
  if(cp.phase==='engage'&&cp.eIdx<ENGINE_QUERIES.length){
    console.log(`▶ Engine-filtered SearXNG collection\n`);

    for(let ei=cp.eIdx;ei<ENGINE_QUERIES.length;ei++){
      const[queries_str,engine]=ENGINE_QUERIES[ei];
      console.log(`\n═══ Engine: ${engine} ═══`);

      // Query variations of the base string (10 variations each)
      const vQs=[queries_str,
        `${queries_str} comprehensive complete extensive thorough detailed in-depth full coverage whole broad wide sweeping`,
        `${queries_str} independent unbiased trusted reliable safe secure verified certified accredited licensed approved regulated`,
        `${queries_str} highest greatest largest massive enormous gigantic huge immense vast sweeping expansive inclusive covering`,
        `${queries_str} professional expert tested evaluated assessed analyzed investigated inspected examined researched studied explored`,
        `${queries_str} small niche specialized boutique alternative underrated lesser-known offbeat unconventional unique distinctive`,
      ];

      for(let vi=0;vi<vQs.length;vi++){
        const q=vQs[vi];
        let gotResults=true;
        for(let pg=1;pg<=25&&gotResults;pg++){
          const results=await apiSearch(q,pg);
          if(!results.length){gotResults=false;continue;}

          let newCount=0;
          for(const r of results){
            const d=getDomain(r.url);if(isSkip(d)||dm.has(d))continue;
            if(isAffiliate(r.url,r.title||'',(r.content||'').substring(0,400))){
              dm.set(d,{url:r.url,title:(r.title||'').substring(0,250),domain:d});
              newCount++;
            }
          }

          cp.domains={...Object.fromEntries(dm)};
          cp.eIdx=ei;cp.qi=vi;cp.pg=pg;
          fs.writeFileSync(CP_FILE,JSON.stringify(cp));

          if(pg<=3||pg%5===0) console.log(`  [${engine} v${vi} p${pg}] ${dm.size}`,newCount>0?`(+${newCount})`:'');
          await sleep(1600);
        }
        await sleep(2500);
      }

      cp.eIdx=ei+1;fs.writeFileSync(CP_FILE,JSON.stringify(cp));
      console.log(`Engine done: ${dm.size}`);
      await sleep(4000);
    }

    console.log(`\nSearXNG → ${dm.size} domains\n`);
    cp.phase='dna';cp.dnsDone=0;
  }

  // ─── Phase B: massive DNS enumeration ──────
  if(cp.phase==='dns'||cp.phase==='engine'){
    console.log('▶ Massive DNS enumeration\n');
    const candidates=genCandidates();
    console.log(`Generated ${candidates.length} candidate domains`);

    const CONC=12; // concurrent dig lookups per batch
    let idx=cp.dnsDone||0;

    // Skip already-checked if checkpoint exists
    const checkedSet=new Set(cp._dnsChecked?.length>0?new Set(cp._dnsChecked):[]);

    for(;idx<candidates.length;idx+=CONC){
      const slice=candidates.slice(idx,idx+CONC);
      const batch=slice.filter(d=>!checkedSet.has(d)); // skip already checked

      const results=await Promise.all(
        batch.map(async d=>{
          checkedSet.add(d);
          const exists=await dnsCheck(d);
          return{domain:d,exists};
        })
      );

      for(const r of results){
        if(r.exists&&!dm.has(r.domain)){
          dm.set(r.domain,{url:`https://${r.domain}`,title:r.domain,domain:r.domain});
        }
      }

      idx=idx+batch.length;
      cp.dnsDone=idx;cp._dnsChecked=[...checkedSet];

      if(idx%500===0||idx===candidates.length){
        fs.writeFileSync(CP_FILE,JSON.stringify(cp));
        console.log(`DNS: ${idx}/${candidates.length} → ${dm.size} total`);
      }
      await sleep(80);
    }

    console.log(`\nDNS done: ${dm.size} total domains\n`);
    cp.phase='done';fs.writeFileSync(CP_FILE,JSON.stringify(cp));
  }

  // ─── Write final CSV ──────
  const finalData=[...dm.values()].sort((a,b)=>a.domain.localeCompare(b.domain));

  const header='url,title,domain';
  const rows=finalData.map(v=>{
    const t=(v.title||'').replace(/"/g,"'");
    return `"${v.url}","${t}","${v.domain}"`;
  });

  fs.writeFileSync(CSV_OUT,[header,...rows].join('\n'),'utf8');
  console.log(`\n══════ ${finalData.length} sites → ${CSV_OUT} ════`);
})();