crawler/dns-enumerator.js

const { execFile } = require('child_process');
const fs = require('fs');

const CSV_FILE = './casino_affiliate_sites.csv';
const CP_FILE  = './.dns.json';

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

function generateCandidates() {
  const PFX = ['best','top','trusted','verified','rated','legal','licensed','pro',
    'ultimate','exclusive','premium','mega','super','ultra','fast','safe'];
  const KW  = ['casinoreviews','casino-reviews','online-casinos','best-casinos','top-casinos',
    'casino-guide','casino-ratings','casino-ranking','casino-directory','casino-list',
    'casino-hub','safe-casino','new-casinos'];
  const TLD = ['.com','.net','.org','.info','.site','.xyz'];

  const candidates = [];
  for (const p of PFX) {
    for (const k of KW) {
      for (const t of TLD) {
        candidates.push(`${p}-${k}${t}`);
      }
    }
    for (const t of TLD) {
      candidates.push(`${p}-casino${t}`);
      candidates.push(`${p}-casinos${t}`);
    }
  }

  for (let n = 1; n <= 20; n++) {
    for (const t of ['.com','.net','.org']) {
      candidates.push(`${n}-best-casinoreviews${t}`);
      candidates.push(`top-${n}-casinos${t}`);
      candidates.push(`best-${n}-online-casinos${t}`);
    }
  }

  for (const t of ['.com','.net','.org']) {
    candidates.push(`casinoreviews${t}`);
    candidates.push(`bestcasinoreviews${t}`);
    candidates.push(`topcasinoreviews${t}`);
    candidates.push(`trustedcasinos${t}`);
    candidates.push(`safecasinos${t}`);
    candidates.push(`legalonlinecasinos${t}`);
    candidates.push(`casino-directory${t}`);
  }

  return [...new Set(candidates)].filter(d => d.length > 0 && !d.endsWith('.'));
}

async function checkDNS(domain) {
  try {
    const s = await new Promise((resolve) => {
      execFile('dig', ['+short','+time=1','+tries=1', domain, 'A'],
        { timeout: 3000 }, (_, o) => resolve(o || ''));
    });
    return !!(s && s.trim().length > 0);
  } catch { return false; }
}

function getDomain(url) {
  try { const u = new URL(url); let h = u.hostname; if (h.startsWith('www.')) h = h.slice(4); return h.toLowerCase(); }
  catch { return url.toLowerCase(); }
}

(async () => {
  console.log('═══ DNS Domain Enumerator ═══\n');

  let cp;
  try {
    if (fs.existsSync(CP_FILE)) cp = JSON.parse(fs.readFileSync(CP_FILE, 'utf8'));
    else cp = { checked: {}, found: [] };
  } catch { cp = { checked: {}, found: [] }; }

  const candidates = generateCandidates();
  console.log(`Generated ${candidates.length} candidate domains\n`);

  const CONC = 8;
  let checkedCount = Object.keys(cp.checked).length || 0;

  for (let i = 0; i < candidates.length; ) {
    if (cp.checked[candidates[i]]) { i++; continue; }

    const end = Math.min(i + CONC, candidates.length);
    const batch = candidates.slice(i, end);
    i += CONC;

    const promises = batch.map(async d => {
      cp.checked[d] = true;
      const exists = await checkDNS(d);
      return { domain: d, exists };
    });

    const results = await Promise.all(promises);

    for (const r of results) {
      if (r.exists && !cp.found.some(f => f.domain === r.domain)) {
        cp.found.push({ url: `https://${r.domain}`, title: '', domain: r.domain });
      }
    }

    checkedCount += batch.length;
    if (checkedCount % 100 === 0) {
      fs.writeFileSync(CP_FILE, JSON.stringify(cp));
      console.log(`Checked: ${checkedCount}/${candidates.length} → Found: ${cp.found.length}`);
    }
    await sleep(50);
  }

  // Merge all collected data
  const allSites = new Map();

  // DNS found domains
  for (const f of cp.found) {
    if (!allSites.has(f.domain)) allSites.set(f.domain, f);
  }

  // Load from mega crawler checkpoint
  try {
    const mcData = JSON.parse(fs.readFileSync('.mega_crawl.json', 'utf8'));
    if (mcData && mcData.results) {
      for (const r of mcData.results) {
        if (!r.url) continue;
        const d = getDomain(r.url);
        if (!allSites.has(d)) {
          allSites.set(d, { url: r.url, title: (r.title||'').substring(0,200), domain: d });
        }
      }
    }
  } catch {}

  const finalData = [...allSites.values()].sort((a,b) => a.domain.localeCompare(b.domain));

  const header = 'url,title,domain';
  const rows = finalData.map(v => {
    const t = (v.title || '').replace(/"/g, "'");
    return `"${v.url}","${t}","${v.domain}"`;
  });

  fs.writeFileSync(CSV_FILE, [header, ...rows].join('\n'), 'utf8');

  console.log(`\n══ Final: ${finalData.length} sites → ${CSV_FILE} ════`);
})();