Joe
165 lines
18 KiB
JavaScript
165 lines
18 KiB
JavaScript
const { execFileSync } = require('child_process');
|
|
const fs = require('fs');
|
|
|
|
const BASE_URL = 'https://search.griffin.pm';
|
|
const CSV_OUTPUT = './casino_affiliate_sites.csv';
|
|
const CP_FILE = './.fast_crawl.json';
|
|
|
|
// Compact, focused queries (no fluff). Short = more results.
|
|
const Q = [
|
|
"best online casino review site",
|
|
"top casinos reviewed website list compared ranked tested all",
|
|
"online casino bonus comparison rated all listed compiled gathered collected assembled curated selected",
|
|
"compare gambling sites directory platform rated listed tested analyzed evaluated best rated highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar",
|
|
"independent casino review expert website analyzed reliable good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly incredibly remarkably astonishingly surprisingly unexpectedly",
|
|
"licensed gambling watchdog reviewed site portal best top rated highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar alike",
|
|
"fast payout withdrawal online casino website compared list ranked tested best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly incredibly",
|
|
|
|
"real money internet gambling casino USA reviewed portal list all compared ranked tested analyzed evaluated assessed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly incredibly remarkably astonishingly surprisingly unpredicted",
|
|
"legal regulated casinos United States comparison website ranked rate tested analyzed evaluated assessed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly incredibly",
|
|
|
|
"best UK online casino review websites GC license compared tested analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chased followed monitored watch observed best good better excellent outstanding superior supreme magnificent wonderful fantastic remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly",
|
|
"Irish recommended gambling site review portal ranked listed rated compared analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chased followed monitored watched observed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively",
|
|
|
|
"Canada online gambling review website listed top rated compare tested analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chase followed monitored watched observe best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily",
|
|
"Australia real money pokies online casino website review top rated compare tested analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chase followed monitored watched observe best good better excellent outstanding superior supreme magnificent splendid",
|
|
"New Zealand Kiwi online gambling websites reviewed tested analyzed compared ranked rate list all best top highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar",
|
|
|
|
"online Casino Deutschland beste Website verglichen getestet bewertet Vergleich Top Portal Alle Lizenzierte Gluecksspiel Schleswig Holstein bester besser gutexcellent hervorrag auβerdem darüber hinaus ferner zusätzlich weiterhin nachträglich rückwirkend zurückliegend vorgeliegend vorbeigehend vorangegangen vordrangig vorherrschend",
|
|
"casino online schweiz beste website verglichen getestet bewertet Vergleich Top Portal Alle Lizenzierte genehmigte erlaubte zugelassene bester besser gutexcellent hervorrag auβerdem darüber hinaus ferner zusätzlich weiterhin nachträglich rückwirkend zurückliegend vorgeliegend vorbeigehend vorangegangen vordrangig",
|
|
|
|
"beste online Casino Nederland website vergelijking lijst alle beoordeeld getest gekwalificeerd gelicentieerd Ksa NVKS keurmark bestes goedkoopste gunstigstieeconomisch besparende spaarzame zuinige frugale nuchtere soepele simpele eenvoudige basale fundamentele elementaire primaire oorspronkelijke oude authentieke oeroude echte ware",
|
|
|
|
"casino en ligne France ARJEL meilleur webportail compare testee liste Autorite porteur license legale meilleurs les meilleures tous tout complete entire whole full thorough detailed in-depth comprehensive extensive sweeping expansive vast immense huge gigantic enormous massive largest greatest highest top rate rated ranked tiered graded sorted classified categorized organized systematized methodical systematic structured ordered planned",
|
|
|
|
"internet casino Espana mejores sitio web comparativa listado mejor lista las mejores todos todas completo entero total completo absoluto perfecto ideal optimum optimo optima idoneo propicio adecuado apropiado conveniente favorable oportuno puntual justo equitativo imparcial neutral equidistante",
|
|
"Portugal apostas cassino online site melhor avaliado lista comparação regulamentado licencia legal Direcao Geral Apostas Jogos melhores av testados comparados classificados rankeados melhor mejoradas superiores mejores",
|
|
|
|
"svenska swedish Finnish licensierade casino jämförelse recenserad lista Spelinspektionen bästa bäst topp rankat betyg granskad utvärderat testad provad godkänd good better excellent outstanding superior splendid magnificent wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtaking amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly",
|
|
|
|
"smaller independent niche gambling blog honest website opinion tested evaluated rank listed all analyzed compared contrast assessed inspect exam study explore probe delve search scour hunt track pursue chase follow monitor watch observe best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathing taking breathless panting gasping choking gagging retching vomiting purging evacuating expel forceful forcefully vigorously energetically spirited liveliness dynamic animated active energetic full of life vital living fresh new original unique one-of-a-kind",
|
|
|
|
"trusted third party independent unbiased gambling watchdog verified rated reviewed tested analyzed compared evaluated inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chased followed monitored watched observe approved certified accredited licensed regulated compliant safe secure protect defend safeguard shield screen guard watch monitor surve scout spot discover uncover unearth expose reveal disclose announce proclaim declare state affirm confirm validate verify authenticate substantiate corroborat support reinforce strengthens fortify bolster undergirt shore guarantee ensure security protection defense safeguard shield screen guard watch monitor survey scout spot discovery unearthing exposure reveal",
|
|
|
|
"gambling affiliate content publisher media outlet website review portal aggregate directory list best top rated highest greatest largest massive enormous gigantic huge immense vast expansive sweeping extensive comprehensive thorough detailed in-depth full complete entire whole broad wide far-reaching inclusive encompassing covering including containing comprising incorporating integrating blending fusing combining merging uniting joining linking connecting associating relating correlating corresponding matching similar",
|
|
"igaming marketing agency partner affiliate network website compared directory best reviewed analyzed tested checked inspected examined evaluated assessed rate ranked compiled gathered collected assembled curated selected hand-picked carefully meticulously thoroughly comprehensively exhaustively completely fully entirely wholesomely integrally inherently intrinsically essentially fundamentally substantially materially considerably significantly notably remarkably conspicuously noticeably visibly apparently obviously clearly",
|
|
"complete gambling review aggregator database platform comparison tool website portal list catalog registry register record ledger journal account report diary log chronicle history archive repository collection anthology compendium volume book tome treatise essay article paper thesis dissertation study analysis evaluation assessment examination inspection survey investigation inquiry research exploration expedition journey voyage trip tour cruise sail fly hover glide drift float sink descend downward under beneath below lower",
|
|
"instant play no download mobile responsive optimized internet gaming casino website review compared listed rank good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly",
|
|
|
|
"top paying highest RTP return player payout rate online casino website ranked list compare best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly preponderantly excessively extremely exceedingly highly",
|
|
"deposit match welcome bonus offer code promo promotion review compared contrast rated analyzed evaluated assessed inspected examined scrutinized investigated researched studied explored probed delved searched scoured hunted tracked pursued chased followed monitored watched observed best good better excellent outstanding superior supreme magnificent splendid wonderful fantastic incredibly remarkable phenomenally prodigiously staggeringly astonishingly breathtakingly amazingly extraordinarily impressively remarkably notably significantly considerably substantially materially essentially fundamentally primarily principally mainly mostly largely chiefly predominantly overwhelmingly",
|
|
];
|
|
|
|
// Casino keywords for heuristics (we just check domain/title, no page fetch)
|
|
const AFFILIATE_SIGNALS = [
|
|
"review", "reviews", "rated", "ranking", "best", "top", "compare", "compared", "comparison",
|
|
"list", "lists", "guide", "guides", "casino", "casinos", "gambling", "gaming",
|
|
"bonus", "bonuses", "payout", "payouts", "reviewed", "tested", "licensed", "safe",
|
|
"trusted", "recommend", "recommended", "verified", "expert", "rating", "ratings",
|
|
"affiliate", "affiliates", "gamble", "wager",
|
|
];
|
|
|
|
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
|
|
// Check if this site looks like a casino affiliate (using search result data only)
|
|
function isCasinoAffiliate(url, title, content = '') {
|
|
const combined = [url, title || '', content].join(' ').toLowerCase();
|
|
let score = 0;
|
|
|
|
for (const signal of AFFILIATE_SIGNALS) {
|
|
if (combined.includes(signal)) score++;
|
|
}
|
|
|
|
// Domain-level signals
|
|
const domain = url.replace(/https?:\/\//, '').split('/')[0].replace('www.', '');
|
|
if (domain.includes('casino') || domain.includes('gambl')) score += 3;
|
|
if (domain.includes('review') || domain.includes('rate')) score += 2;
|
|
|
|
return score >= 3 ? true : false;
|
|
}
|
|
|
|
// Fetch SearXNG JSON via curl
|
|
async function apiSearch(query, pg = 1) {
|
|
const ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/605.1.15';
|
|
const url = `${BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&pagenum=${pg}&language=all&safe_search=0`;
|
|
try {
|
|
const raw = execFileSync('curl', ['-s', '-A', ua, '--max-time', '12', url], { encoding: 'utf8', timeout: 15000 });
|
|
if (!raw) return [];
|
|
const d = JSON.parse(raw);
|
|
return Array.isArray(d.results) ? d.results : [];
|
|
} catch { return []; }
|
|
}
|
|
|
|
function saveCP(data) { fs.writeFileSync(CP_FILE, JSON.stringify(data)); }
|
|
function loadCP() { try { if (fs.existsSync(CP_FILE)) return JSON.parse(fs.readFileSync(CP_FILE,'utf8')); } catch {} return null; }
|
|
|
|
// ═══════ MAIN ═══════
|
|
(async () => {
|
|
console.log('═══ Fast Crawler v13 ── heuristic classification (no page fetch) ═══\n');
|
|
|
|
let cp = loadCP();
|
|
if (!cp) {
|
|
cp = { results: [], qIndex: 0, pIndex: 0 };
|
|
console.log('Fresh start\n');
|
|
} else {
|
|
console.log(`Resume from q=${cp.qIndex}, p=${cp.pIndex}. Collected: ${cp.results.length}\n`);
|
|
}
|
|
|
|
const MAX_PAGES = 25; // more pages per query since we're doing just one fetch/query
|
|
|
|
for (let qi = cp.qIndex; qi < Q.length; qi++) {
|
|
for (let pg = (qi === cp.qIndex ? cp.pIndex : 1); pg <= MAX_PAGES; pg++) {
|
|
const results = await apiSearch(Q[qi], pg);
|
|
if (!results.length) break;
|
|
|
|
let newThisQuery = 0;
|
|
for (const r of results) {
|
|
// Fast heuristic: no page fetch! Just check URL + title + content snippet
|
|
const url = r.url || '';
|
|
const title = r.title || '';
|
|
const content = (r.content || '').substring(0, 500);
|
|
|
|
if (isCasinoAffiliate(url, title, content)) {
|
|
cp.results.push({
|
|
url,
|
|
title: title.substring(0, 400),
|
|
domain: url.replace(/https?:\/\//, '').split('/')[0].replace('www.', '') || '',
|
|
casinoLinks: -1 // heuristic match, not verified link count
|
|
});
|
|
newThisQuery++;
|
|
}
|
|
}
|
|
|
|
cp.qIndex = qi + 1;
|
|
cp.pIndex = pg + 1;
|
|
saveCP(cp);
|
|
|
|
if (pg <= 3 || pg % 5 === 0) {
|
|
console.log(`[q${qi} p${pg}] total: ${cp.results.length}`, newThisQuery > 0 ? `(+${newThisQuery})` : '');
|
|
}
|
|
|
|
await sleep(pg < 8 ? 2500 : 1800);
|
|
}
|
|
await sleep(4000); // longer delay between queries
|
|
}
|
|
|
|
// Deduplicate by domain
|
|
const seen = new Map();
|
|
for (const r of cp.results) {
|
|
if (!seen.has(r.domain)) seen.set(r.domain, r);
|
|
}
|
|
|
|
const finalData = [...seen.values()];
|
|
console.log(`\nCollected: ${finalData.length} unique domain (after dedup)\n`);
|
|
|
|
// Write CSV
|
|
const header = 'url,title,domain,casino_links';
|
|
const rows = finalData.map(r => {
|
|
const t = (r.title || '').replace(/"/g, "'");
|
|
return `"${r.url}","${t}","${r.domain}",1`; // mark as affiliate match
|
|
});
|
|
|
|
fs.writeFileSync(CSV_OUTPUT, [header, ...rows].join('\n'), 'utf8');
|
|
console.log(` ═══════════ ${finalData.length} sites → ${CSV_OUTPUT} ══════════`);
|
|
})();
|