Joe
55 lines
1.7 KiB
JavaScript
55 lines
1.7 KiB
JavaScript
const { pool } = require('./src/db');
|
|
|
|
(async () => {
|
|
var CasinoCrawlerClass = require('./src/services/crawler');
|
|
var crawler = new CasinoCrawlerClass();
|
|
|
|
try {
|
|
await crawler.initialize();
|
|
|
|
// Test two real casino review affiliate sites
|
|
const urlsToTest = [
|
|
'https://chipy.com/casinos',
|
|
'https://gamezinger.com/online-casinos/'
|
|
];
|
|
|
|
for (const testUrl of urlsToTest) {
|
|
console.log('\nTesting:', testUrl);
|
|
|
|
var pg = await crawler.browser.newPage();
|
|
await pg.setUserAgent(
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
|
);
|
|
|
|
// Navigate to site and wait for content
|
|
await page.goto(testUrl, { waitUntil: 'domcontentloaded', timeout: 12000 });
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
const title = await pg.title();
|
|
console.log('Title:', title.substring(0, 80));
|
|
|
|
// Run extractors
|
|
var extracted = await crawler.extractCasinoData(pg);
|
|
console.log('Extractor found', extracted.length, 'casino entries:');
|
|
for (var i = 0; i < Math.min(15, extracted.length); i++) {
|
|
var c = extracted[i];
|
|
var linkStr = '';
|
|
if (c.link) {
|
|
try { var linkStr = new URL(c.link).origin;
|
|
} catch(e) { linkStr = c.link.substring(0, 60); }
|
|
}
|
|
console.log(' #' + (i + 1) + ' Name:', c.name, '| Link:', linkStr.substring(0, 50));
|
|
}
|
|
|
|
try { await pg.close(); } catch(e) {}
|
|
}
|
|
|
|
} catch(err) {
|
|
console.error('\nTest crashed:', err.message);
|
|
} finally {
|
|
if (crawler.browser) await crawler.close().catch(() => {});
|
|
await pool.end().catch(() => {});
|
|
process.exit(0);
|
|
}
|
|
})();
|