chromium.use(stealthPlugin());
let queue = await RequestQueue.open('crawler');
await queue.drop();
queue = await RequestQueue.open('crawler');
const startUrls = [`url`];
const router = await initRouter({ resume, numPages, initialPage });
const crawler = new PlaywrightCrawler({
requestHandler: router,
maxRequestsPerMinute: 100,
log: new Log({
logger: new CrawlerLogger(log.getOptions(), 'CRAWLER_1'), // please ignore, custom logger imp
level: log.LEVELS.DEBUG,
}),
requestQueue: queue,
launchContext: {
launcher: chromium,
launchOptions: {
args: ['--ignore-certificate-errors'],
},
},
...(useProxy && {
proxyConfiguration: new ProxyConfiguration({
proxyUrls: [
proxy,
],
}),
useSessionPool: true,
persistCookiesPerSession: true,
}),
});
await crawler.run(startUrls, {});