const crawler = new PuppeteerCrawler({
proxyConfiguration,
launchContext: {
// launcher: puppeteerExtra,
// useIncognitoPages: true,
launchOptions:{
args: [
'--disable-gpu',
'--disable-extensions',
'--disable-webgl',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-accelerated-jpeg-decoding',
'--disable-accelerated-mjpeg-decode',
'--disable-accelerated-video-decode',
'--disable-software-rasterizer',
'--disable-notifications',
'--disable-background-networking',
'--disable-background-timer-throttling'
],
}
},
requestHandler: router,
maxConcurrency: 6,
maxRequestRetries: 50,
useSessionPool: true,
navigationTimeoutSecs: 90,
requestHandlerTimeoutSecs: 90,
failedRequestHandler({ request }) {
log.debug(`Request ${request.url} failed 50 times.`);
},
preNavigationHooks: [
abortAssets,
async (_, gotoOptions) => {
gotoOptions.waitUntil = "domcontentloaded";
}
],
});
3rd party extensions are disabled.