Apify and Crawlee Official Forum
b
F
A
J
A
const blockedResourceTypes = ['webp', 'svg', 'mp4', 'jpeg', 'gif', 'avif', 'font']
const crawler = new PuppeteerCrawler({
launchContext: {
launchOptions: {
headless: false,
devtools: true,
defaultViewport:{ width: 1920, height: 6000 },
args: [
'--disable-dev-shm-usage',
]
},
useIncognitoPages: true,
},
proxyConfiguration,
requestHandler: router,
maxConcurrency: 16,
maxRequestRetries: 15,
maxRequestsPerMinute: 2,
navigationTimeoutSecs: 120,
useSessionPool: true,
failedRequestHandler({ request }) {
log.debug(`Request ${request.url} failed 15 times.`);
},
preNavigationHooks: [
async ({ addInterceptRequestHandler }) => {
await addInterceptRequestHandler((request) => {
if (blockedResourceTypes.includes(request.resourceType())) {
return request.respond({
status: 200,
body: 'useless shit',
});
}
return request.continue();
});
},
],
});const abortAssets: PlaywrightHook = async ({ page }) => {
const RESOURCE_EXCLUSIONS = ['image', 'media', 'font', 'stylesheet'];
await page.route('**/*', (route) => {
if (RESOURCE_EXCLUSIONS.includes(route.request().resourceType())) {
return route.abort();
}
return route.continue();
});
};
const playwrightCrawler = new PlaywrightCrawler({
// ...
preNavigationHooks: [
abortAssets,
],
// ...
});const abortAssets = async ({ page }) => {
const RESOURCE_EXCLUSIONS = ['image', 'media', 'font', 'stylesheet'];
await page.setRequestInterception(true);
await page.on('request', (request) => {
if (RESOURCE_EXCLUSIONS.includes(request.resourceType())) {
return request.abort();
}
return request.continue();
});
};
const crawler = new PuppeteerCrawler({
preNavigationHooks: [
abortAssets,
],
headless: false,
proxyConfiguration,
requestHandler: router,
});