CRAWLEE_PURGE_ON_START=false
so it only run the previously problematic urls. Iterate few times to catch all bugs, and then run the whole crawler with purged storage.failedRequestHandler: async (context, error) => { if (os.platform() === `darwin`) { const {request} = context fs.writeFileSync(debugFilePath, JSON.stringify(request, null, 2)) console.log(`Stored failed request to ${debugFilePath}, if not deleted, it will be used for request list the next time you run the actor.`) } }
if (os.platform() === `darwin`) { const files = fs.readdirSync(debugDir).filter(file => file.endsWith(`.json`)) // for (const file of files) { // const request = JSON.parse(fs.readFileSync(filePath, `utf8`)) await crawler.run(requests) }