max_request_retries
and max_session_rotations
) and do you handle the cases of getting an error somehow additionally?ConcurrencySettings
to reduce the aggressiveness of scraping.from apify import Actor, Request from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from .routes import router from crawlee import ConcurrencySettings async def main() -> None: concurrency_settings = ConcurrencySettings( max_concurrency=3, ) async with Actor: # Create a crawler. crawler = BeautifulSoupCrawler( request_handler=router, max_requests_per_crawl=100, max_request_retries=10, concurrency_settings=concurrency_settings ) # Run the crawler with the starting requests. await crawler.run(['https://example.com'])
ignore_http_error_status_codes=[403]