from crawlee import ConcurrencySettings crawler = ParselCrawler(concurrency_settings=ConcurrencySettings(max_tasks_per_minute=100))
from crawlee import ConcurrencySettings crawler = ParselCrawler(concurrency_settings=ConcurrencySettings(max_tasks_per_minute=100))
max_tasks_per_minute
will give you better control over your parsing speed, especially if you encounter 429 blocking (too frequent requests to the server)