Apify and Crawlee Official Forum
b
F
A
J
A
requests library or similar. We need to see the code ofc.
from urllib.parse import urljoin
from bs4 import BeautifulSoup
import requests
# Define a function to scrape the given URL up to the specified maximum depth
def scrape(url, depth, max_depth):
if depth > max_depth:
return
print(f'Scraping {url} at depth {depth}...')
# Try to send a GET request to the URL
try:
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
# If we haven't reached the max depth, look for nested links and enqueue their targets
if depth < max_depth:
for link in soup.find_all('a'):
link_href = link.get('href')
if link_href and link_href.startswith(('http://', 'https://')):
link_url = urljoin(url, link_href)
print(f'Found link: {link_url}')
scrape(link_url, depth + 1, max_depth)
# Extract and print the title of the page
title = soup.title.string if soup.title else "No Title"
print(f'Title: {title}')
except requests.exceptions.RequestException as e:
print(f'An error occurred: {e}')
# Main function to start scraping
def main():
start_urls = [{'url': 'https://www.tripadvisor.com/'}] # Example start URL
max_depth = 1 # Example max depth
# Start scraping from the first URL
for start_url in start_urls:
url = start_url.get('url')
print(f'Starting scrape for: {url}')
scrape(url, 0, max_depth)
if __name__ == "__main__":
main()