jivanrij/anno-117-docs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding data and header message

Browse Source
This commit is contained in:
Jonathan van Rij
2025-12-30 15:10:22 +01:00
parent df2b557585
commit 8dbdf7a50e
94 changed files with 179885 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/scraper.py
View File

@@ -105,7 +105,9 @@ def scrape_page(url: str) -> dict:
print(f"Scraping: {url}")
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
"User-Agent": "Anno117DocBot/1.0 (One-time scrape for AI agent docs; https://git.blijnder.com/jivanrij/anno-117-docs; Contact: jivanrij@gmail.com)",
"X-Bot-Purpose": "Creating a markdown file with Anno 117 data for my AI agent (personal use, non-commercial). This way the agent doesn't need to query your website multiple times - each page is only fetched once. Result: https://git.blijnder.com/jivanrij/anno-117-docs/raw/branch/main/docs.md - Thank you for anno.land!",
"From": "jivanrij@gmail.com",
}
response = requests.get(url, headers=headers, timeout=30)