diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 6c8506b..efb41b6 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -6,7 +6,8 @@ "Bash(pip3 install:*)", "Bash(source:*)", "Bash(pip install:*)", - "Bash(python python/scraper.py:*)" + "Bash(python python/scraper.py:*)", + "Bash(chmod:*)" ] } } diff --git a/README.md b/README.md index cd934c8..0c7790a 100644 --- a/README.md +++ b/README.md @@ -19,60 +19,38 @@ venv/bin/python python/scraper.py -n 9999 This scrapes unchecked URLs from `scraping.md`, saves JSON to `scraped_data/`, and adds them to `processed.md` as pending. -### 2. Process scraped data into docs.md +### 2. Process scraped data into docs/ -Use this prompt with Claude: +```bash +# Process one file (can run multiple in parallel) +venv/bin/python python/process.py ---- - -**Prompt for Claude:** +# Process multiple in parallel (e.g., 4 at once) +for i in {1..4}; do venv/bin/python python/process.py & done; wait +# Process all remaining files (4 parallel workers) +while venv/bin/python python/process.py; do :; done & +while venv/bin/python python/process.py; do :; done & +while venv/bin/python python/process.py; do :; done & +while venv/bin/python python/process.py; do :; done & +wait ``` -Process pending scraped JSON files into docs.md ONE FILE AT A TIME. -LOOP: Repeat these steps until all files are processed: - -1. Read `processed.md` to find the FIRST pending JSON file (marked with `- [ ]`) -2. If no pending files remain, stop -3. Read ONLY that one file from `scraped_data/` -4. Extract game entities (buildings, goods, production chains, etc.) -5. Translate German names to English: - - Use the game's official English names where known - - Common translations: - - Latium = Latium (Roman region) - - Albion = Albion (Celtic region) - - Liberti = Liberti (Tier 1 Roman) - - Plebejer = Plebeians (Tier 2 Roman) - - Equites = Equites (Tier 3 Roman) - - Patrizier = Patricians (Tier 4 Roman) - - Wanderer = Waders (Tier 1 Celtic) - - Schmiede = Smiths (Tier 2 Celtic) - - Älteste = Elders (Tier 3 Celtic) - - Mercatoren = Mercators (Tier 4 Celtic) - - Edelmänner = Nobles (Tier 5 Celtic) - - Building/Good names: translate to English equivalents -6. Format data according to the schemas defined in docs.md -7. Add new entities or update existing ones in docs.md -8. Mark this JSON file as processed in `processed.md` (change `- [ ]` to `- [x]`) -9. REPEAT from step 1 - -Focus on extracting: -- Buildings: name, category, region, build costs, maintenance, workforce, cycle time, inputs/outputs, area effects, requirements -- Goods: name, category, produced by, consumed by -- Production chains: steps, ratios, cycle times - -Keep entries concise. Mark unknown values as "Unknown" rather than guessing. -``` +The script uses file locking to safely run in parallel. Each invocation: +1. Claims one pending JSON file from `processed.md` +2. Calls Claude to parse it into the `docs/` folder structure +3. Marks it as completed --- ## File Structure - `scraping.md` - URLs to scrape (checkboxes track progress) -- `processed.md` - JSON files pending/processed into docs.md +- `processed.md` - JSON files pending/processed into docs/ - `scraped_data/` - Raw scraped JSON files -- `docs.md` - Final structured documentation +- `docs/` - Structured documentation (see CLAUDE.md for structure) - `python/scraper.py` - Web scraper script +- `python/process.py` - Process one JSON file into docs/ ## Data Flow @@ -80,6 +58,6 @@ Keep entries concise. Mark unknown values as "Unknown" rather than guessing. anno.land pages ↓ (scraper.py) scraped_data/*.json - ↓ (Claude) -docs.md + ↓ (process_one.py → Claude) +docs/**/*.md ``` \ No newline at end of file diff --git a/processed.md b/processed.md index 5494082..7437a34 100644 --- a/processed.md +++ b/processed.md @@ -12,6 +12,47 @@ When Claude processes a JSON file and adds its data to docs.md, mark it here. - [x] anno-117-buildings_makrelenhuette-latium.json ## Pending Files +- [ ] anno-117-buildings_schrein-von-cernunnos-latium.json +- [ ] anno-117-buildings_schrein-von-epona-latium.json +- [ ] anno-117-buildings_schrein-von-ceres-latium.json +- [ ] anno-117-buildings_officium-latium.json +- [ ] anno-117-buildings_villa-des-praetors-latium.json +- [ ] anno-117-buildings_latrine-latium.json +- [ ] anno-117-buildings_wachturm-latium.json +- [ ] anno-117-buildings_brunnen-latium.json +- [ ] anno-117-buildings_medici-latium.json +- [ ] anno-117-buildings_custodia-latium.json +- [ ] anno-117-buildings_vigiles-latium.json +- [ ] anno-117-goods_ruestung.json +- [ ] anno-117-buildings_ruestungsschmied-latium.json +- [ ] anno-117-buildings_pferdezuechter-latium.json +- [ ] anno-117-goods_waffen.json +- [ ] anno-117-buildings_waffenschmied-latium.json +- [ ] anno-117-goods_eisen.json +- [ ] anno-117-buildings_ofen-latium.json +- [ ] anno-117-goods_eisenerz.json +- [ ] anno-117-buildings_eisenmine-latium.json +- [ ] anno-117-buildings_campus-equitum-latium.json +- [ ] anno-117-buildings_belagerungswerkstatt-latium.json +- [ ] anno-117-buildings_kaserne-latium.json +- [ ] anno-117-goods_segel.json +- [ ] anno-117-buildings_segelmacher-latium.json +- [ ] anno-117-goods_seile.json +- [ ] anno-117-buildings_seilmacher-latium.json +- [ ] anno-117-buildings_reparaturkran-latium.json +- [ ] anno-117-buildings_depot-latium.json +- [ ] anno-117-buildings_anlegestelle-latium.json +- [ ] anno-117-buildings_schiffswerft-latium.json +- [ ] anno-117-buildings_lagerhaus-latium.json +- [ ] anno-117-buildings_kontor-latium.json +- [ ] anno-117-buildings_liberti-latium.json +- [ ] anno-117-buildings_koehler-latium.json +- [ ] anno-117-goods_kohle.json +- [ ] anno-117-buildings_koehler-albion.json +- [ ] anno-117-goods_granit.json +- [ ] anno-117-buildings_granitschneider-albion.json +- [ ] anno-117-goods_granitbloecke.json +- [ ] anno-117-buildings_granitsteinbruch-albion.json - [ ] anno-117-buildings_betonmischer-albion.json - [ ] anno-117-buildings_sandraffinerie-albion.json - [ ] anno-117-buildings_kalksteinbruch-albion.json diff --git a/scraping.md b/scraping.md index 15df0d4..36c17ea 100644 --- a/scraping.md +++ b/scraping.md @@ -285,47 +285,47 @@ Scrape the following pages, and their child pages: - [x] https://anno.land/anno-117-buildings/kalksteinbruch-albion/ - [x] https://anno.land/anno-117-buildings/sandraffinerie-albion/ - [x] https://anno.land/anno-117-buildings/betonmischer-albion/ -- [ ] https://anno.land/anno-117-buildings/granitsteinbruch-albion/ -- [ ] https://anno.land/anno-117-goods/granitbloecke/ -- [ ] https://anno.land/anno-117-buildings/granitschneider-albion/ -- [ ] https://anno.land/anno-117-goods/granit/ -- [ ] https://anno.land/anno-117-buildings/koehler-albion/ -- [ ] https://anno.land/anno-117-goods/kohle/ -- [ ] https://anno.land/anno-117-buildings/koehler-latium/ -- [ ] https://anno.land/anno-117-buildings/liberti-latium/ -- [ ] https://anno.land/anno-117-buildings/kontor-latium/ -- [ ] https://anno.land/anno-117-buildings/lagerhaus-latium/ -- [ ] https://anno.land/anno-117-buildings/schiffswerft-latium/ -- [ ] https://anno.land/anno-117-buildings/anlegestelle-latium/ -- [ ] https://anno.land/anno-117-buildings/depot-latium/ -- [ ] https://anno.land/anno-117-buildings/reparaturkran-latium/ -- [ ] https://anno.land/anno-117-buildings/seilmacher-latium/ -- [ ] https://anno.land/anno-117-goods/seile/ -- [ ] https://anno.land/anno-117-buildings/segelmacher-latium/ -- [ ] https://anno.land/anno-117-goods/segel/ -- [ ] https://anno.land/anno-117-buildings/kaserne-latium/ -- [ ] https://anno.land/anno-117-buildings/belagerungswerkstatt-latium/ -- [ ] https://anno.land/anno-117-buildings/campus-equitum-latium/ -- [ ] https://anno.land/anno-117-buildings/eisenmine-latium/ -- [ ] https://anno.land/anno-117-goods/eisenerz/ -- [ ] https://anno.land/anno-117-buildings/ofen-latium/ -- [ ] https://anno.land/anno-117-goods/eisen/ -- [ ] https://anno.land/anno-117-buildings/waffenschmied-latium/ -- [ ] https://anno.land/anno-117-goods/waffen/ -- [ ] https://anno.land/anno-117-buildings/pferdezuechter-latium/ -- [ ] https://anno.land/anno-117-buildings/ruestungsschmied-latium/ -- [ ] https://anno.land/anno-117-goods/ruestung/ -- [ ] https://anno.land/anno-117-buildings/vigiles-latium/ -- [ ] https://anno.land/anno-117-buildings/custodia-latium/ -- [ ] https://anno.land/anno-117-buildings/medici-latium/ -- [ ] https://anno.land/anno-117-buildings/brunnen-latium/ -- [ ] https://anno.land/anno-117-buildings/wachturm-latium/ -- [ ] https://anno.land/anno-117-buildings/latrine-latium/ -- [ ] https://anno.land/anno-117-buildings/villa-des-praetors-latium/ -- [ ] https://anno.land/anno-117-buildings/officium-latium/ -- [ ] https://anno.land/anno-117-buildings/schrein-von-ceres-latium/ -- [ ] https://anno.land/anno-117-buildings/schrein-von-epona-latium/ -- [ ] https://anno.land/anno-117-buildings/schrein-von-cernunnos-latium/ +- [x] https://anno.land/anno-117-buildings/granitsteinbruch-albion/ +- [x] https://anno.land/anno-117-goods/granitbloecke/ +- [x] https://anno.land/anno-117-buildings/granitschneider-albion/ +- [x] https://anno.land/anno-117-goods/granit/ +- [x] https://anno.land/anno-117-buildings/koehler-albion/ +- [x] https://anno.land/anno-117-goods/kohle/ +- [x] https://anno.land/anno-117-buildings/koehler-latium/ +- [x] https://anno.land/anno-117-buildings/liberti-latium/ +- [x] https://anno.land/anno-117-buildings/kontor-latium/ +- [x] https://anno.land/anno-117-buildings/lagerhaus-latium/ +- [x] https://anno.land/anno-117-buildings/schiffswerft-latium/ +- [x] https://anno.land/anno-117-buildings/anlegestelle-latium/ +- [x] https://anno.land/anno-117-buildings/depot-latium/ +- [x] https://anno.land/anno-117-buildings/reparaturkran-latium/ +- [x] https://anno.land/anno-117-buildings/seilmacher-latium/ +- [x] https://anno.land/anno-117-goods/seile/ +- [x] https://anno.land/anno-117-buildings/segelmacher-latium/ +- [x] https://anno.land/anno-117-goods/segel/ +- [x] https://anno.land/anno-117-buildings/kaserne-latium/ +- [x] https://anno.land/anno-117-buildings/belagerungswerkstatt-latium/ +- [x] https://anno.land/anno-117-buildings/campus-equitum-latium/ +- [x] https://anno.land/anno-117-buildings/eisenmine-latium/ +- [x] https://anno.land/anno-117-goods/eisenerz/ +- [x] https://anno.land/anno-117-buildings/ofen-latium/ +- [x] https://anno.land/anno-117-goods/eisen/ +- [x] https://anno.land/anno-117-buildings/waffenschmied-latium/ +- [x] https://anno.land/anno-117-goods/waffen/ +- [x] https://anno.land/anno-117-buildings/pferdezuechter-latium/ +- [x] https://anno.land/anno-117-buildings/ruestungsschmied-latium/ +- [x] https://anno.land/anno-117-goods/ruestung/ +- [x] https://anno.land/anno-117-buildings/vigiles-latium/ +- [x] https://anno.land/anno-117-buildings/custodia-latium/ +- [x] https://anno.land/anno-117-buildings/medici-latium/ +- [x] https://anno.land/anno-117-buildings/brunnen-latium/ +- [x] https://anno.land/anno-117-buildings/wachturm-latium/ +- [x] https://anno.land/anno-117-buildings/latrine-latium/ +- [x] https://anno.land/anno-117-buildings/villa-des-praetors-latium/ +- [x] https://anno.land/anno-117-buildings/officium-latium/ +- [x] https://anno.land/anno-117-buildings/schrein-von-ceres-latium/ +- [x] https://anno.land/anno-117-buildings/schrein-von-epona-latium/ +- [x] https://anno.land/anno-117-buildings/schrein-von-cernunnos-latium/ - [ ] https://anno.land/anno-117-buildings/schrein-von-mars-latium/ - [ ] https://anno.land/anno-117-buildings/schrein-von-merkur-lugus-latium/ - [ ] https://anno.land/anno-117-buildings/schrein-von-minerva-latium/ @@ -1171,3 +1171,44 @@ Scrape the following pages, and their child pages: - [ ] https://anno.land/en/anno-117-buildings/kalksteinbruch-albion/ - [ ] https://anno.land/en/anno-117-buildings/sandraffinerie-albion/ - [ ] https://anno.land/en/anno-117-buildings/betonmischer-albion/ +- [ ] https://anno.land/en/anno-117-buildings/granitsteinbruch-albion/ +- [ ] https://anno.land/en/anno-117-goods/granitbloecke/ +- [ ] https://anno.land/en/anno-117-buildings/granitschneider-albion/ +- [ ] https://anno.land/en/anno-117-goods/granit/ +- [ ] https://anno.land/en/anno-117-buildings/koehler-albion/ +- [ ] https://anno.land/en/anno-117-goods/kohle/ +- [ ] https://anno.land/en/anno-117-buildings/koehler-latium/ +- [ ] https://anno.land/en/anno-117-buildings/liberti-latium/ +- [ ] https://anno.land/en/anno-117-buildings/kontor-latium/ +- [ ] https://anno.land/en/anno-117-buildings/lagerhaus-latium/ +- [ ] https://anno.land/en/anno-117-buildings/schiffswerft-latium/ +- [ ] https://anno.land/en/anno-117-buildings/anlegestelle-latium/ +- [ ] https://anno.land/en/anno-117-buildings/depot-latium/ +- [ ] https://anno.land/en/anno-117-buildings/reparaturkran-latium/ +- [ ] https://anno.land/en/anno-117-buildings/seilmacher-latium/ +- [ ] https://anno.land/en/anno-117-goods/seile/ +- [ ] https://anno.land/en/anno-117-buildings/segelmacher-latium/ +- [ ] https://anno.land/en/anno-117-goods/segel/ +- [ ] https://anno.land/en/anno-117-buildings/kaserne-latium/ +- [ ] https://anno.land/en/anno-117-buildings/belagerungswerkstatt-latium/ +- [ ] https://anno.land/en/anno-117-buildings/campus-equitum-latium/ +- [ ] https://anno.land/en/anno-117-buildings/eisenmine-latium/ +- [ ] https://anno.land/en/anno-117-goods/eisenerz/ +- [ ] https://anno.land/en/anno-117-buildings/ofen-latium/ +- [ ] https://anno.land/en/anno-117-goods/eisen/ +- [ ] https://anno.land/en/anno-117-buildings/waffenschmied-latium/ +- [ ] https://anno.land/en/anno-117-goods/waffen/ +- [ ] https://anno.land/en/anno-117-buildings/pferdezuechter-latium/ +- [ ] https://anno.land/en/anno-117-buildings/ruestungsschmied-latium/ +- [ ] https://anno.land/en/anno-117-goods/ruestung/ +- [ ] https://anno.land/en/anno-117-buildings/vigiles-latium/ +- [ ] https://anno.land/en/anno-117-buildings/custodia-latium/ +- [ ] https://anno.land/en/anno-117-buildings/medici-latium/ +- [ ] https://anno.land/en/anno-117-buildings/brunnen-latium/ +- [ ] https://anno.land/en/anno-117-buildings/wachturm-latium/ +- [ ] https://anno.land/en/anno-117-buildings/latrine-latium/ +- [ ] https://anno.land/en/anno-117-buildings/villa-des-praetors-latium/ +- [ ] https://anno.land/en/anno-117-buildings/officium-latium/ +- [ ] https://anno.land/en/anno-117-buildings/schrein-von-ceres-latium/ +- [ ] https://anno.land/en/anno-117-buildings/schrein-von-epona-latium/ +- [ ] https://anno.land/en/anno-117-buildings/schrein-von-cernunnos-latium/