#!/usr/bin/env python3
"""
Anno 117 Web Scraper

Scrapes pages listed in scraping.md, extracts game data,
discovers new pages, and updates the scraping list.
"""

import argparse
import re
import json
import time
from pathlib import Path
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup

# Project paths
PROJECT_ROOT = Path(__file__).parent.parent
SCRAPING_MD = PROJECT_ROOT / "scraping.md"
PROCESSED_MD = PROJECT_ROOT / "processed.md"
OUTPUT_DIR = PROJECT_ROOT / "scraped_data"


def read_scraping_md() -> tuple[list[str], list[str], str]:
    """
    Read scraping.md and return (unchecked_urls, checked_urls, full_content).
    """
    content = SCRAPING_MD.read_text(encoding="utf-8")

    unchecked = re.findall(r"^- \[ \] (https?://[^\s]+)", content, re.MULTILINE)
    checked = re.findall(r"^- \[x\] (https?://[^\s]+)", content, re.MULTILINE)

    return unchecked, checked, content


def mark_url_as_done(url: str) -> None:
    """Mark a URL as scraped in scraping.md."""
    content = SCRAPING_MD.read_text(encoding="utf-8")

    # Use regex to match the exact URL (followed by newline or end of string)
    # This prevents partial URL matches
    escaped_url = re.escape(url)
    pattern = rf"^- \[ \] {escaped_url}$"
    replacement = f"- [x] {url}"

    content = re.sub(pattern, replacement, content, flags=re.MULTILINE)
    SCRAPING_MD.write_text(content, encoding="utf-8")
    print(f"Marked as done: {url}")


def add_new_urls(new_urls: list[str]) -> None:
    """Add newly discovered URLs to scraping.md if not already present."""
    unchecked, checked, content = read_scraping_md()
    existing = set(unchecked + checked)

    urls_to_add = [url for url in new_urls if url not in existing]

    if urls_to_add:
        # Append new URLs at the end
        additions = "\n".join(f"- [ ] {url}" for url in urls_to_add)
        content = content.rstrip() + "\n" + additions + "\n"
        SCRAPING_MD.write_text(content, encoding="utf-8")
        print(f"Added {len(urls_to_add)} new URLs to scraping.md")
        for url in urls_to_add:
            print(f"  + {url}")


def extract_links(soup: BeautifulSoup, base_url: str) -> list[str]:
    """Extract relevant anno.land links from the page."""
    links = []
    base_domain = urlparse(base_url).netloc

    for a in soup.find_all("a", href=True):
        href = a["href"]
        full_url = urljoin(base_url, href)
        parsed = urlparse(full_url)

        # Only keep links from the same domain and relevant paths
        if parsed.netloc == base_domain:
            # Filter for Anno 117 related content
            if any(keyword in parsed.path for keyword in [
                "anno-117", "goods", "buildings", "specialists",
                "skills", "guides", "tools", "datenbank"
            ]):
                # Clean the URL (remove fragments and query params for deduplication)
                clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                if clean_url.endswith("/"):
                    clean_url = clean_url.rstrip("/") + "/"
                else:
                    clean_url = clean_url + "/" if not parsed.path.endswith((".html", ".php")) else clean_url

                if clean_url not in links:
                    links.append(clean_url)

    return links


def scrape_page(url: str) -> dict:
    """
    Scrape a single page and extract relevant content.
    Returns a dict with the scraped data.
    """
    print(f"Scraping: {url}")

    headers = {
        "User-Agent": "Anno117DocBot/1.0 (One-time scrape for AI agent docs; https://git.blijnder.com/jivanrij/anno-117-docs; Contact: jivanrij@gmail.com)",
        "X-Bot-Purpose": "Creating a markdown file with Anno 117 data for my AI agent (personal use, non-commercial). This way the agent doesn't need to query your website multiple times - each page is only fetched once. Result: https://git.blijnder.com/jivanrij/anno-117-docs/raw/branch/main/docs.md - Thank you for anno.land!",
        "From": "jivanrij@gmail.com",
    }

    response = requests.get(url, headers=headers, timeout=30)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")

    # Extract page title
    title = soup.find("title")
    title_text = title.get_text(strip=True) if title else "Unknown"

    # Extract main content (adjust selectors based on site structure)
    main_content = soup.find("main") or soup.find("article") or soup.find("div", class_="content")

    # Extract all text content
    if main_content:
        text_content = main_content.get_text(separator="\n", strip=True)
    else:
        # Fallback to body
        body = soup.find("body")
        text_content = body.get_text(separator="\n", strip=True) if body else ""

    # Extract tables (common for game data)
    tables = []
    for table in soup.find_all("table"):
        table_data = []
        for row in table.find_all("tr"):
            cells = [cell.get_text(strip=True) for cell in row.find_all(["td", "th"])]
            if cells:
                table_data.append(cells)
        if table_data:
            tables.append(table_data)

    # Extract images (for item/building icons)
    images = []
    for img in soup.find_all("img"):
        src = img.get("src", "")
        alt = img.get("alt", "")
        if src and any(keyword in src.lower() for keyword in ["icon", "item", "building", "good"]):
            images.append({"src": urljoin(url, src), "alt": alt})

    # Extract discovered links
    discovered_links = extract_links(soup, url)

    # Extract all images (not just filtered ones)
    all_images = []
    for img in soup.find_all("img"):
        src = img.get("src", "")
        alt = img.get("alt", "")
        if src:
            all_images.append({"src": urljoin(url, src), "alt": alt})

    return {
        "url": url,
        "title": title_text,
        "text_content": text_content,  # Full content for Claude to process
        "tables": tables,
        "images": images,  # Filtered icons
        "all_images": all_images,  # All images on page
        "discovered_links": discovered_links,
        "full_html_length": len(response.text)
    }


def save_scraped_data(data: dict, url: str) -> Path:
    """Save scraped data to a JSON file."""
    OUTPUT_DIR.mkdir(exist_ok=True)

    # Create filename from URL
    parsed = urlparse(url)
    path_parts = [p for p in parsed.path.split("/") if p]
    filename = "_".join(path_parts) if path_parts else "index"
    filename = re.sub(r"[^\w\-]", "_", filename)

    output_file = OUTPUT_DIR / f"{filename}.json"

    with open(output_file, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)

    print(f"Saved to: {output_file}")
    return output_file


def add_pending_file(json_filename: str) -> None:
    """Add a JSON file to the pending section of processed.md."""
    content = PROCESSED_MD.read_text(encoding="utf-8")

    # Check if file is already listed
    if json_filename in content:
        return

    # Add to pending section
    pending_marker = "## Pending Files"
    if pending_marker in content:
        content = content.replace(
            pending_marker,
            f"{pending_marker}\n- [ ] {json_filename}"
        )
        PROCESSED_MD.write_text(content, encoding="utf-8")
        print(f"Added to processed.md pending: {json_filename}")


def scrape_one(url: str) -> bool:
    """Scrape a single URL. Returns True on success, False on failure."""
    try:
        # Scrape the page
        data = scrape_page(url)

        # Save the data
        output_file = save_scraped_data(data, url)

        # Track in processed.md as pending
        add_pending_file(output_file.name)

        # Print summary
        print(f"\n--- Summary ---")
        print(f"Title: {data['title']}")
        print(f"Content length: {len(data['text_content'])} chars")
        print(f"Tables found: {len(data['tables'])}")
        print(f"Images found: {len(data['images'])}")
        print(f"Links discovered: {len(data['discovered_links'])}")

        # Add discovered links to scraping.md
        if data["discovered_links"]:
            add_new_urls(data["discovered_links"])

        # Mark this URL as done
        mark_url_as_done(url)

        print(f"\nSuccessfully scraped: {url}")
        return True

    except requests.RequestException as e:
        print(f"Error scraping {url}: {e}")
        return False


def main():
    """Main entry point."""
    parser = argparse.ArgumentParser(description="Scrape Anno 117 pages from scraping.md")
    parser.add_argument(
        "-n", "--count",
        type=int,
        default=1,
        help="Number of URLs to scrape (default: 1)"
    )
    args = parser.parse_args()

    unchecked, checked, _ = read_scraping_md()

    if not unchecked:
        print("No unchecked URLs found in scraping.md")
        return

    print(f"Found {len(unchecked)} unchecked URLs")
    print(f"Already scraped: {len(checked)} URLs")
    print(f"Will scrape: {min(args.count, len(unchecked))} URLs\n")

    scraped = 0
    failed = 0

    urls_to_scrape = unchecked[:args.count]
    total = len(urls_to_scrape)

    for i, url in enumerate(urls_to_scrape):
        print(f"[{i + 1}/{total}] ", end="")
        if scrape_one(url):
            scraped += 1
        else:
            failed += 1
        print()

        # Be nice to the server - wait 3 seconds between requests
        if i < total - 1:
            print("Waiting 3 seconds...")
            time.sleep(3)

    print(f"Done! Scraped: {scraped}, Failed: {failed}")


if __name__ == "__main__":
    main()