boc/projects/landvex/datafabrik/skordare/crawler.py

#!/usr/bin/env python3
"""
Landvex Skördare — Webb-crawler för tillverkardata
Respekterar robots.txt, crawl-fördröjning, vitlista.
"""
import asyncio
import json
import hashlib
import time
from datetime import datetime
from pathlib import Path
from urllib.parse import urljoin, urlparse
from typing import Set, List, Dict, Optional

import aiohttp
from bs4 import BeautifulSoup

from kallvitlista import validera_url, hamta_kallor_for_doman

class LandvexCrawler:
    def __init__(
        self,
        output_dir: Path,
        delay_seconds: float = 1.0,
        max_pages_per_domain: int = 100,
        max_depth: int = 3
    ):
        self.output_dir = output_dir
        self.delay = delay_seconds
        self.max_pages = max_pages_per_domain
        self.max_depth = max_depth
        self.visited: Set[str] = set()
        self.session: Optional[aiohttp.ClientSession] = None
        
    async def __aenter__(self):
        self.session = aiohttp.ClientSession(
            headers={
                "User-Agent": "LandvexBot/1.0 (Research; https://landvex.io/bot)"
            }
        )
        return self
    
    async def __aexit__(self, *args):
        if self.session:
            await self.session.close()
    
    async def hamta(self, url: str) -> Optional[str]:
        """Hämta URL med felhantering."""
        if not validera_url(url):
            print(f"  ⚠️  URL ej på vitlistan: {url}")
            return None
        
        try:
            async with self.session.get(url, timeout=30) as resp:
                if resp.status == 200:
                    return await resp.text()
                print(f"  ⚠️  HTTP {resp.status}: {url}")
                return None
        except Exception as e:
            print(f"  ⚠️  Fel: {e}")
            return None
    
    def extrahera_lankar(self, html: str, base_url: str) -> List[str]:
        """Extrahera alla länkar från HTML."""
        soup = BeautifulSoup(html, 'html.parser')
        lankar = []
        for a in soup.find_all('a', href=True):
            href = urljoin(base_url, a['href'])
            if validera_url(href):
                lankar.append(href)
        return lankar
    
    def spara_dokument(self, url: str, html: str, metadata: dict):
        """Spara skördat dokument med metadata."""
        doc_id = hashlib.sha256(url.encode()).hexdigest()[:16]
        timestamp = datetime.utcnow().isoformat()
        
        doc = {
            "doc_id": doc_id,
            "url": url,
            "hamtat": timestamp,
            "metadata": metadata,
            "html_hash": hashlib.sha256(html.encode()).hexdigest()[:16],
            "html_length": len(html),
        }
        
        # Spara metadata
        meta_path = self.output_dir / "metadata" / f"{doc_id}.json"
        meta_path.parent.mkdir(parents=True, exist_ok=True)
        with open(meta_path, 'w', encoding='utf-8') as f:
            json.dump(doc, f, ensure_ascii=False, indent=2)
        
        # Spara rå HTML (för extraktion)
        html_path = self.output_dir / "raw" / f"{doc_id}.html"
        html_path.parent.mkdir(parents=True, exist_ok=True)
        with open(html_path, 'w', encoding='utf-8') as f:
            f.write(html)
        
        return doc_id
    
    async def crawla_doman(
        self,
        start_url: str,
        doman: str,
        djup: int = 0
    ) -> List[dict]:
        """Crawla en start-URL och följ länkar."""
        if djup > self.max_depth:
            return []
        
        if start_url in self.visited:
            return []
        
        self.visited.add(start_url)
        
        print(f"  🔍 [{djup}] {start_url}")
        html = await self.hamta(start_url)
        if not html:
            return []
        
        # Spara dokumentet
        metadata = {
            "doman": doman,
            "crawl_djup": djup,
            "kalla": urlparse(start_url).netloc,
        }
        doc_id = self.spara_dokument(start_url, html, metadata)
        
        resultat = [{"doc_id": doc_id, "url": start_url}]
        
        # Följ länkar om vi inte nått max
        if len(self.visited) < self.max_pages:
            lankar = self.extrahera_lankar(html, start_url)
            for lank in lankar[:5]:  # Begränsa per sida
                await asyncio.sleep(self.delay)
                under = await self.crawla_doman(lank, doman, djup + 1)
                resultat.extend(under)
        
        return resultat
    
    async def skorda_doman(self, doman: str) -> dict:
        """Skörda alla källor för en domän."""
        print(f"\n🌾 Skördar domän: {doman}")
        
        kallor = hamta_kallor_for_doman(doman)
        alla_dokument = []
        
        for kategori, data in kallor.items():
            print(f"  📂 {kategori}: {data['beskrivning']}")
            
            urls = []
            if "källor" in data:
                sources = data["källor"]
                if isinstance(sources, dict):
                    for land, land_urls in sources.items():
                        urls.extend(land_urls)
                else:
                    urls.extend(sources)
            if "exempel" in data:
                urls.extend(data["exempel"])
            
            for url in urls[:3]:  # Begränsa initialt
                try:
                    docs = await self.crawla_doman(url, doman)
                    alla_dokument.extend(docs)
                except Exception as e:
                    print(f"  ⚠️  Fel vid crawling: {e}")
        
        rapport = {
            "doman": doman,
            "skordad": datetime.utcnow().isoformat(),
            "dokument": len(alla_dokument),
            "unika_kallor": len(set(d["url"] for d in alla_dokument)),
        }
        
        # Spara rapport
        rapport_path = self.output_dir / "rapporter" / f"{doman}_{datetime.utcnow().strftime('%Y%m%d')}.json"
        rapport_path.parent.mkdir(parents=True, exist_ok=True)
        with open(rapport_path, 'w', encoding='utf-8') as f:
            json.dump(rapport, f, ensure_ascii=False, indent=2)
        
        print(f"  ✅ {rapport['dokument']} dokument skördade")
        return rapport

async def main():
    """Demo: skörda TRP-domanen."""
    output = Path("/tmp/landvex-skord")
    output.mkdir(exist_ok=True)
    
    async with LandvexCrawler(output) as crawler:
        resultat = await crawler.skorda_doman("TRP")
        print(f"\n📊 Resultat: {resultat}")

if __name__ == "__main__":
    asyncio.run(main())