boc/projects/landvex/datafabrik/koer/pipeline.py

#!/usr/bin/env python3
"""
Landvex Datafabrik — Pipeline-orkestrering
Tre köer: schemalagd / kunddriven / fält
"""
import asyncio
import json
import hashlib
from datetime import datetime
from pathlib import Path
from typing import List, Dict, Optional
from enum import Enum

class KoTyp(Enum):
    SCHEMALAGD = "schemalagd"      # Periodisk skördning
    KUNDDRIVEN = "kunddriven"      # Feedback → bounty → research
    FALT = "falt"                   # Zoomer-foton → verifiering

class PipelineKo:
    def __init__(self, ko_dir: Path, ko_typ: KoTyp):
        self.ko_dir = ko_dir / ko_typ.value
        self.ko_typ = ko_typ
        self.ko_dir.mkdir(parents=True, exist_ok=True)
        
        # Underkataloger
        (self.ko_dir / "pending").mkdir(exist_ok=True)
        (self.ko_dir / "processing").mkdir(exist_ok=True)
        (self.ko_dir / "completed").mkdir(exist_ok=True)
        (self.ko_dir / "failed").mkdir(exist_ok=True)
    
    def lagg_till(self, uppgift: dict) -> str:
        """Lägg till uppgift i kön."""
        uppgift_id = hashlib.sha256(
            json.dumps(uppgift, sort_keys=True).encode()
        ).hexdigest()[:16]
        
        uppgift["uppgift_id"] = uppgift_id
        uppgift["skapad"] = datetime.utcnow().isoformat()
        uppgift["status"] = "pending"
        
        fil_path = self.ko_dir / "pending" / f"{uppgift_id}.json"
        with open(fil_path, 'w', encoding='utf-8') as f:
            json.dump(uppgift, f, ensure_ascii=False, indent=2)
        
        return uppgift_id
    
    def hamta_nasta(self) -> Optional[dict]:
        """Hämta nästa uppgift från kön."""
        pending = sorted(self.ko_dir / "pending" .glob("*.json"))
        if not pending:
            return None
        
        fil_path = pending[0]
        with open(fil_path, 'r', encoding='utf-8') as f:
            uppgift = json.load(f)
        
        # Flytta till processing
        ny_path = self.ko_dir / "processing" / fil_path.name
        fil_path.rename(ny_path)
        uppgift["status"] = "processing"
        uppgift["startad"] = datetime.utcnow().isoformat()
        
        with open(ny_path, 'w', encoding='utf-8') as f:
            json.dump(uppgift, f, ensure_ascii=False, indent=2)
        
        return uppgift
    
    def markera_klar(self, uppgift_id: str, resultat: dict):
        """Markera uppgift som klar."""
        processing_path = self.ko_dir / "processing" / f"{uppgift_id}.json"
        if not processing_path.exists():
            return
        
        with open(processing_path, 'r', encoding='utf-8') as f:
            uppgift = json.load(f)
        
        uppgift["status"] = "completed"
        uppgift["avslutad"] = datetime.utcnow().isoformat()
        uppgift["resultat"] = resultat
        
        klar_path = self.ko_dir / "completed" / f"{uppgift_id}.json"
        processing_path.rename(klar_path)
        
        with open(klar_path, 'w', encoding='utf-8') as f:
            json.dump(uppgift, f, ensure_ascii=False, indent=2)
    
    def markera_misslyckad(self, uppgift_id: str, fel: str):
        """Markera uppgift som misslyckad."""
        processing_path = self.ko_dir / "processing" / f"{uppgift_id}.json"
        if not processing_path.exists():
            return
        
        with open(processing_path, 'r', encoding='utf-8') as f:
            uppgift = json.load(f)
        
        uppgift["status"] = "failed"
        uppgift["avslutad"] = datetime.utcnow().isoformat()
        uppgift["fel"] = fel
        
        fail_path = self.ko_dir / "failed" / f"{uppgift_id}.json"
        processing_path.rename(fail_path)
        
        with open(fail_path, 'w', encoding='utf-8') as f:
            json.dump(uppgift, f, ensure_ascii=False, indent=2)
    
    def statistik(self) -> dict:
        """Hämta kö-statistik."""
        return {
            "typ": self.ko_typ.value,
            "pending": len(list((self.ko_dir / "pending").glob("*.json"))),
            "processing": len(list((self.ko_dir / "processing").glob("*.json"))),
            "completed": len(list((self.ko_dir / "completed").glob("*.json"))),
            "failed": len(list((self.ko_dir / "failed").glob("*.json"))),
        }

class DatafabrikPipeline:
    def __init__(self, base_dir: Path):
        self.base_dir = base_dir
        self.koer = {
            KoTyp.SCHEMALAGD: PipelineKo(base_dir, KoTyp.SCHEMALAGD),
            KoTyp.KUNDDRIVEN: PipelineKo(base_dir, KoTyp.KUNDDRIVEN),
            KoTyp.FALT: PipelineKo(base_dir, KoTyp.FALT),
        }
    
    def lagg_till_skordning(self, doman: str, kallor: List[str], prioritet: int = 5):
        """Schemalägg en skördning."""
        return self.koer[KoTyp.SCHEMALAGD].lagg_till({
            "typ": "skordning",
            "doman": doman,
            "kallor": kallor,
            "prioritet": prioritet,
        })
    
    def lagg_till_bounty(self, lvx_id: str, position: str, beskrivning: str):
        """Kunddriven bounty → research-uppgift."""
        return self.koer[KoTyp.KUNDDRIVEN].lagg_till({
            "typ": "bounty_research",
            "lvx_id": lvx_id,
            "position": position,
            "beskrivning": beskrivning,
        })
    
    def lagg_till_faltverifiering(self, foto_id: str, lvx_id: str, zoomer_id: str):
        """Zoomer-foto → verifiering."""
        return self.koer[KoTyp.FALT].lagg_till({
            "typ": "faltverifiering",
            "foto_id": foto_id,
            "lvx_id": lvx_id,
            "zoomer_id": zoomer_id,
        })
    
    def statistik(self) -> dict:
        """Hämta statistik för alla köer."""
        return {k.value: v.statistik() for k, v in self.koer.items()}

def main():
    """Demo: skapa pipeline och lägg till uppgifter."""
    pipeline = DatafabrikPipeline(Path("/tmp/landvex-pipeline"))
    
    # Schemalagd skördning
    id1 = pipeline.lagg_till_skordning("TRP", ["https://example.com/vagbelysning"], 1)
    print(f"Schemalagd: {id1}")
    
    # Kunddriven bounty
    id2 = pipeline.lagg_till_bounty("LVX-TRP-0102", "Storgatan 12, Stockholm", "Okänd armaturmodell")
    print(f"Bounty: {id2}")
    
    # Fältverifiering
    id3 = pipeline.lagg_till_faltverifiering("IMG-123", "LVX-TRP-0102", "zoomer-42")
    print(f"Fält: {id3}")
    
    print("\nStatistik:")
    print(json.dumps(pipeline.statistik(), indent=2))

if __name__ == "__main__":
    main()