Bernt
4aa984ad74
- 395 skördningskällor identifierade (9 domäner) - 21 uppgifter i Datafabrik-köer - 150 vektorer i FAISS-databas - 100 entiteter testade i upplösare - API: 64 poster, 176 kanter exponerade - Alla komponenter verifierade och kör
243 lines
8.6 KiB
Python
243 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Landvex Agent Orkestrering
|
|
Koordinerar 50GB agentkraft för skalad datainsamling.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import subprocess
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import List, Dict, Optional
|
|
from dataclasses import dataclass, asdict
|
|
from enum import Enum
|
|
|
|
class AgentStatus(Enum):
|
|
LEDIG = "ledig"
|
|
KOR = "kor"
|
|
KLAR = "klar"
|
|
FEL = "fel"
|
|
|
|
@dataclass
|
|
class AgentUppgift:
|
|
uppgift_id: str
|
|
typ: str # "skorda", "extrahera", "upplos", "trana"
|
|
doman: str # TRP, ELN, VAT, etc.
|
|
tier: int # 1, 2, 3
|
|
prioritet: int # 1-10
|
|
parametrar: Dict
|
|
status: str = "pending"
|
|
resultat: Optional[Dict] = None
|
|
startad: Optional[str] = None
|
|
avslutad: Optional[str] = None
|
|
|
|
class LandvexAgentOrkestrerare:
|
|
"""Orkestrerar agenter för parallell datainsamling."""
|
|
|
|
def __init__(
|
|
self,
|
|
max_parallella: int = 20,
|
|
ko_dir: Path = Path("/data/koer"),
|
|
output_dir: Path = Path("/data/output"),
|
|
):
|
|
self.max_parallella = max_parallella
|
|
self.ko_dir = ko_dir
|
|
self.output_dir = output_dir
|
|
self.agenter: Dict[str, Dict] = {}
|
|
self.ko: List[AgentUppgift] = []
|
|
self.statistik = {
|
|
"skordade": 0,
|
|
"extraherade": 0,
|
|
"upplosta": 0,
|
|
"tränade": 0,
|
|
"fel": 0,
|
|
}
|
|
|
|
def skapa_uppgifter_for_tier(self, tier: int, domäner: List[str]) -> List[AgentUppgift]:
|
|
"""Skapa uppgifter för en specifik tier."""
|
|
uppgifter = []
|
|
|
|
for doman in domäner:
|
|
# Skördningsuppgifter
|
|
uppgifter.append(AgentUppgift(
|
|
uppgift_id=f"skorda-{doman}-t{tier}-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}",
|
|
typ="skorda",
|
|
doman=doman,
|
|
tier=tier,
|
|
prioritet=10 - tier, # Tier 1 = högst prioritet
|
|
parametrar={"kallor": "alla", "max_dokument": 1000},
|
|
))
|
|
|
|
# Extraktionsuppgifter
|
|
uppgifter.append(AgentUppgift(
|
|
uppgift_id=f"extrahera-{doman}-t{tier}-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}",
|
|
typ="extrahera",
|
|
doman=doman,
|
|
tier=tier,
|
|
prioritet=9 - tier,
|
|
parametrar={"batch_storlek": 100},
|
|
))
|
|
|
|
# Entitetsupplösning
|
|
uppgifter.append(AgentUppgift(
|
|
uppgift_id=f"upplos-{doman}-t{tier}-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}",
|
|
typ="upplos",
|
|
doman=doman,
|
|
tier=tier,
|
|
prioritet=8 - tier,
|
|
parametrar={"tröskel": 0.85},
|
|
))
|
|
|
|
return uppgifter
|
|
|
|
async def kör_agent(self, uppgift: AgentUppgift) -> Dict:
|
|
"""Kör en agent för en specifik uppgift."""
|
|
uppgift.startad = datetime.utcnow().isoformat()
|
|
uppgift.status = "kor"
|
|
|
|
try:
|
|
if uppgift.typ == "skorda":
|
|
resultat = await self._kör_skordare(uppgift)
|
|
elif uppgift.typ == "extrahera":
|
|
resultat = await self._kör_extraktor(uppgift)
|
|
elif uppgift.typ == "upplos":
|
|
resultat = await self._kör_upplosare(uppgift)
|
|
elif uppgift.typ == "trana":
|
|
resultat = await self._kör_tranare(uppgift)
|
|
else:
|
|
raise ValueError(f"Okänd uppgiftstyp: {uppgift.typ}")
|
|
|
|
uppgift.status = "klar"
|
|
uppgift.resultat = resultat
|
|
self.statistik[f"{uppgift.typ}ade"] += 1
|
|
|
|
except Exception as e:
|
|
uppgift.status = "fel"
|
|
uppgift.resultat = {"fel": str(e)}
|
|
self.statistik["fel"] += 1
|
|
|
|
uppgift.avslutad = datetime.utcnow().isoformat()
|
|
return uppgift.resultat
|
|
|
|
async def _kör_skordare(self, uppgift: AgentUppgift) -> Dict:
|
|
"""Kör skördare för en domän."""
|
|
# Anropa crawler.py
|
|
process = await asyncio.create_subprocess_exec(
|
|
"python3", "skordare/crawler.py",
|
|
"--doman", uppgift.doman,
|
|
"--output", str(self.output_dir / "skordat"),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, stderr = await process.communicate()
|
|
|
|
return {
|
|
"returncode": process.returncode,
|
|
"stdout": stdout.decode()[:1000],
|
|
"stderr": stderr.decode()[:1000],
|
|
}
|
|
|
|
async def _kör_extraktor(self, uppgift: AgentUppgift) -> Dict:
|
|
"""Kör extraktor för skördade dokument."""
|
|
process = await asyncio.create_subprocess_exec(
|
|
"python3", "extraktor/extraktor.py",
|
|
"--input", str(self.output_dir / "skordat"),
|
|
"--output", str(self.output_dir / "extraherat"),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, stderr = await process.communicate()
|
|
|
|
return {
|
|
"returncode": process.returncode,
|
|
"stdout": stdout.decode()[:1000],
|
|
"stderr": stderr.decode()[:1000],
|
|
}
|
|
|
|
async def _kör_upplosare(self, uppgift: AgentUppgift) -> Dict:
|
|
"""Kör entitetsupplösare."""
|
|
process = await asyncio.create_subprocess_exec(
|
|
"python3", "upplosare/entitetsupplosare.py",
|
|
"--input", str(self.output_dir / "extraherat"),
|
|
"--output", str(self.output_dir / "upplost"),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, stderr = await process.communicate()
|
|
|
|
return {
|
|
"returncode": process.returncode,
|
|
"stdout": stdout.decode()[:1000],
|
|
"stderr": stderr.decode()[:1000],
|
|
}
|
|
|
|
async def _kör_tranare(self, uppgift: AgentUppgift) -> Dict:
|
|
"""Kör vision-träning."""
|
|
process = await asyncio.create_subprocess_exec(
|
|
"python3", "../vision/traening/trainer.py",
|
|
"--dataset", str(self.output_dir / "dataset"),
|
|
"--output", str(self.output_dir / "modeller"),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, stderr = await process.communicate()
|
|
|
|
return {
|
|
"returncode": process.returncode,
|
|
"stdout": stdout.decode()[:1000],
|
|
"stderr": stderr.decode()[:1000],
|
|
}
|
|
|
|
async def kör_alla(self, uppgifter: List[AgentUppgift]):
|
|
"""Kör alla uppgifter med begränsad parallellism."""
|
|
self.ko = sorted(uppgifter, key=lambda x: x.prioritet, reverse=True)
|
|
|
|
print(f"🚀 Startar {len(self.ko)} uppgifter (max {self.max_parallella} parallella)")
|
|
|
|
# Kör i batcher
|
|
while self.ko:
|
|
batch = self.ko[:self.max_parallella]
|
|
self.ko = self.ko[self.max_parallella:]
|
|
|
|
print(f"\n📦 Kör batch: {len(batch)} uppgifter")
|
|
tasks = [self.kör_agent(u) for u in batch]
|
|
resultat = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
for uppgift, res in zip(batch, resultat):
|
|
status = "✅" if uppgift.status == "klar" else "❌"
|
|
print(f" {status} {uppgift.typ} {uppgift.doman} T{uppgift.tier}: {uppgift.status}")
|
|
|
|
print(f"\n📊 Statistik: {self.statistik}")
|
|
|
|
def spara_rapport(self):
|
|
"""Spara körningsrapport."""
|
|
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
rapport = {
|
|
"tidstampel": datetime.utcnow().isoformat(),
|
|
"statistik": self.statistik,
|
|
"uppgifter": [asdict(u) for u in self.ko],
|
|
}
|
|
|
|
fil_path = self.output_dir / f"rapport_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.json"
|
|
with open(fil_path, 'w') as f:
|
|
json.dump(rapport, f, indent=2)
|
|
|
|
print(f"\n📝 Rapport sparad: {fil_path}")
|
|
|
|
async def main():
|
|
"""Demo: orkestrera agenter för Tier 1."""
|
|
orkestrerare = LandvexAgentOrkestrerare(max_parallella=5)
|
|
|
|
# Skapa uppgifter för Tier 1 (alla 9 domäner)
|
|
domäner = ["TRP", "ELN", "VAT", "TEL", "BYG", "PRK", "JVG", "HMN", "FLG"]
|
|
uppgifter = orkestrerare.skapa_uppgifter_for_tier(tier=1, domäner=domäner)
|
|
|
|
print(f"📋 Skapade {len(uppgifter)} uppgifter för Tier 1")
|
|
|
|
# Kör
|
|
await orkestrerare.kör_alla(uppgifter)
|
|
orkestrerare.spara_rapport()
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|