Files
boc/iom/intelligence/global_reality_model.py
T
Bernt bae705aa97 ARCHITECTURE: NFC roadmap, edge AI, audit logging
- Add NFC ePassport roadmap (ICAO 9303, eIDAS)
- Add TensorFlow.js edge face detection (BlazeFace)
- Add structured audit logger (GDPR-compliant)
- Risk scoring support

Part of KYC Apple Native UX v1.1.0
2026-06-29 16:24:48 +00:00

769 lines
30 KiB
Python

"""
Global Reality Model
Continuously learning global intelligence platform where every observation,
every action, and every outcome improves models for all similar environments.
The platform IS the product. Apps, websites, APIs, and AI models are just
interfaces to the same shared knowledge model and domain logic.
Architecture Principle:
- Single shared ontology (IOM)
- Single shared semantic understanding (Semantic Graph)
- Single shared AI reasoning (all engines)
- Single shared data structures (all models)
- Single shared architectural foundation
No code is developed in isolation.
Every line of code contributes to the evolution of the entire platform.
"""
from typing import Dict, List, Optional, Tuple, Any
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
import json
import hashlib
class KnowledgeType(str, Enum):
"""Four types of knowledge in the Reality Knowledge Base"""
OBSERVATION = "observation" # How the world looks
RELATIONSHIP = "relationship" # How objects and signals relate
DECISION = "decision" # Which decisions are recommended
OUTCOME = "outcome" # Which actions actually worked
@dataclass
class RealityPattern:
"""A discovered pattern in the global reality model"""
pattern_id: str
pattern_type: str # "correlation", "causation", "trend", "anomaly"
description: str
confidence: float
evidence_count: int
locations: List[str]
time_range: Tuple[str, str]
metrics: Dict[str, float]
related_patterns: List[str] = field(default_factory=list)
def to_dict(self) -> Dict:
return {
"pattern_id": self.pattern_id,
"pattern_type": self.pattern_type,
"description": self.description,
"confidence": round(self.confidence, 2),
"evidence_count": self.evidence_count,
"locations": self.locations,
"time_range": self.time_range,
"metrics": self.metrics,
"related_patterns": self.related_patterns
}
class GlobalRealityModel:
"""
Global Reality Model — The core intelligence layer
Every new observation, every completed action, and every measured outcome
improves the models for ALL similar environments worldwide.
This is NOT just a digital twin of one city.
This is a global knowledge system that continuously learns which
interventions work best under different conditions.
"""
def __init__(self):
# Knowledge bases
self.observation_knowledge: Dict[str, List[Dict]] = {} # location_id -> observations
self.relationship_knowledge: Dict[str, List[Dict]] = {} # pattern_id -> relationships
self.decision_knowledge: Dict[str, List[Dict]] = {} # decision_type -> decisions
self.outcome_knowledge: Dict[str, List[Dict]] = {} # action_type -> outcomes
# Global patterns
self.patterns: Dict[str, RealityPattern] = {}
self.pattern_index: Dict[str, List[str]] = {} # metric -> pattern_ids
# Location fingerprints
self.location_fingerprints: Dict[str, Dict] = {}
# Transfer learning cache
self.transfer_models: Dict[str, Dict] = {} # fingerprint_hash -> model
# Statistics
self.total_observations = 0
self.total_interventions = 0
self.total_outcomes = 0
self.last_update = datetime.utcnow().isoformat()
def ingest_observation(self, observation: Dict) -> Dict:
"""
Ingest a new observation into the global model
Every observation improves understanding for similar locations
"""
location_id = observation.get("location_id", "unknown")
# Store observation knowledge
if location_id not in self.observation_knowledge:
self.observation_knowledge[location_id] = []
self.observation_knowledge[location_id].append(observation)
self.total_observations += 1
# Update location fingerprint
self._update_location_fingerprint(location_id, observation)
# Extract patterns
new_patterns = self._extract_patterns_from_observation(observation)
# Update transfer models
self._update_transfer_models(location_id)
self.last_update = datetime.utcnow().isoformat()
return {
"status": "ingested",
"location_id": location_id,
"new_patterns": len(new_patterns),
"total_observations": self.total_observations
}
def ingest_outcome(self, outcome: Dict) -> Dict:
"""
Ingest an intervention outcome
Every outcome improves recommendations for ALL similar locations
"""
action_type = outcome.get("action_type", "unknown")
location_id = outcome.get("location_id", "unknown")
# Store outcome knowledge
if action_type not in self.outcome_knowledge:
self.outcome_knowledge[action_type] = []
self.outcome_knowledge[action_type].append(outcome)
self.total_outcomes += 1
# Update intervention effectiveness globally
self._update_global_effectiveness(action_type, outcome)
# Update patterns with new evidence
self._strengthen_patterns_with_outcome(outcome)
# Update transfer models
self._update_transfer_models(location_id)
self.last_update = datetime.utcnow().isoformat()
return {
"status": "ingested",
"action_type": action_type,
"global_effectiveness_updated": True,
"total_outcomes": self.total_outcomes
}
def query(self, query_type: str, params: Dict) -> Dict:
"""
Query the global reality model
Examples:
- "Which interventions work best for safety in residential areas?"
- "What is the typical outcome of tree planting in Nordic cities?"
- "Which locations are most similar to Stockholm City Center?"
"""
if query_type == "intervention_effectiveness":
return self._query_intervention_effectiveness(params)
elif query_type == "location_similarity":
return self._query_location_similarity(params)
elif query_type == "pattern_search":
return self._query_patterns(params)
elif query_type == "transfer_learning":
return self._query_transfer_learning(params)
elif query_type == "global_trends":
return self._query_global_trends(params)
else:
return {"error": f"Unknown query type: {query_type}"}
def _query_intervention_effectiveness(self, params: Dict) -> Dict:
"""Query effectiveness of interventions globally"""
action_type = params.get("action_type")
location_type = params.get("location_type")
climate_zone = params.get("climate_zone")
# Filter outcomes
outcomes = self.outcome_knowledge.get(action_type, [])
filtered = []
for outcome in outcomes:
# Apply filters
if location_type:
loc_fp = self.location_fingerprints.get(outcome.get("location_id"), {})
if loc_fp.get("type") != location_type:
continue
filtered.append(outcome)
if not filtered:
return {
"action_type": action_type,
"evidence_count": 0,
"message": "No evidence yet for these conditions"
}
# Calculate statistics
success_count = sum(1 for o in filtered
if o.get("status") in ["success", "partial"])
avg_changes = {}
for outcome in filtered:
for metric, change in outcome.get("actual_change", {}).items():
if metric not in avg_changes:
avg_changes[metric] = []
avg_changes[metric].append(change)
avg_effects = {m: round(sum(v)/len(v), 2) for m, v in avg_changes.items()}
return {
"action_type": action_type,
"evidence_count": len(filtered),
"success_rate": round(success_count / len(filtered) * 100, 1),
"average_effects": avg_effects,
"confidence": min(0.99, len(filtered) / 100),
"applies_to": {
"location_type": location_type,
"climate_zone": climate_zone
}
}
def _query_location_similarity(self, params: Dict) -> Dict:
"""Find locations similar to a reference"""
reference_id = params.get("reference_location_id")
reference_fp = self.location_fingerprints.get(reference_id, {})
if not reference_fp:
return {"error": "Reference location not found"}
similarities = []
for location_id, fingerprint in self.location_fingerprints.items():
if location_id == reference_id:
continue
similarity = self._calculate_fingerprint_similarity(reference_fp, fingerprint)
if similarity > 0.7: # Threshold
similarities.append({
"location_id": location_id,
"similarity": round(similarity, 3),
"shared_patterns": self._find_shared_patterns(reference_id, location_id)
})
# Sort by similarity
similarities.sort(key=lambda x: x["similarity"], reverse=True)
return {
"reference_location": reference_id,
"similar_locations": similarities[:10],
"total_similar": len(similarities)
}
def _query_patterns(self, params: Dict) -> Dict:
"""Search for patterns in the global model"""
pattern_type = params.get("pattern_type")
metric = params.get("metric")
min_confidence = params.get("min_confidence", 0.5)
results = []
for pattern_id, pattern in self.patterns.items():
if pattern_type and pattern.pattern_type != pattern_type:
continue
if metric and metric not in pattern.metrics:
continue
if pattern.confidence < min_confidence:
continue
results.append(pattern.to_dict())
# Sort by confidence
results.sort(key=lambda x: x["confidence"], reverse=True)
return {
"patterns_found": len(results),
"patterns": results[:20]
}
def _query_transfer_learning(self, params: Dict) -> Dict:
"""Query transfer learning predictions"""
source_location = params.get("source_location_id")
target_location = params.get("target_location_id")
action_type = params.get("action_type")
# Get source effectiveness
source_effectiveness = self._query_intervention_effectiveness({
"action_type": action_type,
"location_type": self.location_fingerprints.get(source_location, {}).get("type")
})
# Calculate transfer confidence
similarity = self._calculate_location_similarity(source_location, target_location)
transfer_confidence = similarity * source_effectiveness.get("confidence", 0)
# Adjust predictions based on similarity
adjusted_effects = {}
for metric, effect in source_effectiveness.get("average_effects", {}).items():
adjusted_effects[metric] = round(effect * similarity, 2)
return {
"source_location": source_location,
"target_location": target_location,
"action_type": action_type,
"similarity": round(similarity, 3),
"transfer_confidence": round(transfer_confidence, 3),
"predicted_effects": adjusted_effects,
"source_evidence": source_effectiveness.get("evidence_count", 0)
}
def _query_global_trends(self, params: Dict) -> Dict:
"""Query global trends across all locations"""
metric = params.get("metric")
time_period = params.get("time_period", "1y")
# Aggregate trends across all locations
trends = []
for location_id, observations in self.observation_knowledge.items():
if not observations:
continue
# Filter by metric
metric_obs = [o for o in observations if metric in o.get("metrics", {})]
if len(metric_obs) < 2:
continue
# Calculate trend
values = [o["metrics"][metric] for o in metric_obs]
trend = (values[-1] - values[0]) / max(abs(values[0]), 1) * 100
trends.append({
"location_id": location_id,
"trend_percent": round(trend, 2),
"current_value": values[-1],
"observation_count": len(metric_obs)
})
# Sort by trend magnitude
trends.sort(key=lambda x: abs(x["trend_percent"]), reverse=True)
# Calculate global average
if trends:
avg_trend = sum(t["trend_percent"] for t in trends) / len(trends)
else:
avg_trend = 0
return {
"metric": metric,
"time_period": time_period,
"locations_tracked": len(trends),
"global_average_trend": round(avg_trend, 2),
"trending_up": len([t for t in trends if t["trend_percent"] > 5]),
"trending_down": len([t for t in trends if t["trend_percent"] < -5]),
"top_trends": trends[:10]
}
def _update_location_fingerprint(self, location_id: str, observation: Dict):
"""Update location fingerprint from observation"""
if location_id not in self.location_fingerprints:
self.location_fingerprints[location_id] = {
"location_id": location_id,
"type": observation.get("location_type", "unknown"),
"climate_zone": observation.get("climate_zone", "unknown"),
"metrics_history": {},
"observation_count": 0,
"intervention_count": 0,
"fingerprint_hash": ""
}
fp = self.location_fingerprints[location_id]
fp["observation_count"] += 1
# Update metrics history
for metric, value in observation.get("metrics", {}).items():
if metric not in fp["metrics_history"]:
fp["metrics_history"][metric] = []
fp["metrics_history"][metric].append({
"value": value,
"timestamp": observation.get("timestamp", datetime.utcnow().isoformat())
})
# Update fingerprint hash
fp["fingerprint_hash"] = self._compute_fingerprint_hash(fp)
def _compute_fingerprint_hash(self, fingerprint: Dict) -> str:
"""Compute hash of location fingerprint"""
# Simplified: hash of metric averages
metrics = fingerprint.get("metrics_history", {})
avg_values = {}
for metric, history in metrics.items():
if history:
avg_values[metric] = sum(h["value"] for h in history) / len(history)
hash_input = json.dumps(avg_values, sort_keys=True)
return hashlib.md5(hash_input.encode()).hexdigest()[:10]
def _calculate_fingerprint_similarity(self, fp1: Dict, fp2: Dict) -> float:
"""Calculate similarity between two location fingerprints"""
metrics1 = fp1.get("metrics_history", {})
metrics2 = fp2.get("metrics_history", {})
if not metrics1 or not metrics2:
return 0.0
# Calculate cosine similarity
common_metrics = set(metrics1.keys()) & set(metrics2.keys())
if not common_metrics:
return 0.0
# Get latest values
values1 = []
values2 = []
for metric in common_metrics:
if metrics1[metric] and metrics2[metric]:
values1.append(metrics1[metric][-1]["value"])
values2.append(metrics2[metric][-1]["value"])
if not values1:
return 0.0
# Cosine similarity
dot_product = sum(a * b for a, b in zip(values1, values2))
magnitude1 = sum(a * a for a in values1) ** 0.5
magnitude2 = sum(b * b for b in values2) ** 0.5
if magnitude1 == 0 or magnitude2 == 0:
return 0.0
return dot_product / (magnitude1 * magnitude2)
def _calculate_location_similarity(self, loc1: str, loc2: str) -> float:
"""Calculate similarity between two locations"""
fp1 = self.location_fingerprints.get(loc1, {})
fp2 = self.location_fingerprints.get(loc2, {})
return self._calculate_fingerprint_similarity(fp1, fp2)
def _extract_patterns_from_observation(self, observation: Dict) -> List[RealityPattern]:
"""Extract patterns from a single observation"""
# Simplified: look for correlations in metrics
new_patterns = []
metrics = observation.get("metrics", {})
# Example: if we have both safety and lighting metrics
if "safety_index" in metrics and "lighting_quality" in metrics:
pattern_id = f"PAT-{len(self.patterns)}"
pattern = RealityPattern(
pattern_id=pattern_id,
pattern_type="correlation",
description="Safety correlates with lighting quality",
confidence=0.6,
evidence_count=1,
locations=[observation.get("location_id", "unknown")],
time_range=(
observation.get("timestamp", datetime.utcnow().isoformat()),
observation.get("timestamp", datetime.utcnow().isoformat())
),
metrics={
"safety_index": metrics["safety_index"],
"lighting_quality": metrics["lighting_quality"]
}
)
self.patterns[pattern_id] = pattern
new_patterns.append(pattern)
return new_patterns
def _strengthen_patterns_with_outcome(self, outcome: Dict):
"""Strengthen patterns based on outcome evidence"""
# Find patterns related to this outcome
action_type = outcome.get("action_type")
actual_changes = outcome.get("actual_change", {})
for pattern_id, pattern in self.patterns.items():
# Check if pattern metrics overlap with outcome metrics
overlapping = set(pattern.metrics.keys()) & set(actual_changes.keys())
if overlapping:
# Strengthen pattern
pattern.confidence = min(0.99, pattern.confidence + 0.05)
pattern.evidence_count += 1
# Add location if new
location_id = outcome.get("location_id")
if location_id and location_id not in pattern.locations:
pattern.locations.append(location_id)
def _update_global_effectiveness(self, action_type: str, outcome: Dict):
"""Update global effectiveness for an action type"""
# This would update global models
# For now, just track in outcome knowledge
pass
def _update_transfer_models(self, location_id: str):
"""Update transfer learning models for a location"""
fp = self.location_fingerprints.get(location_id)
if not fp:
return
fingerprint_hash = fp["fingerprint_hash"]
# Build transfer model for this fingerprint type
self.transfer_models[fingerprint_hash] = {
"fingerprint_hash": fingerprint_hash,
"location_count": sum(
1 for f in self.location_fingerprints.values()
if f["fingerprint_hash"] == fingerprint_hash
),
"effective_interventions": self._get_effective_interventions_for_fingerprint(fingerprint_hash),
"last_updated": datetime.utcnow().isoformat()
}
def _get_effective_interventions_for_fingerprint(self, fingerprint_hash: str) -> List[Dict]:
"""Get effective interventions for a fingerprint type"""
# Find all locations with this fingerprint
locations = [
loc_id for loc_id, fp in self.location_fingerprints.items()
if fp["fingerprint_hash"] == fingerprint_hash
]
# Aggregate outcomes for these locations
effectiveness = {}
for action_type, outcomes in self.outcome_knowledge.items():
loc_outcomes = [o for o in outcomes if o.get("location_id") in locations]
if loc_outcomes:
success_count = sum(1 for o in loc_outcomes if o.get("status") in ["success", "partial"])
effectiveness[action_type] = {
"success_rate": round(success_count / len(loc_outcomes) * 100, 1),
"evidence_count": len(loc_outcomes)
}
# Sort by success rate
sorted_interventions = sorted(
effectiveness.items(),
key=lambda x: x[1]["success_rate"],
reverse=True
)
return [
{"action_type": action, "stats": stats}
for action, stats in sorted_interventions[:5]
]
def _find_shared_patterns(self, loc1: str, loc2: str) -> List[str]:
"""Find patterns shared between two locations"""
shared = []
for pattern_id, pattern in self.patterns.items():
if loc1 in pattern.locations and loc2 in pattern.locations:
shared.append(pattern_id)
return shared
def get_global_stats(self) -> Dict:
"""Get global model statistics"""
return {
"total_observations": self.total_observations,
"total_interventions": self.total_interventions,
"total_outcomes": self.total_outcomes,
"locations_tracked": len(self.location_fingerprints),
"patterns_discovered": len(self.patterns),
"transfer_models": len(self.transfer_models),
"knowledge_base": {
"observation_entries": sum(len(v) for v in self.observation_knowledge.values()),
"relationship_entries": sum(len(v) for v in self.relationship_knowledge.values()),
"decision_entries": sum(len(v) for v in self.decision_knowledge.values()),
"outcome_entries": sum(len(v) for v in self.outcome_knowledge.values())
},
"last_update": self.last_update
}
def get_reality_knowledge_base(self) -> Dict:
"""Get the complete Reality Knowledge Base"""
return {
"observation_knowledge": {
"description": "How the world looks",
"locations": len(self.observation_knowledge),
"total_observations": self.total_observations
},
"relationship_knowledge": {
"description": "How objects and signals relate",
"patterns": len(self.patterns)
},
"decision_knowledge": {
"description": "Which decisions are recommended",
"intervention_types": len(self.decision_knowledge)
},
"outcome_knowledge": {
"description": "Which actions actually worked",
"total_outcomes": self.total_outcomes,
"action_types": list(self.outcome_knowledge.keys())
}
}
# Example usage
def example_global_reality_model():
"""Example: Global Reality Model in action"""
print("=== Global Reality Model Demo ===\n")
model = GlobalRealityModel()
# Ingest observations from multiple locations
observations = [
{
"location_id": "stockholm-city",
"location_type": "city_center",
"climate_zone": "nordic",
"metrics": {
"safety_index": 75,
"lighting_quality": 80,
"walkability": 85,
"cleanliness": 90
},
"timestamp": "2026-01-01T00:00:00Z"
},
{
"location_id": "stockholm-city",
"location_type": "city_center",
"climate_zone": "nordic",
"metrics": {
"safety_index": 78,
"lighting_quality": 82,
"walkability": 85,
"cleanliness": 88
},
"timestamp": "2026-06-01T00:00:00Z"
},
{
"location_id": "copenhagen-city",
"location_type": "city_center",
"climate_zone": "nordic",
"metrics": {
"safety_index": 80,
"lighting_quality": 85,
"walkability": 88,
"cleanliness": 92
},
"timestamp": "2026-01-01T00:00:00Z"
},
{
"location_id": "bangkok-sukhumvit",
"location_type": "commercial",
"climate_zone": "tropical",
"metrics": {
"safety_index": 45,
"lighting_quality": 40,
"walkability": 60,
"cleanliness": 50
},
"timestamp": "2026-01-01T00:00:00Z"
}
]
print("=== Ingesting Observations ===")
for obs in observations:
result = model.ingest_observation(obs)
print(f" {obs['location_id']}: {result['status']} (total: {result['total_observations']})")
# Ingest outcomes
outcomes = [
{
"intervention_id": "INT-001",
"location_id": "stockholm-city",
"action_type": "replace_lighting",
"actual_change": {"safety_index": 15, "lighting_quality": 20},
"status": "success"
},
{
"intervention_id": "INT-002",
"location_id": "copenhagen-city",
"action_type": "replace_lighting",
"actual_change": {"safety_index": 18, "lighting_quality": 22},
"status": "success"
},
{
"intervention_id": "INT-003",
"location_id": "bangkok-sukhumvit",
"action_type": "replace_lighting",
"actual_change": {"safety_index": 12, "lighting_quality": 15},
"status": "partial"
}
]
print("\n=== Ingesting Outcomes ===")
for outcome in outcomes:
result = model.ingest_outcome(outcome)
print(f" {outcome['action_type']} at {outcome['location_id']}: {result['status']}")
# Query intervention effectiveness
print("\n=== Query: Intervention Effectiveness ===")
effectiveness = model.query("intervention_effectiveness", {
"action_type": "replace_lighting",
"location_type": "city_center"
})
print(f" Evidence count: {effectiveness['evidence_count']}")
print(f" Success rate: {effectiveness['success_rate']}%")
print(f" Average effects: {effectiveness['average_effects']}")
# Query location similarity
print("\n=== Query: Location Similarity ===")
similarity = model.query("location_similarity", {
"reference_location_id": "stockholm-city"
})
print(f" Reference: {similarity['reference_location']}")
for loc in similarity['similar_locations'][:3]:
print(f" {loc['location_id']}: {loc['similarity']} similarity")
# Query patterns
print("\n=== Query: Patterns ===")
patterns = model.query("pattern_search", {
"pattern_type": "correlation",
"min_confidence": 0.5
})
print(f" Patterns found: {patterns['patterns_found']}")
for pat in patterns['patterns']:
print(f" {pat['description']} (confidence: {pat['confidence']})")
# Query transfer learning
print("\n=== Query: Transfer Learning ===")
transfer = model.query("transfer_learning", {
"source_location_id": "stockholm-city",
"target_location_id": "copenhagen-city",
"action_type": "replace_lighting"
})
print(f" Similarity: {transfer['similarity']}")
print(f" Transfer confidence: {transfer['transfer_confidence']}")
print(f" Predicted effects: {transfer['predicted_effects']}")
# Query global trends
print("\n=== Query: Global Trends ===")
trends = model.query("global_trends", {
"metric": "safety_index",
"time_period": "6m"
})
print(f" Locations tracked: {trends['locations_tracked']}")
print(f" Global average trend: {trends['global_average_trend']}%")
print(f" Trending up: {trends['trending_up']}")
print(f" Trending down: {trends['trending_down']}")
# Global stats
print("\n=== Global Model Stats ===")
stats = model.get_global_stats()
print(f" Total observations: {stats['total_observations']}")
print(f" Total outcomes: {stats['total_outcomes']}")
print(f" Locations tracked: {stats['locations_tracked']}")
print(f" Patterns discovered: {stats['patterns_discovered']}")
print(f" Transfer models: {stats['transfer_models']}")
# Reality Knowledge Base
print("\n=== Reality Knowledge Base ===")
rkb = model.get_reality_knowledge_base()
for kb_type, info in rkb.items():
print(f" {kb_type}: {info['description']}")
for key, value in info.items():
if key != "description":
print(f" {key}: {value}")
return model
if __name__ == '__main__':
example_global_reality_model()