From 754c89506b71a714c0ef7c325453c42625efb2dc Mon Sep 17 00:00:00 2001 From: Bernt Date: Thu, 2 Jul 2026 16:03:39 +0000 Subject: [PATCH] =?UTF-8?q?ADR-011:=20Four-Layer=20Data=20Architecture=20?= =?UTF-8?q?=E2=80=94=20Raw=20Archive=20=E2=86=92=20Knowledge=20=E2=86=92?= =?UTF-8?q?=20Ontology=20=E2=86=92=20Decision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Layer 1: ArchiveArtifact — immutable original with retention policy - Layer 2: KnowledgeArtifact — extracted knowledge (observations, segmentations, feature vectors, relations) - Layer 3: Knowledge Graph / Ontology (documented, not implemented) - Layer 4: Decision Intelligence (existing DecisionCase) - DataLifecycle: tracks every step with artifact lineage - Key principle: AI models trained on curated datasets, not whole archive - Ontology answers 'what does it mean in our domain?' Long-term goal: Every observation converted once to structured knowledge, reused infinitely for analysis, decisions, training. Next: PR-005A — Minimal Mission Import UI for MVP-0 --- .../ADR-011-Four-Layer-Data-Architecture.md | 67 ++++++++++++++++++ .../domain/src/artifacts/archive-artifact.ts | 68 +++++++++++++++++++ .../domain/src/artifacts/data-lifecycle.ts | 52 ++++++++++++++ .../src/artifacts/knowledge-artifact.ts | 60 ++++++++++++++++ packages/domain/src/index.ts | 21 ++++++ 5 files changed, 268 insertions(+) create mode 100644 docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md create mode 100644 packages/domain/src/artifacts/archive-artifact.ts create mode 100644 packages/domain/src/artifacts/data-lifecycle.ts create mode 100644 packages/domain/src/artifacts/knowledge-artifact.ts diff --git a/docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md b/docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md new file mode 100644 index 000000000..7e603f7cd --- /dev/null +++ b/docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md @@ -0,0 +1,67 @@ +# ADR-011: Four-Layer Data Architecture + +## Status +Accepted + +## Context +We need a clear separation between raw data, extracted knowledge, and decision intelligence to avoid building expensive, tightly-coupled AI systems. + +## Decision +Separate into four layers: + +### Layer 1: Raw Archive (cheap, immutable) +- Original files never changed +- Hash guarantees integrity +- Storage policy: active (30 days) → archive (long-term, cheaper) + +### Layer 2: Knowledge Extraction +- Objects, geometry, GPS, classifications, bounding boxes, embeddings +- Observations and Evidence created +- Structured knowledge used for most queries, not video files + +### Layer 3: Knowledge Graph / Ontology +- Road A12 → has Observation → Crack → belongs to Area → Stockholm +- Answers: "Show all main roads where cracking increased last 12 months" +- Comes from knowledge graph, not by reading video files + +### Layer 4: Decision Intelligence +- Verified Decision Cases +- Curated datasets for AI training +- Business impact tracking + +## Key Principles + +1. **Archive Artifact** = Original file, never modified +2. **Knowledge Artifact** = Extracted knowledge, can be re-generated +3. **Ontology before model** — taxonomy answers "what does it mean in our domain?" +4. **AI models trained on curated datasets**, not whole archive + +## Data Lifecycle + +``` +Upload → Immutable Archive → Metadata Extraction → Knowledge Extraction +→ Ontology Mapping → Decision Pipeline → Learning → Archive Retention +``` + +Each step produces a new Artifact with version history. + +## Long-term Goal + +"Every real observation is converted once to structured knowledge and can then be reused infinitely for analysis, decision support, history, training, and future AI models." + +## Consequences + +### Positive +- Raw data = revisionable archive +- Real product = growing knowledge base + verified Decision Cases +- Can swap AI models without losing originals +- Cheaper storage (archive vs active) + +### Negative +- More complex pipeline +- Need to manage extraction models +- Knowledge graph maintenance + +## Related +- ADR-002: Artifact is common base contract +- ADR-003: Event Sourcing for traceability diff --git a/packages/domain/src/artifacts/archive-artifact.ts b/packages/domain/src/artifacts/archive-artifact.ts new file mode 100644 index 000000000..5c4e7e8ce --- /dev/null +++ b/packages/domain/src/artifacts/archive-artifact.ts @@ -0,0 +1,68 @@ +/** + * Archive Artifact — Immutable Original + * + * Principle: Original files never change. + * Hash guarantees integrity. + * Storage policy: active → archive (cheaper long-term). + * + * ADR-011: Four-Layer Data Architecture + * Layer 1: Raw Archive (this file) + */ + +import { ArtifactId } from '../common/ids'; +import { Hash, StorageUri } from '../common/value-objects'; + +export interface ArchiveArtifact { + readonly id: ArtifactId; + readonly originalName: string; + readonly mimeType: string; + readonly sizeBytes: number; + readonly hash: Hash; + readonly storageUri: StorageUri; + readonly exif?: { + readonly device?: string; + readonly gpsLat?: number; + readonly gpsLng?: number; + readonly timestamp?: Date; + readonly iso?: number; + readonly exposure?: string; + readonly focalLength?: string; + }; + readonly uploadedAt: Date; + readonly uploadedBy: string; + readonly retentionPolicy: 'active' | 'archive'; + readonly archiveAfterDate?: Date; +} + +export interface CreateArchiveArtifactParams { + readonly id: ArtifactId; + readonly originalName: string; + readonly mimeType: string; + readonly sizeBytes: number; + readonly hash: Hash; + readonly storageUri: StorageUri; + readonly exif?: ArchiveArtifact['exif']; + readonly uploadedBy: string; + readonly retentionPolicy?: 'active' | 'archive'; +} + +export class ArchiveArtifactFactory { + static create(params: CreateArchiveArtifactParams): ArchiveArtifact { + return { + ...params, + retentionPolicy: params.retentionPolicy ?? 'active', + archiveAfterDate: params.retentionPolicy === 'archive' + ? undefined + : new Date(Date.now() + 30 * 24 * 60 * 60 * 1000), // 30 days default + uploadedAt: new Date(), + }; + } + + static archive(artifact: ArchiveArtifact): ArchiveArtifact { + return { + ...artifact, + retentionPolicy: 'archive', + archiveAfterDate: undefined, + }; + } +} diff --git a/packages/domain/src/artifacts/data-lifecycle.ts b/packages/domain/src/artifacts/data-lifecycle.ts new file mode 100644 index 000000000..d49803448 --- /dev/null +++ b/packages/domain/src/artifacts/data-lifecycle.ts @@ -0,0 +1,52 @@ +/** + * Data Lifecycle — From Reality to Decision + * + * ADR-011: Four-Layer Data Architecture + * + * Flow: + * Reality (phone, video, images) + * ↓ + * Raw Archive (immutable) + * ↓ + * Knowledge Extraction + * ↓ + * Knowledge Graph / Ontology + * ↓ + * Decision Intelligence + * + * Each step produces a new Artifact with its own version history. + */ + +import { ArchiveArtifact } from './archive-artifact'; +import { KnowledgeArtifact } from './knowledge-artifact'; + +export interface DataLifecycleStep { + readonly step: 'upload' | 'archive' | 'extract_metadata' | 'extract_knowledge' | 'ontology_map' | 'decision' | 'learn' | 'retain'; + readonly inputArtifactIds: string[]; + readonly outputArtifactIds: string[]; + readonly timestamp: Date; + readonly processor: string; // Model, human, or system +} + +export interface DataLifecycle { + readonly archiveArtifact: ArchiveArtifact; + readonly knowledgeArtifacts: KnowledgeArtifact[]; + readonly steps: DataLifecycleStep[]; +} + +export const DataLifecycleRules = { + // Original never changes + immutableArchive: (archive: ArchiveArtifact): boolean => { + return archive.hash !== undefined && archive.hash.value.length > 0; + }, + + // Knowledge can be re-extracted without touching archive + reextractable: (lifecycle: DataLifecycle): boolean => { + return lifecycle.archiveArtifact !== undefined; + }, + + // Every step produces artifacts + artifactAtEveryStep: (lifecycle: DataLifecycle): boolean => { + return lifecycle.steps.every(step => step.outputArtifactIds.length > 0); + }, +} as const; diff --git a/packages/domain/src/artifacts/knowledge-artifact.ts b/packages/domain/src/artifacts/knowledge-artifact.ts new file mode 100644 index 000000000..280ce138a --- /dev/null +++ b/packages/domain/src/artifacts/knowledge-artifact.ts @@ -0,0 +1,60 @@ +/** + * Knowledge Artifact — Extracted Knowledge + * + * Principle: Structured knowledge is the real product. + * Raw data becomes knowledge once, then reused infinitely. + * + * ADR-011: Four-Layer Data Architecture + * Layer 2: Knowledge Extraction (this file) + */ + +import { ArtifactId, ObservationId } from '../common/ids'; +import { Hash, StorageUri, GeoLocation } from '../common/value-objects'; + +export interface KnowledgeArtifact { + readonly id: ArtifactId; + readonly sourceArchiveId: ArtifactId; // Links to original + readonly type: 'observation' | 'segmentation' | 'classification' | 'feature_vector' | 'relation'; + readonly extractedAt: Date; + readonly extractedBy: string; // Model or human + readonly confidence: number; // 0.0 to 1.0 + readonly data: unknown; // Type-specific data + readonly hash: Hash; + readonly storageUri: StorageUri; + readonly lineage: ArtifactId[]; +} + +// Specific knowledge types +export interface ObservationKnowledge { + readonly observationId: ObservationId; + readonly description: string; + readonly location: GeoLocation; + readonly boundingBox?: { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + }; + readonly classifications: string[]; // Taxonomy references + readonly qualityScore: number; +} + +export interface SegmentationKnowledge { + readonly maskUri: string; + readonly objectClass: string; + readonly pixelCount: number; + readonly areaMeters: number; +} + +export interface FeatureVectorKnowledge { + readonly dimensions: number; + readonly vector: number[]; + readonly modelVersion: string; +} + +export interface RelationKnowledge { + readonly subjectId: string; + readonly predicate: string; + readonly objectId: string; + readonly confidence: number; +} diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 4c1a48f2d..9c3ef12d3 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -44,6 +44,27 @@ export { Severity, PriorityValue, } from './common/value-objects'; + +// Four-Layer Data Architecture +export { + ArchiveArtifact, + CreateArchiveArtifactParams, + ArchiveArtifactFactory, +} from './artifacts/archive-artifact'; + +export { + KnowledgeArtifact, + ObservationKnowledge, + SegmentationKnowledge, + FeatureVectorKnowledge, + RelationKnowledge, +} from './artifacts/knowledge-artifact'; + +export { + DataLifecycle, + DataLifecycleStep, + DataLifecycleRules, +} from './artifacts/data-lifecycle'; export * from './common/enums'; export * from './common/errors';