/** * QUIXZOOM Video Pipeline — Source Reliability System * * Varje datapunkt får metadata: * - Källa * - Datum * - Plats * - Licensstatus * - Kamerakvalitet * - Objektionsgrad (hur mycket som är skymt) * - AI:s konfidens * - Verifieringsstatus * * AI:n lär sig ENDAST av observerbara fenomen, inte av åsikter eller narration. */ const crypto = require('crypto'); /** * Källa-typer och deras tillförlitlighet */ const SOURCE_TYPES = { // QUIXZOOM-egen data (högst tillförlitlighet) 'quixzoom_capture': { baseReliability: 0.95, licenseCertainty: 1.0, metadataCompleteness: 0.95, qualityControl: 'active_guided', description: 'QUIXZOOM guided capture with active learning', }, // Creative Commons-videor 'youtube_cc': { baseReliability: 0.75, licenseCertainty: 0.9, metadataCompleteness: 0.6, qualityControl: 'post_hoc', description: 'YouTube Creative Commons video', }, // Öppna dataset 'open_dataset': { baseReliability: 0.85, licenseCertainty: 0.95, metadataCompleteness: 0.8, qualityControl: 'curated', description: 'Open research dataset (Cityscapes, etc.)', }, // Öppna kartdata 'openstreetmap': { baseReliability: 0.8, licenseCertainty: 0.95, metadataCompleteness: 0.7, qualityControl: 'community', description: 'OpenStreetMap data', }, // Offentliga geodata 'public_geodata': { baseReliability: 0.85, licenseCertainty: 0.9, metadataCompleteness: 0.75, qualityControl: 'government', description: 'Public geodata with appropriate license', }, // Dashcam (med tillstånd) 'dashcam_licensed': { baseReliability: 0.7, licenseCertainty: 0.8, metadataCompleteness: 0.5, qualityControl: 'passive', description: 'Dashcam footage with licensing permission', }, // Walking tours (med tillstånd) 'walking_tour_licensed': { baseReliability: 0.7, licenseCertainty: 0.8, metadataCompleteness: 0.5, qualityControl: 'passive', description: 'Walking tour video with licensing permission', }, }; /** * Kvalitetsdimensioner för varje datapunkt */ const QUALITY_DIMENSIONS = { // Kamerakvalitet camera: { resolution: { weight: 0.3, min: 720, max: 4320 }, // p (720p to 8K) fps: { weight: 0.2, min: 24, max: 60 }, stabilization: { weight: 0.2, boolean: true }, lensQuality: { weight: 0.3, min: 0, max: 1 }, }, // Bildkvalitet image: { sharpness: { weight: 0.25, min: 0, max: 1 }, exposure: { weight: 0.2, min: 0, max: 1 }, colorAccuracy: { weight: 0.15, min: 0, max: 1 }, noise: { weight: 0.2, min: 0, max: 1 }, // lower is better compression: { weight: 0.2, min: 0, max: 1 }, // lower is better }, // Objektionsgrad (hur mycket som är skymt) occlusion: { foreground: { weight: 0.3, min: 0, max: 1 }, // lower is better weather: { weight: 0.25, min: 0, max: 1 }, // lower is better lighting: { weight: 0.25, min: 0, max: 1 }, // lower is better motion: { weight: 0.2, min: 0, max: 1 }, // lower is better }, // AI-konfidens aiConfidence: { objectDetection: { weight: 0.3, min: 0, max: 1 }, sceneClassification: { weight: 0.25, min: 0, max: 1 }, ocr: { weight: 0.2, min: 0, max: 1 }, segmentation: { weight: 0.25, min: 0, max: 1 }, }, }; /** * Beräkna tillförlitlighet för en datapunkt */ function calculateReliability(dataPoint) { const sourceConfig = SOURCE_TYPES[dataPoint.sourceType]; if (!sourceConfig) { console.warn(`[RELIABILITY] Unknown source type: ${dataPoint.sourceType}`); return { overall: 0, breakdown: {} }; } // 1. Bas-tillförlitlighet från källa let reliability = sourceConfig.baseReliability; // 2. Justera för licens-säkerhet reliability *= sourceConfig.licenseCertainty; // 3. Justera för metadata-kompletthet const metadataScore = calculateMetadataScore(dataPoint); reliability *= (0.5 + 0.5 * metadataScore); // 0.5-1.0 range // 4. Justera för kamerakvalitet const cameraScore = calculateCameraScore(dataPoint); reliability *= (0.7 + 0.3 * cameraScore); // 0.7-1.0 range // 5. Justera för bildkvalitet const imageScore = calculateImageScore(dataPoint); reliability *= (0.6 + 0.4 * imageScore); // 0.6-1.0 range // 6. Justera för objektionsgrad const occlusionScore = calculateOcclusionScore(dataPoint); reliability *= (0.5 + 0.5 * occlusionScore); // 0.5-1.0 range // 7. Justera för AI-konfidens const aiScore = calculateAIConfidenceScore(dataPoint); reliability *= (0.6 + 0.4 * aiScore); // 0.6-1.0 range // 8. Justera för verifieringsstatus const verificationMultiplier = getVerificationMultiplier(dataPoint.verificationStatus); reliability *= verificationMultiplier; return { overall: Math.min(1.0, reliability), breakdown: { sourceBase: sourceConfig.baseReliability, license: sourceConfig.licenseCertainty, metadata: metadataScore, camera: cameraScore, image: imageScore, occlusion: occlusionScore, aiConfidence: aiScore, verification: verificationMultiplier, }, sourceType: dataPoint.sourceType, qualityControl: sourceConfig.qualityControl, }; } /** * Beräkna metadata-score */ function calculateMetadataScore(dataPoint) { const required = ['timestamp', 'gps', 'source', 'license']; const optional = ['device', 'camera', 'weather', 'operator']; let score = 0; // Required fields for (const field of required) { if (dataPoint.metadata?.[field]) score += 0.15; } // Optional fields for (const field of optional) { if (dataPoint.metadata?.[field]) score += 0.05; } return Math.min(1.0, score); } /** * Beräkna kamerakvalitet */ function calculateCameraScore(dataPoint) { const camera = dataPoint.camera || {}; let score = 0; // Resolution const resolution = camera.resolution || 1080; score += Math.min(1.0, (resolution - 720) / (4320 - 720)) * 0.3; // FPS const fps = camera.fps || 30; score += Math.min(1.0, (fps - 24) / (60 - 24)) * 0.2; // Stabilization if (camera.stabilization) score += 0.2; // Lens quality score += (camera.lensQuality || 0.5) * 0.3; return score; } /** * Beräkna bildkvalitet */ function calculateImageScore(dataPoint) { const image = dataPoint.imageQuality || {}; let score = 0; score += (image.sharpness || 0.5) * 0.25; score += (image.exposure || 0.5) * 0.2; score += (image.colorAccuracy || 0.5) * 0.15; score += (1 - (image.noise || 0.5)) * 0.2; // lower noise is better score += (1 - (image.compression || 0.5)) * 0.2; // lower compression is better return score; } /** * Beräkna objektionsgrad (hur mycket som är skymt) * Lower is better (0 = inget skymt, 1 = helt skymt) */ function calculateOcclusionScore(dataPoint) { const occlusion = dataPoint.occlusion || {}; let score = 0; score += (1 - (occlusion.foreground || 0)) * 0.3; score += (1 - (occlusion.weather || 0)) * 0.25; score += (1 - (occlusion.lighting || 0)) * 0.25; score += (1 - (occlusion.motion || 0)) * 0.2; return score; } /** * Beräkna AI-konfidens */ function calculateAIConfidenceScore(dataPoint) { const ai = dataPoint.aiAnalysis || {}; let score = 0; score += (ai.objectDetection || 0.5) * 0.3; score += (ai.sceneClassification || 0.5) * 0.25; score += (ai.ocr || 0.5) * 0.2; score += (ai.segmentation || 0.5) * 0.25; return score; } /** * Verifierings-multiplier */ function getVerificationMultiplier(status) { const multipliers = { 'unverified': 0.7, 'ai_verified': 0.85, 'human_verified': 0.95, 'expert_verified': 1.0, 'disputed': 0.5, 'rejected': 0.0, }; return multipliers[status] || 0.7; } /** * Filtrera bort opålitliga datapunkter */ function filterReliableData(dataPoints, minReliability = 0.6) { return dataPoints.filter(dp => { const reliability = calculateReliability(dp); return reliability.overall >= minReliability; }); } /** * Gruppera datapunkter efter tillförlitlighet */ function groupByReliability(dataPoints) { const groups = { high: [], // >= 0.8 medium: [], // 0.6-0.8 low: [], // 0.4-0.6 unreliable: [], // < 0.4 }; for (const dp of dataPoints) { const reliability = calculateReliability(dp); if (reliability.overall >= 0.8) groups.high.push({ ...dp, reliability }); else if (reliability.overall >= 0.6) groups.medium.push({ ...dp, reliability }); else if (reliability.overall >= 0.4) groups.low.push({ ...dp, reliability }); else groups.unreliable.push({ ...dp, reliability }); } return groups; } /** * Generera fullständig metadata för en datapunkt */ function generateDataPointMetadata(options) { return { // Identitet id: options.id || crypto.randomUUID(), timestamp: options.timestamp || new Date().toISOString(), // Källa sourceType: options.sourceType || 'unknown', sourceUrl: options.sourceUrl, sourceChannel: options.sourceChannel, sourceUploader: options.sourceUploader, // Licens license: options.license || 'unknown', licenseUrl: options.licenseUrl, attribution: options.attribution, // Plats gps: { latitude: options.latitude, longitude: options.longitude, accuracy: options.gpsAccuracy, altitude: options.altitude, }, // Kamera camera: { model: options.cameraModel, resolution: options.resolution, fps: options.fps, stabilization: options.stabilization, lensQuality: options.lensQuality, }, // Bildkvalitet (beräknas av AI) imageQuality: { sharpness: options.sharpness, exposure: options.exposure, colorAccuracy: options.colorAccuracy, noise: options.noise, compression: options.compression, }, // Objektionsgrad (beräknas av AI) occlusion: { foreground: options.foregroundOcclusion, weather: options.weatherOcclusion, lighting: options.lightingOcclusion, motion: options.motionOcclusion, }, // AI-analys aiAnalysis: { objectDetection: options.objectDetectionConfidence, sceneClassification: options.sceneClassificationConfidence, ocr: options.ocrConfidence, segmentation: options.segmentationConfidence, }, // Verifiering verificationStatus: options.verificationStatus || 'unverified', verifiedBy: options.verifiedBy, verifiedAt: options.verifiedAt, // Narration-filter narrationFiltered: options.narrationFiltered || false, narrationText: options.narrationText, // Observerbara fenomen (detta är vad AI:n lär sig av) observablePhenomena: options.observablePhenomena || [], }; } /** * Extrahera observerbara fenomen från video (inte åsikter) */ function extractObservablePhenomena(analysis) { const phenomena = []; // Fysiska objekt for (const obj of analysis.objects || []) { phenomena.push({ type: 'physical_object', class: obj.class, confidence: obj.confidence, bbox: obj.bbox, observable: true, }); } // Infrastruktur-tillstånd if (analysis.infrastructure) { phenomena.push({ type: 'infrastructure_condition', category: analysis.infrastructure.category, condition: analysis.infrastructure.condition, observable: true, }); } // Miljöförhållanden if (analysis.environment) { phenomena.push({ type: 'environmental_condition', weather: analysis.environment.weather, lighting: analysis.environment.lighting, observable: true, }); } // Människor och aktivitet if (analysis.people) { phenomena.push({ type: 'human_activity', count: analysis.people.count, density: analysis.people.density, observable: true, }); } // Filtrera bort åsikter och narration return phenomena.filter(p => p.observable); } /** * Huvudfunktion — processa en datapunkt */ function processDataPoint(rawData) { // 1. Generera metadata const metadata = generateDataPointMetadata(rawData); // 2. Extrahera observerbara fenomen const phenomena = extractObservablePhenomena(rawData.analysis || {}); // 3. Beräkna tillförlitlighet const reliability = calculateReliability(metadata); // 4. Filtrera narration const narrationFiltered = filterNarration(rawData.narration); return { metadata, phenomena, reliability, narrationFiltered, usableForTraining: reliability.overall >= 0.6, }; } /** * Filtrera bort narration och åsikter */ function filterNarration(narrationText) { if (!narrationText) return null; // Lista över indikatorer på åsikter/narration const opinionIndicators = [ 'think', 'believe', 'feel', 'opinion', 'should', 'must', 'terrible', 'amazing', 'worst', 'best', 'dangerous', 'safe', ]; const lowerText = narrationText.toLowerCase(); for (const indicator of opinionIndicators) { if (lowerText.includes(indicator)) { return { original: narrationText, filtered: true, reason: `Contains opinion indicator: ${indicator}`, usable: false, }; } } return { original: narrationText, filtered: false, usable: true, }; } module.exports = { calculateReliability, filterReliableData, groupByReliability, generateDataPointMetadata, extractObservablePhenomena, processDataPoint, filterNarration, SOURCE_TYPES, QUALITY_DIMENSIONS, };