Bernt
bae705aa97
- Add NFC ePassport roadmap (ICAO 9303, eIDAS) - Add TensorFlow.js edge face detection (BlazeFace) - Add structured audit logger (GDPR-compliant) - Risk scoring support Part of KYC Apple Native UX v1.1.0
334 lines
11 KiB
JavaScript
334 lines
11 KiB
JavaScript
/**
|
|
* QUIXZOOM Benchmark Suite
|
|
*
|
|
* Automatisk testning av OIE och Change Detection.
|
|
* Kör efter varje ändring och genererar rapport.
|
|
*
|
|
* Exempel på output:
|
|
* Metric Version A Version B
|
|
* Precision 96.1% 97.4%
|
|
* Recall 94.8% 96.2%
|
|
* False merges 41 19
|
|
* Missed merges 112 73
|
|
* Avg latency 34 ms 28 ms
|
|
*/
|
|
|
|
class BenchmarkSuite {
|
|
constructor(options = {}) {
|
|
this.groundTruth = new Map(); // observationId -> expectedObjectId
|
|
this.results = [];
|
|
this.metrics = {
|
|
precision: 0,
|
|
recall: 0,
|
|
falseMerges: 0,
|
|
missedMerges: 0,
|
|
avgLatency: 0,
|
|
throughput: 0,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* LADDA GROUND TRUTH
|
|
* ============================================================
|
|
*/
|
|
|
|
loadGroundTruth(annotations) {
|
|
// annotations: [{ observationId, expectedObjectId, isSameObject }]
|
|
for (const ann of annotations) {
|
|
this.groundTruth.set(ann.observationId, ann);
|
|
}
|
|
console.log(`[BENCH] Loaded ${annotations.length} ground truth annotations`);
|
|
}
|
|
|
|
generateGroundTruthFromSimulator(simulator) {
|
|
// Generera ground truth från simulatorn
|
|
// Objekt som skapats från samma "fysiska" objekt bör matchas
|
|
const annotations = [];
|
|
|
|
for (const [objectId, obj] of simulator.objects) {
|
|
// Hitta alla observationer för detta objekt
|
|
const observations = simulator.observations.filter(o => o.objectId === objectId);
|
|
|
|
for (const obs of observations) {
|
|
annotations.push({
|
|
observationId: obs.id,
|
|
expectedObjectId: objectId,
|
|
isSameObject: true,
|
|
});
|
|
}
|
|
}
|
|
|
|
this.loadGroundTruth(annotations);
|
|
return annotations;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* KÖR BENCHMARK
|
|
* ============================================================
|
|
*/
|
|
|
|
async run(engine, events, options = {}) {
|
|
console.log('[BENCH] Starting benchmark...');
|
|
const startTime = Date.now();
|
|
|
|
const results = {
|
|
version: options.version || 'unknown',
|
|
engine: engine.constructor.name,
|
|
totalEvents: events.length,
|
|
processedEvents: 0,
|
|
correctMatches: 0,
|
|
falseMatches: 0,
|
|
missedMatches: 0,
|
|
newObjects: 0,
|
|
mergedObjects: 0,
|
|
latencies: [],
|
|
errors: [],
|
|
};
|
|
|
|
// Kör varje event
|
|
for (let i = 0; i < events.length; i++) {
|
|
const event = events[i];
|
|
const eventStart = Date.now();
|
|
|
|
try {
|
|
if (event.type === 'observation_created') {
|
|
const result = await engine.process({
|
|
id: event.payload.observationId,
|
|
objectType: event.payload.objectType,
|
|
location: event.payload.location,
|
|
timestamp: event.payload.timestamp,
|
|
attributes: event.payload.attributes,
|
|
});
|
|
|
|
const latency = Date.now() - eventStart;
|
|
results.latencies.push(latency);
|
|
|
|
// Utvärdera mot ground truth
|
|
const groundTruth = this.groundTruth.get(event.payload.observationId);
|
|
if (groundTruth) {
|
|
if (result.action === 'merge' || result.action === 'new') {
|
|
if (result.objectId === groundTruth.expectedObjectId) {
|
|
results.correctMatches++;
|
|
} else {
|
|
results.falseMatches++;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (result.action === 'merge') results.mergedObjects++;
|
|
if (result.action === 'new') results.newObjects++;
|
|
}
|
|
|
|
results.processedEvents++;
|
|
} catch (error) {
|
|
results.errors.push({
|
|
eventId: event.id,
|
|
error: error.message,
|
|
});
|
|
}
|
|
|
|
// Progress
|
|
if (i % 1000 === 0) {
|
|
console.log(`[BENCH] Processed ${i}/${events.length} events...`);
|
|
}
|
|
}
|
|
|
|
const totalTime = Date.now() - startTime;
|
|
|
|
// Beräkna metrics
|
|
const totalObservations = results.correctMatches + results.falseMatches + results.missedMatches;
|
|
results.precision = totalObservations > 0 ? (results.correctMatches / (results.correctMatches + results.falseMatches)) : 0;
|
|
results.recall = totalObservations > 0 ? (results.correctMatches / totalObservations) : 0;
|
|
results.avgLatency = results.latencies.length > 0 ?
|
|
(results.latencies.reduce((a, b) => a + b, 0) / results.latencies.length) : 0;
|
|
results.throughput = totalTime > 0 ? (results.processedEvents / (totalTime / 1000)) : 0;
|
|
results.totalTime = totalTime;
|
|
|
|
this.results.push(results);
|
|
|
|
console.log('[BENCH] Benchmark complete');
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* JÄMFÖR VERSIONER
|
|
* ============================================================
|
|
*/
|
|
|
|
compare(versionA, versionB) {
|
|
const a = this.results.find(r => r.version === versionA);
|
|
const b = this.results.find(r => r.version === versionB);
|
|
|
|
if (!a || !b) {
|
|
throw new Error('Version not found in results');
|
|
}
|
|
|
|
return {
|
|
versionA: a,
|
|
versionB: b,
|
|
comparison: {
|
|
precision: {
|
|
a: (a.precision * 100).toFixed(1) + '%',
|
|
b: (b.precision * 100).toFixed(1) + '%',
|
|
winner: a.precision > b.precision ? 'A' : 'B',
|
|
delta: ((b.precision - a.precision) * 100).toFixed(1) + '%',
|
|
},
|
|
recall: {
|
|
a: (a.recall * 100).toFixed(1) + '%',
|
|
b: (b.recall * 100).toFixed(1) + '%',
|
|
winner: a.recall > b.recall ? 'A' : 'B',
|
|
delta: ((b.recall - a.recall) * 100).toFixed(1) + '%',
|
|
},
|
|
falseMerges: {
|
|
a: a.falseMatches,
|
|
b: b.falseMatches,
|
|
winner: a.falseMatches < b.falseMatches ? 'A' : 'B',
|
|
delta: (b.falseMatches - a.falseMatches),
|
|
},
|
|
missedMerges: {
|
|
a: a.missedMatches || 0,
|
|
b: b.missedMatches || 0,
|
|
winner: (a.missedMatches || 0) < (b.missedMatches || 0) ? 'A' : 'B',
|
|
delta: (b.missedMatches || 0) - (a.missedMatches || 0),
|
|
},
|
|
avgLatency: {
|
|
a: a.avgLatency.toFixed(0) + ' ms',
|
|
b: b.avgLatency.toFixed(0) + ' ms',
|
|
winner: a.avgLatency < b.avgLatency ? 'A' : 'B',
|
|
delta: (b.avgLatency - a.avgLatency).toFixed(0) + ' ms',
|
|
},
|
|
throughput: {
|
|
a: a.throughput.toFixed(0) + ' events/s',
|
|
b: b.throughput.toFixed(0) + ' events/s',
|
|
winner: a.throughput > b.throughput ? 'A' : 'B',
|
|
delta: (b.throughput - a.throughput).toFixed(0) + ' events/s',
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* RAPPORT
|
|
* ============================================================
|
|
*/
|
|
|
|
generateReport() {
|
|
console.log('\n=== BENCHMARK REPORT ===\n');
|
|
|
|
for (const result of this.results) {
|
|
console.log(`Version: ${result.version}`);
|
|
console.log(` Engine: ${result.engine}`);
|
|
console.log(` Events: ${result.processedEvents}/${result.totalEvents}`);
|
|
console.log(` Precision: ${(result.precision * 100).toFixed(1)}%`);
|
|
console.log(` Recall: ${(result.recall * 100).toFixed(1)}%`);
|
|
console.log(` False merges: ${result.falseMatches}`);
|
|
console.log(` New objects: ${result.newObjects}`);
|
|
console.log(` Merged objects: ${result.mergedObjects}`);
|
|
console.log(` Avg latency: ${result.avgLatency.toFixed(0)} ms`);
|
|
console.log(` Throughput: ${result.throughput.toFixed(0)} events/s`);
|
|
console.log(` Total time: ${(result.totalTime / 1000).toFixed(1)} s`);
|
|
console.log(` Errors: ${result.errors.length}`);
|
|
console.log();
|
|
}
|
|
|
|
if (this.results.length >= 2) {
|
|
const comparison = this.compare(
|
|
this.results[0].version,
|
|
this.results[1].version
|
|
);
|
|
|
|
console.log('=== COMPARISON ===\n');
|
|
console.log('Metric Version A Version B Winner Delta');
|
|
console.log('─────────────────────────────────────────────────────────────────────');
|
|
|
|
const metrics = ['precision', 'recall', 'falseMerges', 'missedMerges', 'avgLatency', 'throughput'];
|
|
for (const metric of metrics) {
|
|
const comp = comparison.comparison[metric];
|
|
console.log(
|
|
`${metric.padEnd(18)} ${comp.a.padEnd(16)} ${comp.b.padEnd(16)} ${comp.winner.padEnd(9)} ${comp.delta}`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
exportToFile(path) {
|
|
const fs = require('fs');
|
|
fs.writeFileSync(path, JSON.stringify({
|
|
results: this.results,
|
|
generatedAt: new Date().toISOString(),
|
|
}, null, 2));
|
|
console.log(`[BENCH] Report exported to ${path}`);
|
|
}
|
|
}
|
|
|
|
// Exportera
|
|
module.exports = BenchmarkSuite;
|
|
|
|
// Demo
|
|
if (require.main === module) {
|
|
const ObjectIdentityPipeline = require('../identity-engine/pipeline');
|
|
const CitySimulator = require('../test-data/simulator');
|
|
|
|
async function runBenchmark() {
|
|
console.log('=== BENCHMARK SUITE DEMO ===\n');
|
|
|
|
// 1. Generera testdata
|
|
console.log('Generating test data...');
|
|
const simulator = new CitySimulator('Stockholm', {
|
|
north: 59.4, south: 59.2, east: 18.2, west: 17.9,
|
|
});
|
|
simulator.generate();
|
|
|
|
// 2. Generera ground truth (förenklad: använd subset)
|
|
const benchmark = new BenchmarkSuite();
|
|
const subsetObservations = simulator.observations.slice(0, 1000);
|
|
const annotations = subsetObservations.map(obs => ({
|
|
observationId: obs.id,
|
|
expectedObjectId: obs.objectId,
|
|
isSameObject: true,
|
|
}));
|
|
benchmark.loadGroundTruth(annotations);
|
|
|
|
// 3. Skapa events från observationer
|
|
const events = subsetObservations.map(obs => ({
|
|
id: `evt_${obs.id}`,
|
|
type: 'observation_created',
|
|
timestamp: obs.timestamp.getTime(),
|
|
payload: {
|
|
observationId: obs.id,
|
|
objectType: obs.objectType,
|
|
location: obs.location,
|
|
timestamp: obs.timestamp.getTime(),
|
|
attributes: obs.attributes,
|
|
},
|
|
}));
|
|
|
|
// 4. Kör Version A (default weights)
|
|
console.log('\n--- Running Version A (default weights) ---');
|
|
const engineA = new ObjectIdentityPipeline();
|
|
const resultA = await benchmark.run(engineA, events, { version: 'A' });
|
|
|
|
// 5. Kör Version B (adjusted weights)
|
|
console.log('\n--- Running Version B (spatial-heavy weights) ---');
|
|
const engineB = new ObjectIdentityPipeline({
|
|
spatialWeight: 0.5,
|
|
visualWeight: 0.25,
|
|
contextWeight: 0.15,
|
|
temporalWeight: 0.10,
|
|
});
|
|
const resultB = await benchmark.run(engineB, events, { version: 'B' });
|
|
|
|
// 6. Generera rapport
|
|
benchmark.generateReport();
|
|
|
|
// 7. Exportera
|
|
benchmark.exportToFile('/tmp/benchmark-report.json');
|
|
}
|
|
|
|
runBenchmark().catch(console.error);
|
|
}
|