377 lines
14 KiB
JavaScript
377 lines
14 KiB
JavaScript
|
#!/usr/bin/env node
|
||
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
||
|
|
// SIL Metrics — Diagnostiska metriker för System Intelligence Layer
|
||
|
|
// Erik-krav: recall, precision, false negatives, false positives,
|
||
|
|
// freshness, confidence calibration
|
||
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
||
|
|
|
||
|
|
import { readFileSync, existsSync } from 'fs';
|
||
|
|
|
||
|
|
const SHADOW_LOG = '/home/bernt/.openclaw/workspace/SIL/shadow-log.jsonl';
|
||
|
|
const GOLD_SET = '/home/bernt/.openclaw/workspace/SIL/gold-set.jsonl';
|
||
|
|
|
||
|
|
class SILMetrics {
|
||
|
|
constructor() {
|
||
|
|
this.entries = this.loadLog(SHADOW_LOG);
|
||
|
|
this.goldEntries = this.loadLog(GOLD_SET);
|
||
|
|
}
|
||
|
|
|
||
|
|
loadLog(path) {
|
||
|
|
if (!existsSync(path)) return [];
|
||
|
|
return readFileSync(path, 'utf8')
|
||
|
|
.split('\n')
|
||
|
|
.filter(line => line.trim())
|
||
|
|
.map(line => {
|
||
|
|
try { return JSON.parse(line); } catch { return null; }
|
||
|
|
})
|
||
|
|
.filter(Boolean);
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── Core Diagnostic Metrics ─────────────────────────────────────────────
|
||
|
|
|
||
|
|
/**
|
||
|
|
* RECALL: Hur många verkligt påverkade komponenter hittade SIL?
|
||
|
|
* recall = true_positives / (true_positives + false_negatives)
|
||
|
|
*/
|
||
|
|
calculateRecall(goldEntry) {
|
||
|
|
if (!goldEntry || !goldEntry.actual) return null;
|
||
|
|
|
||
|
|
const predicted = new Set(goldEntry.predicted || []);
|
||
|
|
const actual = new Set(goldEntry.actual || []);
|
||
|
|
|
||
|
|
let truePositives = 0;
|
||
|
|
let falseNegatives = 0;
|
||
|
|
|
||
|
|
for (const comp of actual) {
|
||
|
|
if (predicted.has(comp)) {
|
||
|
|
truePositives++;
|
||
|
|
} else {
|
||
|
|
falseNegatives++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const total = truePositives + falseNegatives;
|
||
|
|
return {
|
||
|
|
truePositives,
|
||
|
|
falseNegatives,
|
||
|
|
recall: total > 0 ? truePositives / total : 0,
|
||
|
|
// Erik: "false negatives är farligare än false positives"
|
||
|
|
missRate: total > 0 ? falseNegatives / total : 0
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* PRECISION: Hur många av de rapporterade beroendena var relevanta?
|
||
|
|
* precision = true_positives / (true_positives + false_positives)
|
||
|
|
*/
|
||
|
|
calculatePrecision(goldEntry) {
|
||
|
|
if (!goldEntry || !goldEntry.actual) return null;
|
||
|
|
|
||
|
|
const predicted = new Set(goldEntry.predicted || []);
|
||
|
|
const actual = new Set(goldEntry.actual || []);
|
||
|
|
|
||
|
|
let truePositives = 0;
|
||
|
|
let falsePositives = 0;
|
||
|
|
|
||
|
|
for (const comp of predicted) {
|
||
|
|
if (actual.has(comp)) {
|
||
|
|
truePositives++;
|
||
|
|
} else {
|
||
|
|
falsePositives++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const total = truePositives + falsePositives;
|
||
|
|
return {
|
||
|
|
truePositives,
|
||
|
|
falsePositives,
|
||
|
|
precision: total > 0 ? truePositives / total : 0,
|
||
|
|
falseDiscoveryRate: total > 0 ? falsePositives / total : 0
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* F1-SCORE: Harmoniskt medelvärde av precision och recall
|
||
|
|
*/
|
||
|
|
calculateF1(precision, recall) {
|
||
|
|
if (precision + recall === 0) return 0;
|
||
|
|
return 2 * (precision * recall) / (precision + recall);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* FRESHNESS: Hur lång tid efter en ändring uppdaterades grafen?
|
||
|
|
*/
|
||
|
|
calculateFreshness() {
|
||
|
|
const now = Date.now();
|
||
|
|
const freshnessScores = [];
|
||
|
|
|
||
|
|
for (const entry of this.entries) {
|
||
|
|
if (!entry.timestamp) continue;
|
||
|
|
const entryTime = new Date(entry.timestamp).getTime();
|
||
|
|
const age = (now - entryTime) / 1000 / 60; // minuter
|
||
|
|
|
||
|
|
// Exponential decay: 100% vid 0 min, 50% vid 60 min, ~0% vid 4h
|
||
|
|
const freshness = Math.exp(-age / 60);
|
||
|
|
freshnessScores.push({
|
||
|
|
timestamp: entry.timestamp,
|
||
|
|
ageMinutes: Math.round(age),
|
||
|
|
freshness: Math.round(freshness * 100) / 100
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
if (freshnessScores.length === 0) return null;
|
||
|
|
|
||
|
|
const avgFreshness = freshnessScores.reduce((a, b) => a + b.freshness, 0) / freshnessScores.length;
|
||
|
|
const maxAge = Math.max(...freshnessScores.map(f => f.ageMinutes));
|
||
|
|
|
||
|
|
return {
|
||
|
|
averageFreshness: Math.round(avgFreshness * 100) / 100,
|
||
|
|
maxAgeMinutes: maxAge,
|
||
|
|
entries: freshnessScores.slice(-10) // senaste 10
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* CONFIDENCE CALIBRATION: Stämmer confidence-värdena med verklig träffsäkerhet?
|
||
|
|
* E.g. om SIL säger 90% confidence, bör ~90% av förutsägelserna vara korrekta
|
||
|
|
*/
|
||
|
|
calculateCalibration() {
|
||
|
|
const bins = {
|
||
|
|
'0.5-0.6': { predicted: 0, correct: 0 },
|
||
|
|
'0.6-0.7': { predicted: 0, correct: 0 },
|
||
|
|
'0.7-0.8': { predicted: 0, correct: 0 },
|
||
|
|
'0.8-0.9': { predicted: 0, correct: 0 },
|
||
|
|
'0.9-1.0': { predicted: 0, correct: 0 }
|
||
|
|
};
|
||
|
|
|
||
|
|
for (const gold of this.goldEntries) {
|
||
|
|
if (!gold.predictions) continue;
|
||
|
|
|
||
|
|
for (const pred of gold.predictions) {
|
||
|
|
const conf = pred.confidence || 0;
|
||
|
|
const bin = this.confidenceToBin(conf);
|
||
|
|
if (!bin) continue;
|
||
|
|
|
||
|
|
bins[bin].predicted++;
|
||
|
|
if (pred.verified === true) {
|
||
|
|
bins[bin].correct++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const calibration = [];
|
||
|
|
for (const [bin, data] of Object.entries(bins)) {
|
||
|
|
if (data.predicted === 0) continue;
|
||
|
|
|
||
|
|
const actualAccuracy = data.correct / data.predicted;
|
||
|
|
const expectedConfidence = this.binToExpected(bin);
|
||
|
|
const calibrationError = Math.abs(expectedConfidence - actualAccuracy);
|
||
|
|
|
||
|
|
calibration.push({
|
||
|
|
bin,
|
||
|
|
predicted: data.predicted,
|
||
|
|
correct: data.correct,
|
||
|
|
expectedConfidence: Math.round(expectedConfidence * 100) + '%',
|
||
|
|
actualAccuracy: Math.round(actualAccuracy * 100) + '%',
|
||
|
|
calibrationError: Math.round(calibrationError * 100) + '%',
|
||
|
|
wellCalibrated: calibrationError < 0.1
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
const avgError = calibration.length > 0
|
||
|
|
? calibration.reduce((a, c) => a + parseFloat(c.calibrationError), 0) / calibration.length
|
||
|
|
: 0;
|
||
|
|
|
||
|
|
return {
|
||
|
|
bins: calibration,
|
||
|
|
averageCalibrationError: Math.round(avgError * 100) / 100 + '%',
|
||
|
|
wellCalibrated: avgError < 0.1
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
confidenceToBin(confidence) {
|
||
|
|
if (confidence >= 0.9) return '0.9-1.0';
|
||
|
|
if (confidence >= 0.8) return '0.8-0.9';
|
||
|
|
if (confidence >= 0.7) return '0.7-0.8';
|
||
|
|
if (confidence >= 0.6) return '0.6-0.7';
|
||
|
|
if (confidence >= 0.5) return '0.5-0.6';
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
binToExpected(bin) {
|
||
|
|
const map = { '0.5-0.6': 0.55, '0.6-0.7': 0.65, '0.7-0.8': 0.75, '0.8-0.9': 0.85, '0.9-1.0': 0.95 };
|
||
|
|
return map[bin] || 0.5;
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── Aggregate Metrics ───────────────────────────────────────────────────
|
||
|
|
|
||
|
|
calculateAggregate() {
|
||
|
|
if (this.goldEntries.length === 0) {
|
||
|
|
return { error: 'Inga gold set-entries hittades. Kör gold-set.mjs först.' };
|
||
|
|
}
|
||
|
|
|
||
|
|
let totalTP = 0, totalFP = 0, totalFN = 0;
|
||
|
|
const perPREntries = [];
|
||
|
|
|
||
|
|
for (const gold of this.goldEntries) {
|
||
|
|
const recall = this.calculateRecall(gold);
|
||
|
|
const precision = this.calculatePrecision(gold);
|
||
|
|
|
||
|
|
if (recall && precision) {
|
||
|
|
totalTP += recall.truePositives;
|
||
|
|
totalFP += precision.falsePositives;
|
||
|
|
totalFN += recall.falseNegatives;
|
||
|
|
|
||
|
|
perPREntries.push({
|
||
|
|
pr: gold.pr || 'unknown',
|
||
|
|
recall: Math.round(recall.recall * 100) + '%',
|
||
|
|
precision: Math.round(precision.precision * 100) + '%',
|
||
|
|
falseNegatives: recall.falseNegatives,
|
||
|
|
falsePositives: precision.falsePositives,
|
||
|
|
f1: Math.round(this.calculateF1(precision.precision, recall.recall) * 100) + '%'
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const aggregateRecall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 0;
|
||
|
|
const aggregatePrecision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 0;
|
||
|
|
const aggregateF1 = this.calculateF1(aggregatePrecision, aggregateRecall);
|
||
|
|
|
||
|
|
return {
|
||
|
|
totalPRsAnalyzed: this.goldEntries.length,
|
||
|
|
aggregate: {
|
||
|
|
recall: Math.round(aggregateRecall * 100) + '%',
|
||
|
|
precision: Math.round(aggregatePrecision * 100) + '%',
|
||
|
|
f1: Math.round(aggregateF1 * 100) + '%',
|
||
|
|
falseNegativeRate: totalTP + totalFN > 0 ? Math.round(totalFN / (totalTP + totalFN) * 100) + '%' : 'N/A',
|
||
|
|
falsePositiveRate: totalTP + totalFP > 0 ? Math.round(totalFP / (totalTP + totalFP) * 100) + '%' : 'N/A'
|
||
|
|
},
|
||
|
|
perPR: perPREntries.slice(-20), // senaste 20
|
||
|
|
// Erik: "false negatives är farligare än false positives"
|
||
|
|
riskAssessment: {
|
||
|
|
falseNegativeSeverity: totalFN > totalFP ? 'HIGH' : 'LOW',
|
||
|
|
recommendation: totalFN > totalFP
|
||
|
|
? 'Förbättra recall: SIL missar kritiska påverkade komponenter'
|
||
|
|
: 'OK: Fler falska larm än missade komponenter (acceptabelt)'
|
||
|
|
}
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── Main Report ─────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
generateReport() {
|
||
|
|
const aggregate = this.calculateAggregate();
|
||
|
|
const freshness = this.calculateFreshness();
|
||
|
|
const calibration = this.calculateCalibration();
|
||
|
|
|
||
|
|
const report = {
|
||
|
|
timestamp: new Date().toISOString(),
|
||
|
|
period: {
|
||
|
|
totalEntries: this.entries.length,
|
||
|
|
goldSetEntries: this.goldEntries.length,
|
||
|
|
firstEntry: this.entries[0]?.timestamp || null,
|
||
|
|
lastEntry: this.entries[this.entries.length - 1]?.timestamp || null
|
||
|
|
},
|
||
|
|
diagnostics: aggregate,
|
||
|
|
freshness,
|
||
|
|
calibration,
|
||
|
|
// Erik: experiment-fas
|
||
|
|
experimentStatus: {
|
||
|
|
phase: 'EXPERIMENT',
|
||
|
|
criteria: {
|
||
|
|
recallTarget: '> 85%',
|
||
|
|
precisionTarget: '> 80%',
|
||
|
|
falseNegativeRateTarget: '< 10%',
|
||
|
|
calibrationErrorTarget: '< 10%'
|
||
|
|
},
|
||
|
|
readyForActivation: aggregate.aggregate
|
||
|
|
&& parseFloat(aggregate.aggregate.recall) > 85
|
||
|
|
&& parseFloat(aggregate.aggregate.precision) > 80
|
||
|
|
&& parseFloat(aggregate.aggregate.falseNegativeRate) < 10
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
this.printReport(report);
|
||
|
|
return report;
|
||
|
|
}
|
||
|
|
|
||
|
|
printReport(report) {
|
||
|
|
console.log('═══════════════════════════════════════════════════════════════');
|
||
|
|
console.log(' SIL DIAGNOSTIC METRICS');
|
||
|
|
console.log(' (Experiment-fas — ej aktiverad för merge)');
|
||
|
|
console.log('═══════════════════════════════════════════════════════════════\n');
|
||
|
|
|
||
|
|
if (report.diagnostics.error) {
|
||
|
|
console.log(`❌ ${report.diagnostics.error}\n`);
|
||
|
|
console.log(' Kör: node SIL/gold-set.mjs --build\n');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Aggregate
|
||
|
|
console.log('📊 AGGREGERADE METRIKER (alla Gold Set PR:er)\n');
|
||
|
|
const agg = report.diagnostics.aggregate;
|
||
|
|
console.log(` Recall: ${agg.recall} (hur många påverkade komponenter hittade vi?)`);
|
||
|
|
console.log(` Precision: ${agg.precision} (hur många rapporterade var faktiskt relevanta?)`);
|
||
|
|
console.log(` F1: ${agg.f1} (harmoniskt medelvärde)`);
|
||
|
|
console.log(` False Negative Rate: ${agg.falseNegativeRate} (missade kritiska påverkan?)`);
|
||
|
|
console.log(` False Positive Rate: ${agg.falsePositiveRate} (varnade i onödan?)`);
|
||
|
|
console.log();
|
||
|
|
|
||
|
|
// Risk assessment
|
||
|
|
const risk = report.diagnostics.riskAssessment;
|
||
|
|
const riskIcon = risk.falseNegativeSeverity === 'HIGH' ? '🔴' : '🟢';
|
||
|
|
console.log(`${riskIcon} RISKBEDÖMNING`);
|
||
|
|
console.log(` ${risk.recommendation}\n`);
|
||
|
|
|
||
|
|
// Freshness
|
||
|
|
if (report.freshness) {
|
||
|
|
console.log('⏱️ FRESHNESS\n');
|
||
|
|
console.log(` Genomsnittlig: ${report.freshness.averageFreshness}`);
|
||
|
|
console.log(` Max ålder: ${report.freshness.maxAgeMinutes} minuter`);
|
||
|
|
console.log(` Status: ${report.freshness.maxAgeMinutes < 60 ? '✅ OK' : '⚠️ För gammal'}\n`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Calibration
|
||
|
|
if (report.calibration && report.calibration.bins.length > 0) {
|
||
|
|
console.log('🎯 CONFIDENCE CALIBRATION\n');
|
||
|
|
console.log(` Genomsnittligt kalibreringsfel: ${report.calibration.averageCalibrationError}`);
|
||
|
|
console.log(` Väl kalibrerad: ${report.calibration.wellCalibrated ? '✅ Ja' : '⚠️ Nej'}\n`);
|
||
|
|
|
||
|
|
for (const bin of report.calibration.bins) {
|
||
|
|
const icon = bin.wellCalibrated ? '✅' : '⚠️';
|
||
|
|
console.log(` ${icon} ${bin.bin}: förväntat ${bin.expectedConfidence}, faktiskt ${bin.actualAccuracy}`);
|
||
|
|
console.log(` (${bin.correct}/${bin.predicted} korrekta, fel: ${bin.calibrationError})`);
|
||
|
|
}
|
||
|
|
console.log();
|
||
|
|
}
|
||
|
|
|
||
|
|
// Experiment status
|
||
|
|
console.log('🔬 EXPERIMENTSTATUS\n');
|
||
|
|
console.log(` Fas: ${report.experimentStatus.phase}`);
|
||
|
|
console.log(` Kriterier för aktivering:`);
|
||
|
|
console.log(` • Recall > 85%: ${report.experimentStatus.readyForActivation ? '✅' : '❌'}`);
|
||
|
|
console.log(` • Precision > 80%: ${report.experimentStatus.readyForActivation ? '✅' : '❌'}`);
|
||
|
|
console.log(` • False Negative Rate < 10%: ${report.experimentStatus.readyForActivation ? '✅' : '❌'}`);
|
||
|
|
console.log();
|
||
|
|
console.log(` ${report.experimentStatus.readyForActivation ? '✅ KLAR för aktivering vid merge' : '❌ INTE klar — fortsätt samla data'}\n`);
|
||
|
|
|
||
|
|
// Per-PR (senaste)
|
||
|
|
if (report.diagnostics.perPR.length > 0) {
|
||
|
|
console.log('📋 SENASTE PR-ANALYSER (Gold Set)\n');
|
||
|
|
for (const pr of report.diagnostics.perPR.slice(-10)) {
|
||
|
|
console.log(` ${pr.pr}:`);
|
||
|
|
console.log(` recall=${pr.recall} precision=${pr.precision} f1=${pr.f1}`);
|
||
|
|
console.log(` FN=${pr.falseNegatives} FP=${pr.falsePositives}`);
|
||
|
|
}
|
||
|
|
console.log();
|
||
|
|
}
|
||
|
|
|
||
|
|
console.log('═══════════════════════════════════════════════════════════════\n');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── Main ──────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
const metrics = new SILMetrics();
|
||
|
|
metrics.generateReport();
|