boc/SIL/metrics.mjs

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════════════════
// SIL Metrics — Diagnostiska metriker för System Intelligence Layer
// Erik-krav: recall, precision, false negatives, false positives,
//            freshness, confidence calibration
// ═══════════════════════════════════════════════════════════════════════════

import { readFileSync, existsSync } from 'fs';

const SHADOW_LOG = '/home/bernt/.openclaw/workspace/SIL/shadow-log.jsonl';
const GOLD_SET = '/home/bernt/.openclaw/workspace/SIL/gold-set.jsonl';

class SILMetrics {
  constructor() {
    this.entries = this.loadLog(SHADOW_LOG);
    this.goldEntries = this.loadLog(GOLD_SET);
  }

  loadLog(path) {
    if (!existsSync(path)) return [];
    return readFileSync(path, 'utf8')
      .split('\n')
      .filter(line => line.trim())
      .map(line => {
        try { return JSON.parse(line); } catch { return null; }
      })
      .filter(Boolean);
  }

  // ── Core Diagnostic Metrics ─────────────────────────────────────────────

  /**
   * RECALL: Hur många verkligt påverkade komponenter hittade SIL?
   * recall = true_positives / (true_positives + false_negatives)
   */
  calculateRecall(goldEntry) {
    if (!goldEntry || !goldEntry.actual) return null;

    const predicted = new Set(goldEntry.predicted || []);
    const actual = new Set(goldEntry.actual || []);

    let truePositives = 0;
    let falseNegatives = 0;

    for (const comp of actual) {
      if (predicted.has(comp)) {
        truePositives++;
      } else {
        falseNegatives++;
      }
    }

    const total = truePositives + falseNegatives;
    return {
      truePositives,
      falseNegatives,
      recall: total > 0 ? truePositives / total : 0,
      // Erik: "false negatives är farligare än false positives"
      missRate: total > 0 ? falseNegatives / total : 0
    };
  }

  /**
   * PRECISION: Hur många av de rapporterade beroendena var relevanta?
   * precision = true_positives / (true_positives + false_positives)
   */
  calculatePrecision(goldEntry) {
    if (!goldEntry || !goldEntry.actual) return null;

    const predicted = new Set(goldEntry.predicted || []);
    const actual = new Set(goldEntry.actual || []);

    let truePositives = 0;
    let falsePositives = 0;

    for (const comp of predicted) {
      if (actual.has(comp)) {
        truePositives++;
      } else {
        falsePositives++;
      }
    }

    const total = truePositives + falsePositives;
    return {
      truePositives,
      falsePositives,
      precision: total > 0 ? truePositives / total : 0,
      falseDiscoveryRate: total > 0 ? falsePositives / total : 0
    };
  }

  /**
   * F1-SCORE: Harmoniskt medelvärde av precision och recall
   */
  calculateF1(precision, recall) {
    if (precision + recall === 0) return 0;
    return 2 * (precision * recall) / (precision + recall);
  }

  /**
   * FRESHNESS: Hur lång tid efter en ändring uppdaterades grafen?
   */
  calculateFreshness() {
    const now = Date.now();
    const freshnessScores = [];

    for (const entry of this.entries) {
      if (!entry.timestamp) continue;
      const entryTime = new Date(entry.timestamp).getTime();
      const age = (now - entryTime) / 1000 / 60; // minuter

      // Exponential decay: 100% vid 0 min, 50% vid 60 min, ~0% vid 4h
      const freshness = Math.exp(-age / 60);
      freshnessScores.push({
        timestamp: entry.timestamp,
        ageMinutes: Math.round(age),
        freshness: Math.round(freshness * 100) / 100
      });
    }

    if (freshnessScores.length === 0) return null;

    const avgFreshness = freshnessScores.reduce((a, b) => a + b.freshness, 0) / freshnessScores.length;
    const maxAge = Math.max(...freshnessScores.map(f => f.ageMinutes));

    return {
      averageFreshness: Math.round(avgFreshness * 100) / 100,
      maxAgeMinutes: maxAge,
      entries: freshnessScores.slice(-10) // senaste 10
    };
  }

  /**
   * CONFIDENCE CALIBRATION: Stämmer confidence-värdena med verklig träffsäkerhet?
   * E.g. om SIL säger 90% confidence, bör ~90% av förutsägelserna vara korrekta
   */
  calculateCalibration() {
    const bins = {
      '0.5-0.6': { predicted: 0, correct: 0 },
      '0.6-0.7': { predicted: 0, correct: 0 },
      '0.7-0.8': { predicted: 0, correct: 0 },
      '0.8-0.9': { predicted: 0, correct: 0 },
      '0.9-1.0': { predicted: 0, correct: 0 }
    };

    for (const gold of this.goldEntries) {
      if (!gold.predictions) continue;

      for (const pred of gold.predictions) {
        const conf = pred.confidence || 0;
        const bin = this.confidenceToBin(conf);
        if (!bin) continue;

        bins[bin].predicted++;
        if (pred.verified === true) {
          bins[bin].correct++;
        }
      }
    }

    const calibration = [];
    for (const [bin, data] of Object.entries(bins)) {
      if (data.predicted === 0) continue;

      const actualAccuracy = data.correct / data.predicted;
      const expectedConfidence = this.binToExpected(bin);
      const calibrationError = Math.abs(expectedConfidence - actualAccuracy);

      calibration.push({
        bin,
        predicted: data.predicted,
        correct: data.correct,
        expectedConfidence: Math.round(expectedConfidence * 100) + '%',
        actualAccuracy: Math.round(actualAccuracy * 100) + '%',
        calibrationError: Math.round(calibrationError * 100) + '%',
        wellCalibrated: calibrationError < 0.1
      });
    }

    const avgError = calibration.length > 0
      ? calibration.reduce((a, c) => a + parseFloat(c.calibrationError), 0) / calibration.length
      : 0;

    return {
      bins: calibration,
      averageCalibrationError: Math.round(avgError * 100) / 100 + '%',
      wellCalibrated: avgError < 0.1
    };
  }

  confidenceToBin(confidence) {
    if (confidence >= 0.9) return '0.9-1.0';
    if (confidence >= 0.8) return '0.8-0.9';
    if (confidence >= 0.7) return '0.7-0.8';
    if (confidence >= 0.6) return '0.6-0.7';
    if (confidence >= 0.5) return '0.5-0.6';
    return null;
  }

  binToExpected(bin) {
    const map = { '0.5-0.6': 0.55, '0.6-0.7': 0.65, '0.7-0.8': 0.75, '0.8-0.9': 0.85, '0.9-1.0': 0.95 };
    return map[bin] || 0.5;
  }

  // ── Aggregate Metrics ───────────────────────────────────────────────────

  calculateAggregate() {
    if (this.goldEntries.length === 0) {
      return { error: 'Inga gold set-entries hittades. Kör gold-set.mjs först.' };
    }

    let totalTP = 0, totalFP = 0, totalFN = 0;
    const perPREntries = [];

    for (const gold of this.goldEntries) {
      const recall = this.calculateRecall(gold);
      const precision = this.calculatePrecision(gold);

      if (recall && precision) {
        totalTP += recall.truePositives;
        totalFP += precision.falsePositives;
        totalFN += recall.falseNegatives;

        perPREntries.push({
          pr: gold.pr || 'unknown',
          recall: Math.round(recall.recall * 100) + '%',
          precision: Math.round(precision.precision * 100) + '%',
          falseNegatives: recall.falseNegatives,
          falsePositives: precision.falsePositives,
          f1: Math.round(this.calculateF1(precision.precision, recall.recall) * 100) + '%'
        });
      }
    }

    const aggregateRecall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 0;
    const aggregatePrecision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 0;
    const aggregateF1 = this.calculateF1(aggregatePrecision, aggregateRecall);

    return {
      totalPRsAnalyzed: this.goldEntries.length,
      aggregate: {
        recall: Math.round(aggregateRecall * 100) + '%',
        precision: Math.round(aggregatePrecision * 100) + '%',
        f1: Math.round(aggregateF1 * 100) + '%',
        falseNegativeRate: totalTP + totalFN > 0 ? Math.round(totalFN / (totalTP + totalFN) * 100) + '%' : 'N/A',
        falsePositiveRate: totalTP + totalFP > 0 ? Math.round(totalFP / (totalTP + totalFP) * 100) + '%' : 'N/A'
      },
      perPR: perPREntries.slice(-20), // senaste 20
      // Erik: "false negatives är farligare än false positives"
      riskAssessment: {
        falseNegativeSeverity: totalFN > totalFP ? 'HIGH' : 'LOW',
        recommendation: totalFN > totalFP
          ? 'Förbättra recall: SIL missar kritiska påverkade komponenter'
          : 'OK: Fler falska larm än missade komponenter (acceptabelt)'
      }
    };
  }

  // ── Main Report ─────────────────────────────────────────────────────────

  generateReport() {
    const aggregate = this.calculateAggregate();
    const freshness = this.calculateFreshness();
    const calibration = this.calculateCalibration();

    const report = {
      timestamp: new Date().toISOString(),
      period: {
        totalEntries: this.entries.length,
        goldSetEntries: this.goldEntries.length,
        firstEntry: this.entries[0]?.timestamp || null,
        lastEntry: this.entries[this.entries.length - 1]?.timestamp || null
      },
      diagnostics: aggregate,
      freshness,
      calibration,
      // Erik: experiment-fas
      experimentStatus: {
        phase: 'EXPERIMENT',
        criteria: {
          recallTarget: '> 85%',
          precisionTarget: '> 80%',
          falseNegativeRateTarget: '< 10%',
          calibrationErrorTarget: '< 10%'
        },
        readyForActivation: aggregate.aggregate
          && parseFloat(aggregate.aggregate.recall) > 85
          && parseFloat(aggregate.aggregate.precision) > 80
          && parseFloat(aggregate.aggregate.falseNegativeRate) < 10
      }
    };

    this.printReport(report);
    return report;
  }

  printReport(report) {
    console.log('═══════════════════════════════════════════════════════════════');
    console.log('           SIL DIAGNOSTIC METRICS');
    console.log('           (Experiment-fas — ej aktiverad för merge)');
    console.log('═══════════════════════════════════════════════════════════════\n');

    if (report.diagnostics.error) {
      console.log(`❌ ${report.diagnostics.error}\n`);
      console.log('   Kör: node SIL/gold-set.mjs --build\n');
      return;
    }

    // Aggregate
    console.log('📊 AGGREGERADE METRIKER (alla Gold Set PR:er)\n');
    const agg = report.diagnostics.aggregate;
    console.log(`  Recall:              ${agg.recall}    (hur många påverkade komponenter hittade vi?)`);
    console.log(`  Precision:           ${agg.precision}    (hur många rapporterade var faktiskt relevanta?)`);
    console.log(`  F1:                  ${agg.f1}    (harmoniskt medelvärde)`);
    console.log(`  False Negative Rate: ${agg.falseNegativeRate}    (missade kritiska påverkan?)`);
    console.log(`  False Positive Rate: ${agg.falsePositiveRate}    (varnade i onödan?)`);
    console.log();

    // Risk assessment
    const risk = report.diagnostics.riskAssessment;
    const riskIcon = risk.falseNegativeSeverity === 'HIGH' ? '🔴' : '🟢';
    console.log(`${riskIcon} RISKBEDÖMNING`);
    console.log(`   ${risk.recommendation}\n`);

    // Freshness
    if (report.freshness) {
      console.log('⏱️  FRESHNESS\n');
      console.log(`   Genomsnittlig: ${report.freshness.averageFreshness}`);
      console.log(`   Max ålder: ${report.freshness.maxAgeMinutes} minuter`);
      console.log(`   Status: ${report.freshness.maxAgeMinutes < 60 ? '✅ OK' : '⚠️  För gammal'}\n`);
    }

    // Calibration
    if (report.calibration && report.calibration.bins.length > 0) {
      console.log('🎯 CONFIDENCE CALIBRATION\n');
      console.log(`   Genomsnittligt kalibreringsfel: ${report.calibration.averageCalibrationError}`);
      console.log(`   Väl kalibrerad: ${report.calibration.wellCalibrated ? '✅ Ja' : '⚠️  Nej'}\n`);

      for (const bin of report.calibration.bins) {
        const icon = bin.wellCalibrated ? '✅' : '⚠️';
        console.log(`   ${icon} ${bin.bin}: förväntat ${bin.expectedConfidence}, faktiskt ${bin.actualAccuracy}`);
        console.log(`      (${bin.correct}/${bin.predicted} korrekta, fel: ${bin.calibrationError})`);
      }
      console.log();
    }

    // Experiment status
    console.log('🔬 EXPERIMENTSTATUS\n');
    console.log(`   Fas: ${report.experimentStatus.phase}`);
    console.log(`   Kriterier för aktivering:`);
    console.log(`     • Recall > 85%: ${report.experimentStatus.readyForActivation ? '✅' : '❌'}`);
    console.log(`     • Precision > 80%: ${report.experimentStatus.readyForActivation ? '✅' : '❌'}`);
    console.log(`     • False Negative Rate < 10%: ${report.experimentStatus.readyForActivation ? '✅' : '❌'}`);
    console.log();
    console.log(`   ${report.experimentStatus.readyForActivation ? '✅ KLAR för aktivering vid merge' : '❌ INTE klar — fortsätt samla data'}\n`);

    // Per-PR (senaste)
    if (report.diagnostics.perPR.length > 0) {
      console.log('📋 SENASTE PR-ANALYSER (Gold Set)\n');
      for (const pr of report.diagnostics.perPR.slice(-10)) {
        console.log(`  ${pr.pr}:`);
        console.log(`    recall=${pr.recall} precision=${pr.precision} f1=${pr.f1}`);
        console.log(`    FN=${pr.falseNegatives} FP=${pr.falsePositives}`);
      }
      console.log();
    }

    console.log('═══════════════════════════════════════════════════════════════\n');
  }
}

// ── Main ──────────────────────────────────────────────────────────────────

const metrics = new SILMetrics();
metrics.generateReport();