boc/SIL/validation/cross-project-metrics.mjs

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════════════════
// SIL Validation — Tvärsnittsanalys över projekt
// Jämför SIL:s prestanda mellan olika projekt
// ═══════════════════════════════════════════════════════════════════════════

import { readFileSync, existsSync } from 'fs';

const EXTERNAL_RESULTS = '/home/bernt/.openclaw/workspace/SIL/validation/external-results.jsonl';
const INTERNAL_RESULTS = '/home/bernt/.openclaw/workspace/SIL/gold-set.jsonl';

class CrossProjectMetrics {
  constructor() {
    this.external = this.loadLog(EXTERNAL_RESULTS);
    this.internal = this.loadLog(INTERNAL_RESULTS);
  }

  loadLog(path) {
    if (!existsSync(path)) return [];
    return readFileSync(path, 'utf8')
      .split('\n')
      .filter(line => line.trim())
      .map(line => {
        try { return JSON.parse(line); } catch { return null; }
      })
      .filter(Boolean);
  }

  /**
   * Jämför interna vs externa projekt
   */
  compareInternalVsExternal() {
    const internalMetrics = this.calculateMetrics(this.internal, 'internal');
    const externalMetrics = this.calculateMetrics(this.external, 'external');

    return {
      internal: internalMetrics,
      external: externalMetrics,
      comparison: {
        recallDiff: externalMetrics.avgRecall - internalMetrics.avgRecall,
        precisionDiff: externalMetrics.avgPrecision - internalMetrics.avgPrecision,
        // Om externa projekt presterar nästan lika bra → SIL fångar generella mönster
        generalizesWell: Math.abs(externalMetrics.avgRecall - internalMetrics.avgRecall) < 15
      }
    };
  }

  calculateMetrics(entries, type) {
    if (entries.length === 0) {
      return { count: 0, avgRecall: 0, avgPrecision: 0 };
    }

    const recalls = entries
      .map(e => parseFloat(e.recall) || 0)
      .filter(r => !isNaN(r));

    const precisions = entries
      .map(e => parseFloat(e.precision) || 0)
      .filter(p => !isNaN(p));

    return {
      count: entries.length,
      avgRecall: Math.round(recalls.reduce((a, b) => a + b, 0) / recalls.length),
      avgPrecision: Math.round(precisions.reduce((a, b) => a + b, 0) / precisions.length),
      type
    };
  }

  /**
   * Analysera vilka typer av projekt SIL klarar bäst/sämst
   */
  analyzeByProjectType() {
    const byLanguage = {};

    for (const entry of this.external) {
      const lang = entry.language || 'unknown';
      if (!byLanguage[lang]) {
        byLanguage[lang] = { count: 0, recalls: [], precisions: [] };
      }

      byLanguage[lang].count++;
      if (entry.recall !== undefined) byLanguage[lang].recalls.push(entry.recall);
      if (entry.precision !== undefined) byLanguage[lang].precisions.push(entry.precision);
    }

    return Object.entries(byLanguage).map(([lang, data]) => ({
      language: lang,
      count: data.count,
      avgRecall: Math.round(data.recalls.reduce((a, b) => a + b, 0) / data.recalls.length),
      avgPrecision: Math.round(data.precisions.reduce((a, b) => a + b, 0) / data.precisions.length)
    }));
  }

  /**
   * Generera rapport
   */
  generateReport() {
    const comparison = this.compareInternalVsExternal();
    const byType = this.analyzeByProjectType();

    const report = {
      timestamp: new Date().toISOString(),
      comparison,
      byType,
      conclusion: comparison.comparison.generalizesWell
        ? 'SIL generaliserar väl — fångar generella mönster'
        : 'SIL är för specialiserad för internt projekt'
    };

    this.printReport(report);
    return report;
  }

  printReport(report) {
    console.log('╔═══════════════════════════════════════════════════════════════╗');
    console.log('║           CROSS-PROJECT ANALYSIS                              ║');
    console.log('║           Jämför SIL över olika projekt                       ║');
    console.log('╚═══════════════════════════════════════════════════════════════╝');
    console.log();

    console.log('📊 INTERNT VS EXTERNT\n');
    console.log(`   Internt:`);
    console.log(`     PR:er: ${report.comparison.internal.count}`);
    console.log(`     Recall: ${report.comparison.internal.avgRecall}%`);
    console.log(`     Precision: ${report.comparison.internal.avgPrecision}%`);
    console.log();
    console.log(`   Externt:`);
    console.log(`     Projekt: ${report.comparison.external.count}`);
    console.log(`     Recall: ${report.comparison.external.avgRecall}%`);
    console.log(`     Precision: ${report.comparison.external.avgPrecision}%`);
    console.log();
    console.log(`   Skillnad:`);
    console.log(`     Recall: ${report.comparison.comparison.recallDiff > 0 ? '+' : ''}${report.comparison.comparison.recallDiff}%`);
    console.log(`     Precision: ${report.comparison.comparison.precisionDiff > 0 ? '+' : ''}${report.comparison.comparison.precisionDiff}%`);
    console.log();

    console.log('📋 PER SPRÅK\n');
    for (const lang of report.byType) {
      console.log(`   ${lang.language}:`);
      console.log(`     Projekt: ${lang.count}`);
      console.log(`     Recall: ${lang.avgRecall}%`);
      console.log(`     Precision: ${lang.avgPrecision}%`);
    }
    console.log();

    console.log('🎯 SLUTSATS\n');
    console.log(`   ${report.conclusion}`);
    console.log();

    console.log('═══════════════════════════════════════════════════════════════\n');
  }
}

// ── Main ──────────────────────────────────────────────────────────────────

const metrics = new CrossProjectMetrics();
metrics.generateReport();