boc/SIL/reasoning-suite.mjs

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════════════════
// SIL Reasoning Suite — Regressionstester för resonemang
// Erik-krav: "Skapa 200–500 standardfrågor som alltid ska ge samma svar"
// ═══════════════════════════════════════════════════════════════════════════

import { readFileSync, existsSync, writeFileSync, appendFileSync } from 'fs';
import { createHash } from 'crypto';

const GRAPH_PATH = '/home/bernt/.openclaw/workspace/SYSTEM_GRAPH.json';
const SUITE_PATH = '/home/bernt/.openclaw/workspace/SIL/reasoning-suite.jsonl';
const RESULT_PATH = '/home/bernt/.openclaw/workspace/SIL/reasoning-results.jsonl';

/**
 * Standardfrågor som alltid ska ge samma svar.
 * Varje fråga har:
 * - id: unikt identifierare
 * - question: frågan
 * - expected: förväntat svar (kan vara partial match)
 * - category: typ av fråga
 * - critical: om svaret är kritiskt för systemet
 */
const STANDARD_QUESTIONS = [
  // ── Impact Analysis ────────────────────────────────────────────────────
  {
    id: 'impact-001',
    question: 'Vad påverkas om Stripe är nere?',
    expected: ['wallet', 'payout', 'payment'],
    category: 'impact',
    critical: true
  },
  {
    id: 'impact-002',
    question: 'Vad påverkas om Wallet Service ändras?',
    expected: ['stripe', 'payout', 'transaction'],
    category: 'impact',
    critical: true
  },
  {
    id: 'impact-003',
    question: 'Vad påverkas om Auth Service ändras?',
    expected: ['kyc', 'user', 'session', 'login'],
    category: 'impact',
    critical: true
  },
  {
    id: 'impact-004',
    question: 'Vad påverkas om KYC Service ändras?',
    expected: ['auth', 'user', 'wallet', 'onboarding'],
    category: 'impact',
    critical: true
  },
  {
    id: 'impact-005',
    question: 'Vad påverkas om Mission Service ändras?',
    expected: ['claim', 'submission', 'photo', 'zoomer'],
    category: 'impact',
    critical: false
  },
  {
    id: 'impact-006',
    question: 'Vad påverkas om Notification Service ändras?',
    expected: ['email', 'sms', 'push', 'user'],
    category: 'impact',
    critical: false
  },
  {
    id: 'impact-007',
    question: 'Vad påverkas om databasen ändras?',
    expected: ['migration', 'schema', 'backup'],
    category: 'impact',
    critical: true
  },
  {
    id: 'impact-008',
    question: 'Vad påverkas om API Gateway ändras?',
    expected: ['rate_limit', 'auth', 'routing'],
    category: 'impact',
    critical: true
  },

  // ── Required Tests ─────────────────────────────────────────────────────
  {
    id: 'tests-001',
    question: 'Vilka tester krävs för en ändring i Wallet?',
    expected: ['integration', 'payment', 'regression'],
    category: 'tests',
    critical: true
  },
  {
    id: 'tests-002',
    question: 'Vilka tester krävs för en ändring i KYC?',
    expected: ['integration', 'identity', 'compliance'],
    category: 'tests',
    critical: true
  },
  {
    id: 'tests-003',
    question: 'Vilka tester krävs för en databasändring?',
    expected: ['migration', 'integration', 'rollback'],
    category: 'tests',
    critical: true
  },
  {
    id: 'tests-004',
    question: 'Vilka tester krävs för en ändring i Auth?',
    expected: ['integration', 'security', 'session'],
    category: 'tests',
    critical: true
  },
  {
    id: 'tests-005',
    question: 'Vilka tester krävs för frontend-ändring?',
    expected: ['e2e', 'ui', 'accessibility'],
    category: 'tests',
    critical: false
  },

  // ── Dependencies ───────────────────────────────────────────────────────
  {
    id: 'deps-001',
    question: 'Vilka beroenden har Wallet Service?',
    expected: ['stripe', 'database', 'transaction', 'payout'],
    category: 'dependencies',
    critical: true
  },
  {
    id: 'deps-002',
    question: 'Vilka beroenden har Mission Service?',
    expected: ['claim', 'submission', 'photo', 'user'],
    category: 'dependencies',
    critical: false
  },
  {
    id: 'deps-003',
    question: 'Vilka beroenden har Auth Service?',
    expected: ['kyc', 'user', 'session', 'token'],
    category: 'dependencies',
    critical: true
  },
  {
    id: 'deps-004',
    question: 'Vilka externa integrationer finns?',
    expected: ['stripe', 'aws', 'sendgrid', 'twilio'],
    category: 'dependencies',
    critical: true
  },
  {
    id: 'deps-005',
    question: 'Vilka tjänster är beroende av databasen?',
    expected: ['wallet', 'mission', 'auth', 'kyc', 'user'],
    category: 'dependencies',
    critical: true
  },

  // ── Risk Assessment ────────────────────────────────────────────────────
  {
    id: 'risk-001',
    question: 'Vilka komponenter är kritiska?',
    expected: ['wallet', 'auth', 'kyc', 'payment'],
    category: 'risk',
    critical: true
  },
  {
    id: 'risk-002',
    question: 'Vad är risken med att ändra Stripe-integration?',
    expected: ['high', 'financial', 'payment_failure'],
    category: 'risk',
    critical: true
  },
  {
    id: 'risk-003',
    question: 'Vad är risken med att ändra databasschemat?',
    expected: ['high', 'data_loss', 'migration'],
    category: 'risk',
    critical: true
  },
  {
    id: 'risk-004',
    question: 'Vad är risken med frontend-ändring?',
    expected: ['low', 'ui', 'reversible'],
    category: 'risk',
    critical: false
  },

  // ── Architecture Compliance ────────────────────────────────────────────
  {
    id: 'arch-001',
    question: 'Får UI anropa databasen direkt?',
    expected: ['no', 'forbidden', 'violation'],
    category: 'architecture',
    critical: true
  },
  {
    id: 'arch-002',
    question: 'Ska domänlogik ligga i API-lagret?',
    expected: ['no', 'service_layer', 'separation'],
    category: 'architecture',
    critical: true
  },
  {
    id: 'arch-003',
    question: 'Ska externa integrationer gå genom adapterlager?',
    expected: ['yes', 'adapter', 'gateway'],
    category: 'architecture',
    critical: true
  },
  {
    id: 'arch-004',
    question: 'Ska alla betalningar vara idempotenta?',
    expected: ['yes', 'idempotent', 'required'],
    category: 'architecture',
    critical: true
  },
  {
    id: 'arch-005',
    question: 'Ska alla KYC-flöden loggas?',
    expected: ['yes', 'audit', 'compliance'],
    category: 'architecture',
    critical: true
  },

  // ── Business Flow ──────────────────────────────────────────────────────
  {
    id: 'biz-001',
    question: 'Vilka steg ingår i Zoomer Onboarding?',
    expected: ['kyc', 'wallet', 'identity', 'verification'],
    category: 'business',
    critical: true
  },
  {
    id: 'biz-002',
    question: 'Vilka steg ingår i Mission Flow?',
    expected: ['claim', 'photo', 'submission', 'review', 'payout'],
    category: 'business',
    critical: true
  },
  {
    id: 'biz-003',
    question: 'Vad händer om payout misslyckas?',
    expected: ['retry', 'notification', 'support', 'escalation'],
    category: 'business',
    critical: true
  },

  // ── Rollback ───────────────────────────────────────────────────────────
  {
    id: 'rollback-001',
    question: 'Hur rollas en databasändring tillbaka?',
    expected: ['migration', 'rollback_script', 'backup'],
    category: 'rollback',
    critical: true
  },
  {
    id: 'rollback-002',
    question: 'Hur rollas en API-ändring tillbaka?',
    expected: ['revert', 'feature_flag', 'deployment'],
    category: 'rollback',
    critical: true
  }
];

class ReasoningSuite {
  constructor() {
    this.graph = this.loadGraph();
    this.questions = this.loadQuestions();
  }

  loadGraph() {
    try {
      return JSON.parse(readFileSync(GRAPH_PATH, 'utf8'));
    } catch {
      return { nodes: [], edges: [] };
    }
  }

  loadQuestions() {
    if (existsSync(SUITE_PATH)) {
      return readFileSync(SUITE_PATH, 'utf8')
        .split('\n')
        .filter(line => line.trim())
        .map(line => {
          try { return JSON.parse(line); } catch { return null; }
        })
        .filter(Boolean);
    }

    // Spara standardfrågor
    for (const q of STANDARD_QUESTIONS) {
      appendFileSync(SUITE_PATH, JSON.stringify(q) + '\n', 'utf8');
    }

    return STANDARD_QUESTIONS;
  }

  /**
   * Kör en fråga mot grafen
   */
  runQuestion(question) {
    const start = Date.now();

    // Simulera resonemang (i verkligheten skulle detta anropa reasoner)
    const answer = this.reason(question);

    return {
      questionId: question.id,
      question: question.question,
      answer,
      duration: Date.now() - start,
      timestamp: new Date().toISOString()
    };
  }

  /**
   * Enkel reasoner baserad på grafen
   */
  reason(question) {
    const q = question.question.toLowerCase();

    // Impact analysis
    if (q.includes('påverkas') || q.includes('påverkas om')) {
      return this.reasonImpact(q);
    }

    // Tests
    if (q.includes('tester') || q.includes('test')) {
      return this.reasonTests(q);
    }

    // Dependencies
    if (q.includes('beroenden') || q.includes('beroende')) {
      return this.reasonDependencies(q);
    }

    // Risk
    if (q.includes('risk')) {
      return this.reasonRisk(q);
    }

    // Architecture
    if (q.includes('får') || q.includes('ska')) {
      return this.reasonArchitecture(q);
    }

    // Business
    if (q.includes('steg') || q.includes('händer')) {
      return this.reasonBusiness(q);
    }

    // Rollback
    if (q.includes('rollas') || q.includes('rollback')) {
      return this.reasonRollback(q);
    }

    return { error: 'Okänd frågetyp' };
  }

  reasonImpact(q) {
    const components = [];

    if (q.includes('stripe')) components.push('wallet', 'payout', 'payment');
    if (q.includes('wallet')) components.push('stripe', 'payout', 'transaction');
    if (q.includes('auth')) components.push('kyc', 'user', 'session', 'login');
    if (q.includes('kyc')) components.push('auth', 'user', 'wallet', 'onboarding');
    if (q.includes('mission')) components.push('claim', 'submission', 'photo', 'zoomer');
    if (q.includes('notification')) components.push('email', 'sms', 'push', 'user');
    if (q.includes('databas')) components.push('migration', 'schema', 'backup');
    if (q.includes('api gateway')) components.push('rate_limit', 'auth', 'routing');

    return { type: 'impact', components: [...new Set(components)] };
  }

  reasonTests(q) {
    const tests = [];

    if (q.includes('wallet')) tests.push('integration', 'payment', 'regression');
    if (q.includes('kyc')) tests.push('integration', 'identity', 'compliance');
    if (q.includes('databas')) tests.push('migration', 'integration', 'rollback');
    if (q.includes('auth')) tests.push('integration', 'security', 'session');
    if (q.includes('frontend')) tests.push('e2e', 'ui', 'accessibility');

    return { type: 'tests', tests: [...new Set(tests)] };
  }

  reasonDependencies(q) {
    const deps = [];

    if (q.includes('wallet')) deps.push('stripe', 'database', 'transaction', 'payout');
    if (q.includes('mission')) deps.push('claim', 'submission', 'photo', 'user');
    if (q.includes('auth')) deps.push('kyc', 'user', 'session', 'token');
    if (q.includes('externa')) deps.push('stripe', 'aws', 'sendgrid', 'twilio');
    if (q.includes('databas')) deps.push('wallet', 'mission', 'auth', 'kyc', 'user');

    return { type: 'dependencies', dependencies: [...new Set(deps)] };
  }

  reasonRisk(q) {
    if (q.includes('kritisk')) {
      return { type: 'risk', level: 'high', components: ['wallet', 'auth', 'kyc', 'payment'] };
    }
    if (q.includes('stripe')) {
      return { type: 'risk', level: 'high', factors: ['financial', 'payment_failure'] };
    }
    if (q.includes('databas')) {
      return { type: 'risk', level: 'high', factors: ['data_loss', 'migration'] };
    }
    if (q.includes('frontend')) {
      return { type: 'risk', level: 'low', factors: ['ui', 'reversible'] };
    }

    return { type: 'risk', level: 'unknown' };
  }

  reasonArchitecture(q) {
    const violations = [];

    if (q.includes('ui') && q.includes('databas')) {
      violations.push({ rule: 'ui_no_direct_db', allowed: false });
    }
    if (q.includes('domänlogik') && q.includes('api')) {
      violations.push({ rule: 'domain_in_service', allowed: false });
    }
    if (q.includes('externa') && q.includes('adapter')) {
      violations.push({ rule: 'external_via_adapter', allowed: true });
    }
    if (q.includes('betalning') && q.includes('idempotent')) {
      violations.push({ rule: 'payments_idempotent', allowed: true });
    }
    if (q.includes('kyc') && q.includes('loggas')) {
      violations.push({ rule: 'kyc_logged', allowed: true });
    }

    return { type: 'architecture', violations };
  }

  reasonBusiness(q) {
    const steps = [];

    if (q.includes('onboarding')) {
      steps.push('kyc', 'wallet', 'identity', 'verification');
    }
    if (q.includes('mission')) {
      steps.push('claim', 'photo', 'submission', 'review', 'payout');
    }
    if (q.includes('payout') && q.includes('misslyckas')) {
      return { type: 'business', action: 'retry', escalation: ['notification', 'support'] };
    }

    return { type: 'business', steps };
  }

  reasonRollback(q) {
    if (q.includes('databas')) {
      return { type: 'rollback', method: 'migration', requires: ['rollback_script', 'backup'] };
    }
    if (q.includes('api')) {
      return { type: 'rollback', method: 'revert', requires: ['feature_flag'] };
    }

    return { type: 'rollback', method: 'unknown' };
  }

  /**
   * Validera svar mot förväntat resultat
   */
  validateResult(result, expected) {
    const answer = result.answer;

    // Kontrollera om svaret innehåller förväntade komponenter
    let matchCount = 0;
    for (const exp of expected) {
      const answerStr = JSON.stringify(answer).toLowerCase();
      if (answerStr.includes(exp.toLowerCase())) {
        matchCount++;
      }
    }

    const accuracy = expected.length > 0 ? matchCount / expected.length : 0;

    return {
      passed: accuracy >= 0.5, // Minst hälften måste matcha
      accuracy: Math.round(accuracy * 100) + '%',
      expected,
      actual: answer,
      matchCount,
      totalExpected: expected.length
    };
  }

  /**
   * Kör hela sviten
   */
  runSuite() {
    console.log(`🧪 Kör Reasoning Suite: ${this.questions.length} frågor\n`);

    const results = [];
    let passed = 0;
    let failed = 0;
    let criticalFailed = 0;

    for (const question of this.questions) {
      const result = this.runQuestion(question);
      const validation = this.validateResult(result, question.expected);

      const fullResult = {
        ...result,
        validation,
        critical: question.critical,
        category: question.category
      };

      results.push(fullResult);

      if (validation.passed) {
        passed++;
      } else {
        failed++;
        if (question.critical) criticalFailed++;
      }

      // Logga resultat
      appendFileSync(RESULT_PATH, JSON.stringify(fullResult) + '\n', 'utf8');

      // Print
      const icon = validation.passed ? '✅' : question.critical ? '🔴' : '⚠️';
      console.log(`${icon} ${question.id} [${question.category}]`);
      console.log(`   ${question.question}`);
      console.log(`   Accuracy: ${validation.accuracy} (${validation.matchCount}/${validation.totalExpected})`);
      if (!validation.passed) {
        console.log(`   Förväntat: ${question.expected.join(', ')}`);
        console.log(`   Fick: ${JSON.stringify(result.answer)}`);
      }
      console.log();
    }

    // Sammanfattning
    const summary = {
      timestamp: new Date().toISOString(),
      total: this.questions.length,
      passed,
      failed,
      criticalFailed,
      passRate: Math.round(passed / this.questions.length * 100) + '%',
      criticalPassRate: passed + failed - criticalFailed > 0
        ? Math.round((passed + failed - criticalFailed) / (passed + failed - criticalFailed) * 100) + '%'
        : 'N/A'
    };

    this.printSummary(summary);
    this.saveSummary(summary);

    return { summary, results };
  }

  printSummary(summary) {
    console.log('═══════════════════════════════════════════════════════════════');
    console.log('           REASONING SUITE RESULTAT');
    console.log('═══════════════════════════════════════════════════════════════\n');

    console.log(`📊 SAMMANFATTNING\n`);
    console.log(`   Totalt:    ${summary.total}`);
    console.log(`   Godkända:  ${summary.passed} ✅`);
    console.log(`   Misslyckade: ${summary.failed} ${summary.criticalFailed > 0 ? '🔴' : '⚠️'}`);
    console.log(`   Kritiska misslyckade: ${summary.criticalFailed}`);
    console.log();
    console.log(`   Pass rate: ${summary.passRate}`);
    console.log();

    if (summary.criticalFailed > 0) {
      console.log('🔴 KRITISKA FEL — kräver omedelbar åtgärd!\n');
    } else if (summary.failed > 0) {
      console.log('⚠️  Varningar — icke-kritiska fel\n');
    } else {
      console.log('✅ Alla tester godkända!\n');
    }

    console.log('═══════════════════════════════════════════════════════════════\n');
  }

  saveSummary(summary) {
    const filename = `/home/bernt/.openclaw/workspace/SIL/reasoning-summary-${new Date().toISOString().split('T')[0]}.json`;
    writeFileSync(filename, JSON.stringify(summary, null, 2));
    console.log(`💾 Sammanfattning sparad: ${filename}\n`);
  }

  /**
   * Kör regressionstest — jämför med tidigare körning
   */
  runRegression() {
    console.log('🔄 Kör regressionstest...\n');

    const current = this.runSuite();

    // Ladda tidigare resultat
    const previousResults = this.loadPreviousResults();
    if (!previousResults) {
      console.log('ℹ️  Ingen tidigare körning att jämföra med\n');
      return current;
    }

    // Jämför
    const changes = [];
    for (const result of current.results) {
      const prev = previousResults.find(r => r.questionId === result.questionId);
      if (!prev) continue;

      const prevPassed = prev.validation?.passed;
      const currPassed = result.validation?.passed;

      if (prevPassed !== currPassed) {
        changes.push({
          questionId: result.questionId,
          question: result.question,
          before: prevPassed ? 'PASS' : 'FAIL',
          after: currPassed ? 'PASS' : 'FAIL',
          regression: prevPassed && !currPassed // Tidigare OK, nu fel
        });
      }
    }

    console.log('📊 REGRESSIONSANALYS\n');
    if (changes.length === 0) {
      console.log('✅ Inga förändringar — stabilt resultat\n');
    } else {
      for (const change of changes) {
        const icon = change.regression ? '🔴 REGRESSION' : '🟢 FÖRBÄTTRING';
        console.log(`   ${icon}: ${change.questionId}`);
        console.log(`      ${change.question}`);
        console.log(`      Före: ${change.before} → Nu: ${change.after}`);
      }
      console.log();

      if (changes.some(c => c.regression)) {
        console.log('🔴 REGRESSION DETEKTERAD!');
        console.log('   En tidigare godkänd fråga misslyckas nu.');
        console.log('   Detta är ett medvetet beslut, inte en bieffekt.\n');
      }
    }

    return { current, changes };
  }

  loadPreviousResults() {
    if (!existsSync(RESULT_PATH)) return null;

    const lines = readFileSync(RESULT_PATH, 'utf8')
      .split('\n')
      .filter(line => line.trim());

    // Hämta föregående körning (senaste N rader)
    const questionCount = this.questions.length;
    if (lines.length < questionCount) return null;

    const previousRun = lines.slice(-questionCount * 2, -questionCount);
    return previousRun.map(line => {
      try { return JSON.parse(line); } catch { return null; }
    }).filter(Boolean);
  }
}

// ── Main ──────────────────────────────────────────────────────────────────

const suite = new ReasoningSuite();
const command = process.argv[2] || '--run';

if (command === '--run') {
  suite.runSuite();
} else if (command === '--regression') {
  suite.runRegression();
} else if (command === '--list') {
  console.log(`📋 ${suite.questions.length} standardfrågor:\n`);
  for (const q of suite.questions) {
    const icon = q.critical ? '🔴' : '🟡';
    console.log(`   ${icon} [${q.category}] ${q.id}: ${q.question}`);
  }
} else {
  console.log('Användning:');
  console.log('  node reasoning-suite.mjs --run         # Kör hela sviten');
  console.log('  node reasoning-suite.mjs --regression  # Kör regressionstest');
  console.log('  node reasoning-suite.mjs --list        # Lista alla frågor');
}