#!/usr/bin/env node // ═══════════════════════════════════════════════════════════════════════════ // SIL Validation — Fråga 4: Kan SIL analysera ett okänt projekt? // Testar på open source-projekt // ═══════════════════════════════════════════════════════════════════════════ import { execSync } from 'child_process'; import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs'; const TEST_DIR = '/home/bernt/.openclaw/workspace/SIL/validation/external-projects'; const RESULTS_LOG = '/home/bernt/.openclaw/workspace/SIL/validation/external-results.jsonl'; /** * Testprojekt att utvärdera */ const TEST_PROJECTS = [ { name: 'express', url: 'https://github.com/expressjs/express.git', language: 'javascript', description: 'Node.js web framework' }, { name: 'fastapi', url: 'https://github.com/tiangolo/fastapi.git', language: 'python', description: 'Python web framework' }, { name: 'gin', url: 'https://github.com/gin-gonic/gin.git', language: 'go', description: 'Go web framework' } ]; class ExternalProjectTester { constructor() { if (!existsSync(TEST_DIR)) { mkdirSync(TEST_DIR, { recursive: true }); } } /** * Kör test på alla externa projekt */ async runAllTests() { console.log('🌍 Kör externa projekttester...\n'); const results = []; for (const project of TEST_PROJECTS) { console.log(`🔍 Testar: ${project.name} (${project.language})`); try { const result = await this.testProject(project); results.push(result); console.log(` ${result.passed ? '✅' : '❌'} Recall: ${result.recall || 'N/A'}`); } catch (e) { results.push({ name: project.name, error: e.message, passed: false }); console.log(` ❌ Fel: ${e.message}`); } } const report = this.generateReport(results); this.saveResults(results); this.printReport(report); return report; } /** * Testa ett enskilt projekt */ async testProject(project) { const projectDir = `${TEST_DIR}/${project.name}`; // 1. Klona om inte finns if (!existsSync(projectDir)) { console.log(` 📥 Klonar ${project.name}...`); execSync(`git clone --depth 100 ${project.url} ${projectDir}`, { stdio: 'pipe', timeout: 120000 }); } // 2. Bygg graf från kod console.log(` 🕸️ Bygger graf...`); const graph = this.buildGraph(projectDir, project.language); // 3. Hitta en historisk PR console.log(` 📋 Hittar historisk PR...`); const pr = this.findHistoricalPR(projectDir); // 4. Analysera PR console.log(` 🔍 Analyserar PR...`); const analysis = this.analyzePR(projectDir, pr); // 5. Utvärdera resultat console.log(` 📊 Utvärderar...`); const evaluation = this.evaluateAnalysis(analysis, project); return { name: project.name, language: project.language, graph: { nodes: graph.nodes.length, edges: graph.edges.length }, pr: pr.commit, analysis: { components: analysis.components?.length || 0, tests: analysis.tests?.length || 0 }, recall: evaluation.recall, precision: evaluation.precision, passed: evaluation.recall >= 70, ...evaluation }; } /** * Bygg graf från kod (förenklad) */ buildGraph(projectDir, language) { const nodes = []; const edges = []; // Identifiera filer const extensions = { javascript: 'js', python: 'py', go: 'go' }; try { const files = execSync( `find ${projectDir}/src ${projectDir}/lib ${projectDir} -name "*.${extensions[language]}" -type f 2>/dev/null | head -50`, { encoding: 'utf8' } ).trim().split('\n').filter(f => f); // Skapa noder från filer for (const file of files) { const name = file.split('/').pop().replace(/\.(js|py|go)$/, ''); nodes.push({ id: name, label: name, type: 'module', file }); } // Skapa kanter från imports (förenklad) for (const file of files) { try { const content = readFileSync(file, 'utf8'); const imports = this.extractImports(content, language); const fromNode = file.split('/').pop().replace(/\.(js|py|go)$/, ''); for (const imp of imports) { edges.push({ from: fromNode, to: imp, relation: 'imports', confidence: 0.8 }); } } catch {} } } catch {} return { nodes, edges }; } extractImports(content, language) { const imports = []; if (language === 'javascript') { const matches = content.match(/require\(['"]([^'"]+)['"]\)|import.*from\s+['"]([^'"]+)['"]/g); if (matches) { for (const match of matches) { const name = match.match(/['"]([^'"]+)['"]/)?.[1]; if (name && !name.startsWith('.')) { imports.push(name.split('/').pop()); } } } } else if (language === 'python') { const matches = content.match(/import\s+(\w+)|from\s+(\w+)\s+import/g); if (matches) { for (const match of matches) { const name = match.match(/import\s+(\w+)|from\s+(\w+)/)?.[1] || match.match(/import\s+(\w+)|from\s+(\w+)/)?.[2]; if (name) imports.push(name); } } } else if (language === 'go') { const matches = content.match(/import\s+["']([^"']+)["']/g); if (matches) { for (const match of matches) { const name = match.match(/["']([^"']+)["']/)?.[1]; if (name) imports.push(name.split('/').pop()); } } } return [...new Set(imports)]; } /** * Hitta en historisk PR */ findHistoricalPR(projectDir) { try { const log = execSync( `git log --merges --pretty=format:"%H|%s|%P" -n 5`, { cwd: projectDir, encoding: 'utf8' } ); const merges = log.trim().split('\n'); if (merges.length === 0) return { commit: 'HEAD', base: 'HEAD~1' }; const [hash, subject, parents] = merges[0].split('|'); const parentArray = parents ? parents.split(' ') : []; return { commit: hash.substring(0, 8), subject, base: parentArray[0] || 'HEAD~1', head: parentArray[1] || 'HEAD' }; } catch { return { commit: 'HEAD', base: 'HEAD~1' }; } } /** * Analysera PR */ analyzePR(projectDir, pr) { try { // Kör förenklad analys const files = execSync( `git diff --name-only ${pr.base}...${pr.head}`, { cwd: projectDir, encoding: 'utf8' } ).trim().split('\n').filter(f => f); const components = this.identifyComponents(files); const tests = this.suggestTests(components); return { components, tests, files }; } catch (e) { return { error: e.message }; } } identifyComponents(files) { const components = []; for (const file of files) { const name = file.split('/').pop()?.replace(/\.(js|ts|py|go)$/, ''); if (name) components.push(name); } return [...new Set(components)]; } suggestTests(components) { const tests = []; for (const comp of components) { tests.push(`${comp}.test`); tests.push(`${comp}.integration`); } return tests; } /** * Utvärdera analys */ evaluateAnalysis(analysis, project) { // Jämför med faktiska ändringar (förenklad) const actualComponents = analysis.files?.map(f => f.split('/').pop()?.replace(/\.(js|ts|py|go)$/, '') ).filter(Boolean) || []; const predicted = new Set(analysis.components || []); const actual = new Set(actualComponents); const tp = [...predicted].filter(c => actual.has(c)).length; const fp = [...predicted].filter(c => !actual.has(c)).length; const fn = [...actual].filter(c => !predicted.has(c)).length; const recall = actual.size > 0 ? Math.round(tp / actual.size * 100) : 0; const precision = predicted.size > 0 ? Math.round(tp / predicted.size * 100) : 0; return { recall, precision, truePositives: tp, falsePositives: fp, falseNegatives: fn, totalFiles: analysis.files?.length || 0 }; } // ── Rapport ───────────────────────────────────────────────────────────── generateReport(results) { const total = results.length; const passed = results.filter(r => r.passed).length; const avgRecall = results .filter(r => r.recall !== undefined) .reduce((a, r) => a + r.recall, 0) / total; return { timestamp: new Date().toISOString(), totalProjects: total, passed, failed: total - passed, avgRecall: Math.round(avgRecall) + '%', meetsCriteria: avgRecall >= 70, results }; } printReport(report) { console.log('\n╔═══════════════════════════════════════════════════════════════╗'); console.log('║ VALIDATION: FRÅGA 4 ║'); console.log('║ Kan SIL analysera okända projekt? ║'); console.log('╚═══════════════════════════════════════════════════════════════╝'); console.log(); console.log('📊 RESULTAT\n'); console.log(` Projekt: ${report.totalProjects}`); console.log(` Godkända: ${report.passed} ✅`); console.log(` Misslyckade: ${report.failed} ❌`); console.log(` Genomsnittlig recall: ${report.avgRecall}`); console.log(); console.log('📋 PER PROJEKT\n'); for (const result of report.results) { const icon = result.passed ? '✅' : '❌'; console.log(` ${icon} ${result.name} (${result.language})`); console.log(` Noder: ${result.graph?.nodes || 0}, Kanter: ${result.graph?.edges || 0}`); console.log(` Recall: ${result.recall || 'N/A'}%`); console.log(` Precision: ${result.precision || 'N/A'}%`); if (result.error) { console.log(` Fel: ${result.error}`); } } console.log(); console.log('🎯 SUCCESS-KRITERIUM\n'); console.log(` Krav: ≥70% recall på nya projekt`); console.log(` Resultat: ${report.avgRecall}`); console.log(` ${report.meetsCriteria ? '✅ UPPFYLLT' : '❌ EJ UPPFYLLT'}`); console.log(); console.log('═══════════════════════════════════════════════════════════════\n'); } saveResults(results) { for (const result of results) { writeFileSync(RESULTS_LOG, JSON.stringify(result) + '\n', { flag: 'a' }); } } } // ── Main ────────────────────────────────────────────────────────────────── const tester = new ExternalProjectTester(); tester.runAllTests();