KaliBench 2026: Arsitektur Benchmark Cybersecurity CLI pada 1.642 Tools Kali Linux, Runtime-Free Verifiable Rewards, dan Peningkatan Model 8B Setara 685B MoE

// Production Implementation: KaliBench Command Parser & Runtime-Free Verifiable Reward Evaluator
// Berdasarkan Arsitektur NeurIPS 2026 (arXiv:2610.02206 - Pengfei Li, Naufal Suryanto et al.)
//
// Karakteristik Inti:
// 1. Deterministic Canonicalization: Normalisasi alias sub-tool, flag order, dan quoting rules
// 2. Multi-Component Reward Decomposition: Output format (2.0), Tool (1.0), Opt-Args F1 (1.5), Pos-Args F1 (1.5), Exact (2.0)
// 3. Runtime-Free Grading: Verifikasi sintaksis & semantik langsung tanpa eksekusi sandbox berbahaya
// 4. Mode Routing: Hinted, Restricted (20 candidate tools), & Unrestricted (1.642 tools)

export interface ParsedCommand {
  rawCommand: string;
  toolName: string;
  subtool?: string;
  positionalArgs: string[];
  optionalFlags: Map;
  isValidSyntax: boolean;
}

export interface GroundTruthPair {
  id: string;
  phaseId: '1' | '2' | '3' | '4' | '5';
  query: string;
  groundTruthCommand: string;
  canonicalTool: string;
  canonicalSubtool?: string;
  allowedAliases: string[];
  requiredPositionalArgs: string[];
  requiredOptionalFlags: Record;
}

export interface VerificationRewardResult {
  exactMatchScore: number;       // r_exact in {0, 1} (weight 2.0)
  toolSelectionScore: number;    // r_tool in {0, 1}  (weight 1.0)
  optionalArgsF1: number;        // r_opt in [0, 1]   (weight 1.5)
  positionalArgsF1: number;      // r_pos in [0, 1]   (weight 1.5)
  outputFormatScore: number;     // r_fmt in {0, 0.5, 1} (weight 2.0)
  aggregateReward: number;       // Total reward untuk GRPO training
}

export class KaliBenchRewardEvaluator {
  private weightFormat: number = 2.0;
  private weightTool: number = 1.0;
  private weightOptionalF1: number = 1.5;
  private weightPositionalF1: number = 1.5;
  private weightExact: number = 2.0;

  /**
   * Deterministic Command Parser & Canonicalizer
   * Membedah command string menjadi komponen tool, sub-tool, flags, dan positional arguments
   */
  public parseAndCanonicalize(command: string): ParsedCommand {
    const trimmed = command.trim();
    if (!trimmed) {
      return { rawCommand: '', toolName: '', positionalArgs: [], optionalFlags: new Map(), isValidSyntax: false };
    }

    // Ekstraksi token dengan penanganan quotes (single & double)
    const tokens: string[] = [];
    const regex = /"([^"]*)"|'([^']*)'|(\S+)/g;
    let match: RegExpExecArray | null;

    while ((match = regex.exec(trimmed)) !== null) {
      tokens.push(match[1] ?? match[2] ?? match[3]);
    }

    if (tokens.length === 0) {
      return { rawCommand: trimmed, toolName: '', positionalArgs: [], optionalFlags: new Map(), isValidSyntax: false };
    }

    const toolName = tokens[0].toLowerCase();
    const positionalArgs: string[] = [];
    const optionalFlags: Map = new Map();

    let i = 1;
    let subtool: string | undefined = undefined;

    // Sub-tool check (contoh: 'aircrack-ng -w' vs 'git clone' vs 'msfvenom -p')
    if (i < tokens.length && !tokens[i].startsWith('-')) {
      subtool = tokens[i].toLowerCase();
      i++;
    }

    while (i < tokens.length) {
      const token = tokens[i];
      if (token.startsWith('--')) {
        const eqIdx = token.indexOf('=');
        if (eqIdx !== -1) {
          const key = token.slice(2, eqIdx).toLowerCase();
          const val = token.slice(eqIdx + 1);
          optionalFlags.set(key, val);
        } else if (i + 1 < tokens.length && !tokens[i + 1].startsWith('-')) {
          optionalFlags.set(token.slice(2).toLowerCase(), tokens[i + 1]);
          i++;
        } else {
          optionalFlags.set(token.slice(2).toLowerCase(), true);
        }
      } else if (token.startsWith('-') && token.length > 1) {
        const flag = token.slice(1);
        if (i + 1 < tokens.length && !tokens[i + 1].startsWith('-')) {
          optionalFlags.set(flag, tokens[i + 1]);
          i++;
        } else {
          optionalFlags.set(flag, true);
        }
      } else {
        positionalArgs.push(token);
      }
      i++;
    }

    return {
      rawCommand: trimmed,
      toolName,
      subtool,
      positionalArgs,
      optionalFlags,
      isValidSyntax: true,
    };
  }

  /**
   * Menghitung Runtime-Free Verifiable Reward untuk RLVR / GRPO Optimization
   */
  public evaluateVerifiableReward(
    generatedText: string,
    groundTruth: GroundTruthPair
  ): VerificationRewardResult {
    // 1. Evaluasi Output Format Reward
    let formatScore = 0.0;
    let extractedCommand = generatedText.trim();

    // Regex check untuk format target (misal: code block atau tag )
    const codeBlockMatch = generatedText.match(/```(?:bash|sh)?\s*([\s\S]*?)```/);
    if (codeBlockMatch) {
      extractedCommand = codeBlockMatch[1].trim();
      formatScore = 1.0;
    } else if (generatedText.includes('\n') || generatedText.length > 250) {
      formatScore = 0.5; // Approximate format
    } else {
      formatScore = 1.0; // Clean one-line response
    }

    const parsed = this.parseAndCanonicalize(extractedCommand);

    // 2. Tool Selection Score
    const isToolCorrect = parsed.toolName === groundTruth.canonicalTool.toLowerCase() ||
      groundTruth.allowedAliases.map(a => a.toLowerCase()).includes(parsed.toolName);
    const toolScore = isToolCorrect ? 1.0 : 0.0;

    // 3. Optional Arguments F1 Score
    const gtFlags = groundTruth.requiredOptionalFlags;
    const gtFlagKeys = Object.keys(gtFlags);
    let matchedFlags = 0;

    parsed.optionalFlags.forEach((val, key) => {
      if (gtFlags[key] !== undefined) {
        if (typeof gtFlags[key] === 'boolean' || gtFlags[key] === val) {
          matchedFlags++;
        }
      }
    });

    const precisionOpt = parsed.optionalFlags.size > 0 ? matchedFlags / parsed.optionalFlags.size : (gtFlagKeys.length === 0 ? 1.0 : 0.0);
    const recallOpt = gtFlagKeys.length > 0 ? matchedFlags / gtFlagKeys.length : (parsed.optionalFlags.size === 0 ? 1.0 : 0.0);
    const optF1 = (precisionOpt + recallOpt > 0) ? (2 * precisionOpt * recallOpt) / (precisionOpt + recallOpt) : 0.0;

    // 4. Positional Arguments F1 Score
    const gtPos = groundTruth.requiredPositionalArgs;
    let matchedPos = 0;
    parsed.positionalArgs.forEach(arg => {
      if (gtPos.includes(arg)) matchedPos++;
    });

    const precisionPos = parsed.positionalArgs.length > 0 ? matchedPos / parsed.positionalArgs.length : (gtPos.length === 0 ? 1.0 : 0.0);
    const recallPos = gtPos.length > 0 ? matchedPos / gtPos.length : (parsed.positionalArgs.length === 0 ? 1.0 : 0.0);
    const posF1 = (precisionPos + recallPos > 0) ? (2 * precisionPos * recallPos) / (precisionPos + recallPos) : 0.0;

    // 5. Exact Match Score
    const isExact = isToolCorrect && optF1 === 1.0 && posF1 === 1.0 && 
      (parsed.subtool === groundTruth.canonicalSubtool || !groundTruth.canonicalSubtool);
    const exactScore = isExact ? 1.0 : 0.0;

    // Total Weighted Reward (GRPO Objective)
    const aggregateReward = 
      (formatScore * this.weightFormat) +
      (toolScore * this.weightTool) +
      (optF1 * this.weightOptionalF1) +
      (posF1 * this.weightPositionalF1) +
      (exactScore * this.weightExact);

    return {
      exactMatchScore: exactScore,
      toolSelectionScore: toolScore,
      optionalArgsF1: Number(optF1.toFixed(3)),
      positionalArgsF1: Number(posF1.toFixed(3)),
      outputFormatScore: formatScore,
      aggregateReward: Number(aggregateReward.toFixed(3)),
    };
  }
}