τ^τ-Bench 2026: Tolok Ukur End-to-End Konstruksi AI Agents, Analisis 6 Karakteristik Kegagalan Coding Agents, dan Disparitas 23.9% vs 82.2% Expert Ceiling

// τ^τ-Bench: Autonomous Agent Construction Harness & Production Verification Kernel
// Spesifikasi Berdasarkan Riset arXiv:2609.04611 (Sierra & Princeton University, September 2026)
// Mengorkestrasi Ekstraksi Spesifikasi Korpus, Wawancara Klien Interaktif, Penanganan Quiet API Defects, dan Budget Enforcement

export interface ConstructionTaskSpec {
  taskId: string;
  domain: 'airline' | 'retail' | 'telecom' | 'banking';
  evidenceCorpus: {
    handbooks: Array<{ id: string; title: string; contentText: string }>;
    transcripts: Array<{ id: string; conversationLog: string }>;
    spreadsheets: Array<{ id: string; sheetName: string; rows: Record[] }>;
  };
  clientInterface: {
    enabled: boolean;
    interviewClient: (question: string) => Promise<{ answer: string; clarifyingContext?: string }>;
  };
  clientRestApi: {
    baseUrl: string;
    endpoints: Array<{ path: string; method: string; requiresCursorPagination: boolean }>;
    callEndpoint: (path: string, payload: any) => Promise<{ statusCode: number; data: any; nextCursor?: string }>;
  };
  budgetConstraints: {
    maxCreditsPerConversation: number;
    allowedModels: string[];
  };
}

export interface SpecificationState {
  establishedRules: Map;
  unresolvedAmbiguities: string[];
  discoveredApiDefects: Array<{ endpoint: string; defectType: string; mitigation: string }>;
}

export class HyperTauAgentConstructor {
  private specState: SpecificationState = {
    establishedRules: new Map(),
    unresolvedAmbiguities: [],
    discoveredApiDefects: [],
  };

  private clientInterviewCount = 0;
  private totalServingSpend = 0;

  constructor(private task: ConstructionTaskSpec) {}

  /**
   * 1. Deep Corpus Comprehension (Mengatasi 'Searched, but not read')
   * Tidak sekadar ripgrep kata kunci, melainkan ekstraksi proposisi kebijakan lengkap.
   */
  public async digestEvidenceCorpus(): Promise {
    for (const handbook of this.task.evidenceCorpus.handbooks) {
      this.extractPolicyConstraints(handbook.contentText);
    }

    for (const sheet of this.task.evidenceCorpus.spreadsheets) {
      for (const row of sheet.rows) {
        if (row.key && row.value !== undefined) {
          this.specState.establishedRules.set(String(row.key), row.value);
        }
      }
    }
  }

  private extractPolicyConstraints(text: string): void {
    // Parser aturan kebijakan bisnis (contoh: batas refund, minimal balance, waktu pembatalan)
    const policyRegex = /(?:minimum|maksimal|threshold|cutoff|fee|penalty)\s+([a-zA-Z_]+)\s+(?:is|=|adalah|sebesar)\s+([\$0-9\.,]+)/gi;
    let match;
    while ((match = policyRegex.exec(text)) !== null) {
      const paramName = match[1].toLowerCase();
      const paramValue = match[2];
      this.specState.establishedRules.set(paramName, paramValue);
    }
  }

  /**
   * 2. Proactive Client Interview Loop (Mengatasi kelalaian 0.3% client calls)
   * Menginterogasi pemangku kepentingan saat mendeteksi ambiguitas atau aturan yang hilang dari korpus tertulis.
   */
  public async reconcileAmbiguitiesWithClient(): Promise {
    if (!this.task.clientInterface.enabled) return;

    // Identifikasi aturan penting domain perbankan/telekomunikasi yang tidak memiliki nilai definitif
    const criticalRequirements = ['funding_floor_minimum', 'unauthorized_dispute_window_days', 'cancellation_grace_period_hours'];
    
    for (const req of criticalRequirements) {
      if (!this.specState.establishedRules.has(req)) {
        this.clientInterviewCount++;
        const res = await this.task.clientInterface.interviewClient(
          `Mohon klarifikasi kebijakan bisnis: Berapa nilai pasti untuk parameter '${req}' yang belum tercantum dalam dokumen resmi?`
        );
        this.specState.establishedRules.set(req, res.answer);
      }
    }
  }

  /**
   * 3. Robust API Adapter with Quiet Defect Detection (Mengatasi kegagalan next_cursor)
   * Menangani paginasi tak terlihat dan pemulihan timeout transaksi dengan idempotency key.
   */
  public async queryClientApiExhaustive(endpointPath: string, queryPayload: any): Promise {
    const results: any[] = [];
    let currentCursor: string | undefined = undefined;
    let pageCount = 0;

    do {
      pageCount++;
      const payloadWithCursor = currentCursor ? { ...queryPayload, cursor: currentCursor } : queryPayload;
      
      const response = await this.task.clientRestApi.callEndpoint(endpointPath, payloadWithCursor);
      
      if (response.statusCode === 200 && response.data) {
        if (Array.isArray(response.data.items)) {
          results.push(...response.data.items);
        } else {
          results.push(response.data);
        }
        
        // Deteksi quiet defect: cursor ada pada response namun kerap diabaikan agent naive
        if (response.nextCursor && response.nextCursor !== currentCursor) {
          currentCursor = response.nextCursor;
          this.specState.discoveredApiDefects.push({
            endpoint: endpointPath,
            defectType: 'silent_pagination_cursor',
            mitigation: 'Recursive cursor-following loop activated.',
          });
        } else {
          currentCursor = undefined;
        }
      } else {
        break;
      }
    } while (currentCursor && pageCount < 50);

    return results;
  }

  /**
   * 4. Serving Budget Gate & Model Selection
   * Memastikan model inferensi yang dirilis tidak melampaui batas kredit per sesi.
   */
  public selectOptimalServingModel(): { modelId: string; estimatedSpend: number } {
    const budget = this.task.budgetConstraints.maxCreditsPerConversation;
    
    // Model seleksi bergradasi biaya (pricing credits per 1k token conversation turn)
    const candidates = [
      { id: 'claude-haiku-4-5', costPerTurn: 0.015 },
      { id: 'qwen3-30b-a3b', costPerTurn: 0.018 },
      { id: 'gpt-5-6-mini', costPerTurn: 0.025 },
      { id: 'claude-sonnet-5', costPerTurn: 0.055 },
      { id: 'claude-opus-5', costPerTurn: 0.180 },
    ].filter(m => this.task.budgetConstraints.allowedModels.includes(m.id));

    // Pilih model paling cerdas yang estimasi biayanya < 85% budget cap untuk menghindari penalti diskualifikasi
    const eligible = candidates.filter(m => m.costPerTurn * 4.2 <= budget * 0.85);
    const chosen = eligible.length > 0 ? eligible[eligible.length - 1] : candidates[0];

    return {
      modelId: chosen.id,
      estimatedSpend: Number((chosen.costPerTurn * 4.2).toFixed(4)),
    };
  }

  /**
   * 5. Anti-Cheating Invariant: Strict Assertion Lock
   * Menolak perubahan assertion unit test yang sengaja dilemahkan oleh developer agent.
   */
  public verifySelfTestIntegrity(initialAssertionHash: string, finalAssertionHash: string): boolean {
    if (initialAssertionHash !== finalAssertionHash) {
      throw new Error('[HARNESS_GATE_VIOLATION] Developer agent terdeteksi memanipulasi atau melemahkan unit test suite!');
    }
    return true;
  }
}