ViSkill (arXiv:2610.12403): Reinforcing VLM Agents with Evolving Visual-Native Skills, Mengatasi Representation Gap, dan Lonjakan Akurasi 0.91 Melampaui GPT-5

// Production Implementation: ViSkill Visual-Native Memory & Policy Co-Evolution Engine
// Mengimplementasikan Closed-Loop Visual Skill Cards, Geometric Matching, Dual-Gate Distillation & Agreement Reward
// Berdasarkan spesifikasi formal arXiv:2610.12403 (Zhejiang University)

export interface GeometricDescriptor {
  objectCentroids: Array<{ label: string; x: number; y: number; z?: number }>;
  relativeDistances: Array<{ pair: [string, string]; distance: number }>;
  obstacleBoundaries: Array<{ minX: number; maxX: number; minY: number; maxY: number }>;
  spatialFeatureVector: number[]; // Normalized vector embedding for cosine/euclidean similarity
}

export interface TrajectoryStep {
  observationImageUri: string;
  actionToken: string;
  stepIndex: number;
  coordinates?: { x: number; y: number; z?: number };
}

export interface CompositeSkillCard {
  id: string;
  compositeImageUri: string;      // Rendered unified image containing annotated trajectory + strategy text
  distilledStrategyText: string;   // High-level intent distilled by policy
  referenceTrajectory: TrajectoryStep[];
  geometry: GeometricDescriptor;
  historicalUtility: number;       // u_k in [0, 1]
  usageCount: number;              // n_k
  createdAtStep: number;
}

export interface ViSkillConfig {
  alphaSimilarityWeight: number;   // alpha: bobot kesamaan geometris vs utilitas historis (default: 0.90)
  emaBeta: number;                 // beta: koefisien EMA pembaruan utilitas (default: 0.10)
  lambdaSkillReward: number;       // lambda: bobot trajectory agreement reward (default: 0.80)
  deltaRetrievalThreshold: number; // delta_r: ambang batas skor kueri untuk retrieval (default: 0.80)
  deltaNoveltyThreshold: number;   // delta_d: ambang batas kemiripan maksimum untuk admisi (default: 0.90)
  maxLibraryCapacity: number;      // Kapasitas maksimum skill card sebelum LRU/utility eviction
}

export class ViSkillEngine {
  private library: Map<string, CompositeSkillCard> = new Map();
  private config: ViSkillConfig;

  constructor(customConfig?: Partial<ViSkillConfig>) {
    this.config = {
      alphaSimilarityWeight: customConfig?.alphaSimilarityWeight ?? 0.90,
      emaBeta: customConfig?.emaBeta ?? 0.10,
      lambdaSkillReward: customConfig?.lambdaSkillReward ?? 0.80,
      deltaRetrievalThreshold: customConfig?.deltaRetrievalThreshold ?? 0.80,
      deltaNoveltyThreshold: customConfig?.deltaNoveltyThreshold ?? 0.90,
      maxLibraryCapacity: customConfig?.maxLibraryCapacity ?? 500,
    };
  }

  /**
   * 1. Skill Retrieval: Geometric Similarity Matching + Historical Utility Scoring
   * Formula Eq. (2): q_k = alpha * s_k + (1 - alpha) * u_k
   */
  public retrieveBestSkill(currentGeometry: GeometricDescriptor): {
    skill: CompositeSkillCard | null;
    compositeScore: number;
    highestMatchScore: number;
  } {
    if (this.library.size === 0) {
      return { skill: null, compositeScore: 0, highestMatchScore: 0 };
    }

    let bestSkill: CompositeSkillCard | null = null;
    let highestScore = -Infinity;
    let maxSimilarityAcrossLibrary = 0;

    for (const skill of this.library.values()) {
      const geometricSimilarity = this.calculateGeometricSimilarity(
        currentGeometry.spatialFeatureVector,
        skill.geometry.spatialFeatureVector
      );

      if (geometricSimilarity > maxSimilarityAcrossLibrary) {
        maxSimilarityAcrossLibrary = geometricSimilarity;
      }

      const compositeScore =
        this.config.alphaSimilarityWeight * geometricSimilarity +
        (1 - this.config.alphaSimilarityWeight) * skill.historicalUtility;

      if (compositeScore > highestScore) {
        highestScore = compositeScore;
        bestSkill = skill;
      }
    }

    // Gating Retrieval: Hanya kembalikan skill jika composite score >= delta_r
    const isRetrieved = highestScore >= this.config.deltaRetrievalThreshold;

    return {
      skill: isRetrieved ? bestSkill : null,
      compositeScore: highestScore,
      highestMatchScore: maxSimilarityAcrossLibrary,
    };
  }

  /**
   * 2. Trajectory Agreement Reward Shaping
   * Formula Eq. (6): r_skill(tau, z) = max(0, (ell - (L(tau) - ell)) / L_z)
   * Memberikan partial credit terukur untuk trajektori gagal yang selaras dengan panduan visual
   */
  public computeSkillGuidedReward(
    executedTrajectory: TrajectoryStep[],
    retrievedSkill: CompositeSkillCard | null,
    taskSuccess: boolean,
    environmentReward: number
  ): { finalReward: number; skillAgreementReward: number } {
    if (taskSuccess || !retrievedSkill) {
      return { finalReward: environmentReward, skillAgreementReward: 0 };
    }

    const referenceSteps = retrievedSkill.referenceTrajectory;
    const lz = referenceSteps.length;
    const ltau = executedTrajectory.length;

    if (lz === 0) return { finalReward: environmentReward, skillAgreementReward: 0 };

    // Hitung titik terjauh (ell) di mana trajektori agen konsisten dengan trajektori referensi visual
    let furthestMatchedIndex = 0;
    for (let i = 0; i < Math.min(ltau, lz); i++) {
      if (executedTrajectory[i].actionToken === referenceSteps[i].actionToken) {
        furthestMatchedIndex = i + 1;
      } else {
        break; // Divergence detected
      }
    }

    // Penalti deviasi langkah ekstra di luar kecocokan: (ltau - furthestMatchedIndex)
    const rawAgreement = (furthestMatchedIndex - (ltau - furthestMatchedIndex)) / lz;
    const skillAgreementReward = Math.max(0, rawAgreement);

    // Formula Eq. (7): R(tau) = r_env(tau) + lambda * r_skill(tau, z)
    const finalReward = environmentReward + this.config.lambdaSkillReward * skillAgreementReward;

    return { finalReward, skillAgreementReward };
  }

  /**
   * 3. Update Utility via Exponential Moving Average (EMA)
   * Formula Eq. (3): u_k <- (1 - beta) * u_k + beta * y(tau)
   */
  public updateSkillUtility(skillId: string, taskSuccess: boolean): void {
    const skill = this.library.get(skillId);
    if (!skill) return;

    const outcomeIndicator = taskSuccess ? 1.0 : 0.0;
    skill.historicalUtility =
      (1 - this.config.emaBeta) * skill.historicalUtility +
      this.config.emaBeta * outcomeIndicator;
    skill.usageCount += 1;
  }

  /**
   * 4. Online Skill Distillation with Dual-Gating (Quality & Novelty)
   * Formula Eq. (4): L(tau) <= L_bud(x) AND q_max < delta_d
   */
  public async distillTrajectoryIfEligible(
    trajectory: TrajectoryStep[],
    taskSuccess: boolean,
    lengthBudget: number,
    highestMatchScore: number,
    geometry: GeometricDescriptor,
    policyStrategyDistiller: (traj: TrajectoryStep[]) => Promise<string>,
    visualComposer: (strategyText: string, traj: TrajectoryStep[]) => Promise<string>
  ): Promise<CompositeSkillCard | null> {
    if (!taskSuccess) return null;

    // Quality Gate: Trajektori tidak boleh melebihi batas panjang optimal (menolak rute memutar/suboptimal)
    if (trajectory.length > lengthBudget) {
      return null;
    }

    // Novelty Gate: Library belum mencakup konfigurasi geometris ini
    if (highestMatchScore >= this.config.deltaNoveltyThreshold) {
      return null;
    }

    // Sintesis deskripsi strategi tingkat tinggi & komposisi visual-native card
    const distilledStrategy = await policyStrategyDistiller(trajectory);
    const compositeImageUri = await visualComposer(distilledStrategy, trajectory);

    const newSkill: CompositeSkillCard = {
      id: `viskill-${nanoid(8)}`,
      compositeImageUri,
      distilledStrategyText: distilledStrategy,
      referenceTrajectory: trajectory,
      geometry,
      historicalUtility: 1.0, // Initial optimism for newly verified trajectory
      usageCount: 1,
      createdAtStep: Date.now(),
    };

    // Jika library mencapai batas kapasitas, lakukan eviksi berdasarkan lowest utility-frequency
    // Formula Eq. (5): k_evict = argmin_k (u_k * log(n_k + 1))
    if (this.library.size >= this.config.maxLibraryCapacity) {
      this.evictLowestValueSkill();
    }

    this.library.set(newSkill.id, newSkill);
    return newSkill;
  }

  private evictLowestValueSkill(): void {
    let lowestScore = Infinity;
    let targetEvictionKey: string | null = null;

    for (const [id, skill] of this.library.entries()) {
      const score = skill.historicalUtility * Math.log(skill.usageCount + 1);
      if (score < lowestScore) {
        lowestScore = score;
        targetEvictionKey = id;
      }
    }

    if (targetEvictionKey) {
      this.library.delete(targetEvictionKey);
    }
  }

  private calculateGeometricSimilarity(vecA: number[], vecB: number[]): number {
    if (vecA.length !== vecB.length || vecA.length === 0) return 0;
    let dot = 0;
    let normA = 0;
    let normB = 0;
    for (let i = 0; i < vecA.length; i++) {
      dot += vecA[i] * vecB[i];
      normA += vecA[i] * vecA[i];
      normB += vecB[i] * vecB[i];
    }
    if (normA === 0 || normB === 0) return 0;
    const cosine = dot / (Math.sqrt(normA) * Math.sqrt(normB));
    return Math.max(0, Math.min(1, (cosine + 1) / 2));
  }

  public getLibrarySize(): number {
    return this.library.size;
  }
}