RetireOPD 2026: Arsitektur Self-Retiring On-Policy Distillation & Decoupled Skill Teachers untuk Mengatasi Ceiling Reinforcement Learning pada Autonomous LLM Agents

"""
RetireOPD Runtime Engine: Decoupled Skill Teacher & Adaptive Retirement Gate
Berdasarkan Arsitektur Riset arXiv:2609.20784 (September 2026, Yu et al., Zhejiang University)
Mekanisme:
1. Privileged Teacher Initialization with Group Relative Policy Optimization (GRPO)
2. Joint GRPO + On-Policy Distillation (OPD) Loss Engine
3. Multi-Window Discrepancy Tracking (ρ_m^(K)) & Competence-Gated Retirement Trigger (η_m)
"""

import numpy as np
from dataclasses import dataclass, field
from typing import List, Dict, Optional, Tuple

@dataclass
class TrajectorySample:
    trajectory_id: str
    tokens: List[int]
    log_probs_student: List[float]
    log_probs_teacher: List[float]
    reward: float

@dataclass
class WindowMonitoringMetrics:
    window_index: int
    mean_token_gap_Kn: float
    window_avg_gap_Km: float
    discrepancy_reduction_rho: float
    relative_competence_eta: float
    student_success_rate: float
    teacher_success_rate: float
    is_retired: bool

class AdaptiveRetirementGate:
    """
    Mengontrol terminasi otomatis supervisi guru (Teacher Retirement)
    berdasarkan konvergensi disparitas token (shrink rate) dan ambang kompetensi siswa.
    """
    def __init__(
        self,
        window_size_W: int = 16,
        delta_threshold: float = 0.05,
        gamma_competence_threshold: float = 0.90,
        initial_lambda_opd: float = 1.0,
    ):
        self.W = window_size_W
        self.delta = delta_threshold
        self.gamma = gamma_competence_threshold
        self.current_lambda = initial_lambda_opd
        
        self.step_counter = 0
        self.current_window_K_steps: List[float] = []
        self.window_averages_Km: List[float] = []
        self.window_success_rates: List[float] = []
        self.teacher_baseline_sr: float = 0.85
        self.has_retired: bool = False
        self.retirement_step: Optional[int] = None

    def record_step_progress(self, trajectories: List[TrajectorySample], teacher_sr: float) -> Optional[WindowMonitoringMetrics]:
        self.teacher_baseline_sr = teacher_sr
        G = len(trajectories)
        if G == 0:
            return None

        # Hitung K_n: rata-rata selisih log-probabilitas token antara guru dan siswa
        step_token_gaps = []
        step_success_count = sum(1 for tau in trajectories if tau.reward >= 1.0)
        step_sr = step_success_count / G

        for tau in trajectories:
            seq_len = len(tau.tokens)
            if seq_len == 0:
                continue
            # Δ_t = log π_T(y_t | x, c^+, y_= self.W:
            window_idx = len(self.window_averages_Km)
            K_bar_m = float(np.mean(self.current_window_K_steps))
            self.window_averages_Km.append(K_bar_m)
            SR_m = float(np.mean(self.window_success_rates[-self.W:]))

            rho_m = 0.0
            eta_m = 0.0

            if window_idx >= 1:
                prev_K_bar = self.window_averages_Km[window_idx - 1]
                # Formula (8): ρ_m^(K) = (K̄_m - K̄_{m-1}) / K̄_m
                if abs(K_bar_m) > 1e-7:
                    rho_m = (K_bar_m - prev_K_bar) / K_bar_m
                
                # Formula (8): η_m = (SR_m + SR_{m-1}) / (2 * SR_T)
                prev_SR = float(np.mean(self.window_success_rates[-2*self.W : -self.W])) if len(self.window_success_rates) >= 2*self.W else SR_m
                eta_m = (SR_m + prev_SR) / (2.0 * max(self.teacher_baseline_sr, 1e-4))

                # Evaluasi kriteria retirement: m* = inf { m >= 2 : ρ_m^(K) >= δ ∧ η_m >= γ }
                if not self.has_retired and rho_m >= self.delta and eta_m >= self.gamma:
                    self.has_retired = True
                    self.current_lambda = 0.0  # Teacher dilepas, pelatihan berlanjut murni RL
                    self.retirement_step = self.step_counter

            metrics = WindowMonitoringMetrics(
                window_index=window_idx,
                mean_token_gap_Kn=K_n,
                window_avg_gap_Km=K_bar_m,
                discrepancy_reduction_rho=rho_m,
                relative_competence_eta=eta_m,
                student_success_rate=SR_m,
                teacher_success_rate=self.teacher_baseline_sr,
                is_retired=self.has_retired,
            )

            # Reset window buffer
            self.current_window_K_steps = []
            return metrics

        return None

class RetireOPDOrchestrator:
    """
    Pipeline pengelola siklus hidup pelatihan gabungan GRPO + On-Policy Distillation (OPD).
    """
    def __init__(self, clip_epsilon: float = 0.2, initial_lambda: float = 1.0):
        self.clip_epsilon = clip_epsilon
        self.gate = AdaptiveRetirementGate(initial_lambda_opd=initial_lambda)

    def compute_grpo_advantages(self, rewards: List[float]) -> np.ndarray:
        R = np.array(rewards, dtype=np.float32)
        std = np.std(R)
        if std < 1e-6:
            return np.zeros_like(R)
        return (R - np.mean(R)) / std

    def compute_joint_loss(
        self,
        trajectories: List[TrajectorySample],
        advantages: np.ndarray,
        old_student_log_probs: List[List[float]],
    ) -> Tuple[float, float, float]:
        """
        Menghitung: L_student(θ) = L_GRPO(θ) + λ * L_OPD(θ)
        """
        grpo_losses = []
        opd_losses = []

        for i, (tau, A_i) in enumerate(zip(trajectories, advantages)):
            old_lp = old_student_log_probs[i]
            cur_lp = tau.log_probs_student
            teacher_lp = tau.log_probs_teacher

            token_grpo = []
            token_opd = []

            for t in range(len(tau.tokens)):
                # r_t(θ) = π_θ(y_t) / π_{θ_old}(y_t)
                ratio = np.exp(cur_lp[t] - old_lp[t])
                surr1 = ratio * A_i
                surr2 = np.clip(ratio, 1.0 - self.clip_epsilon, 1.0 + self.clip_epsilon) * A_i
                token_grpo.append(min(surr1, surr2))

                # D_KL approximation pada sampling on-policy
                # L_OPD token gap = log π_T(y_t) - log π_θ(y_t)
                kl_t = -(teacher_lp[t] - cur_lp[t])
                token_opd.append(kl_t)

            grpo_losses.append(-np.mean(token_grpo))
            opd_losses.append(np.mean(token_opd))

        l_grpo = float(np.mean(grpo_losses))
        l_opd = float(np.mean(opd_losses))
        total_loss = l_grpo + (self.gate.current_lambda * l_opd)

        return total_loss, l_grpo, l_opd