model:
  name_or_path: Qwen/Qwen3.6-35B-A3B
  revision: main
  tokenizer_revision: main
  torch_dtype: bfloat16
  trust_remote_code: false
  load_in_4bit: false
  device_map: auto
  attn_implementation: sdpa
target:
  total_reduction_ratio: 0.35
  expert_svd_ratio: 100.0
calibration:
  source: nvidia-cascade
  dataset: nvidia/Nemotron-Cascade-2-SFT-Data
  seed: 1337
  num_sequences: 1000
  sequence_length: 2048
  subset_weights:
    math: 0.21
    science: 0.11
    chat: 0.56
    instruction_following: 0.0338
    conversational_agent: 0.0331
    swe: 0.02
    terminal_agent: 0.0331
stage1_grape:
  num_calibration_samples: 1024
  phase_a_batch_size: 32
  phase_b_batch_size: 16
  super_expert_detection:
    ma_growth_ratio: 3.0
    moe_output_growth_ratio: 2.0
    ma_ratio: 100.0
    a_max_fraction: 0.1
    aimer_enabled: true
    aimer_bottom_pct: 0.01
    aimer_layer_max_fraction: 0.1
    sink_token_enabled: true
    sink_token_score_ratio: 10.0
    sink_token_freq_threshold: 0.99
    sink_token_max_per_layer_cap: 10
    magnitude_topk_per_l_layer: 16
    zscore_threshold: 2.5
    max_blacklisted_per_layer: 4
    global_blacklist_cap_pct: 0.05
  ablation_filter:
    enabled: true
    holdout_samples: 100
    blacklist_threshold: 0.001
    batch_size: 8
  similarity_metric: cka
  min_experts_per_layer: 128
  entropy_tolerance: 0.1
stage2_reap_ream:
  experts_implementation: grouped_mm
  batch_size: 32
  num_calibration_samples: 1000
  reap_min_active_tokens: 32
  covariance_storage_dtype: float16
  max_merge_group_size: 8
  ream_cost_sigma_threshold: 1.5
  ream_cost_bump_ratio: 0.1
  ream:
    frequency_weighted_merge: true
  assignment_solver: greedy
  sinkhorn_epsilon_init: 1.0
  sinkhorn_epsilon_final: 0.01
  sinkhorn_iters: 200
  cost_alignment: pre
  cost_whitening: none
  cost_asymmetric: false
  cost_topk_filter: 48
  capacity_util_threshold: 0.25
  em_refinement_rounds: 0
  em_convergence_break: true
  expert_distill_steps: 0
  expert_distill_lr: 0.0001
  expert_distill_betas:
  - 0.9
  - 0.95
  expert_distill_token_cap: 8192
  expert_distill_skip_singletons: true
  expert_distill_loss_plateau_steps: 50
  expert_distill_loss_plateau_eps: 0.0001
  expert_distill_parallel_groups: 1
  expert_distill_optimizer_dtype: fp32
  merge_heal_enabled: true
  merge_heal_train_router: true
  merge_heal_lr: 0.0001
  merge_heal_adamw_betas:
  - 0.9
  - 0.95
  merge_heal_grad_clip: 1.0
  merge_heal_holdout_fraction: 0.1
  merge_heal_patience: 5
  merge_heal_eval_interval: 25
  merge_heal_min_rel_delta: 0.01
  merge_heal_max_steps: 20000
  merge_heal_token_cap: 262144
  merge_heal_minibatch_size: 8192
stage3_svd:
  scope: moe_experts_only
  batch_size: 16
  d_rank:
    parameter_cost_omega_mode: auto
    per_projection_weight:
      gate_proj: 1.33
      up_proj: 0.67
      down_proj: 1.0
  swift_svd_plus:
    alpha_grid:
    - 0.0
    - 0.1
    - 0.2
    - 0.3
    - 0.4
    - 0.5
    - 0.6
    - 0.7
    - 0.8
    - 0.9
    - 1.0
    validation_samples: 512
    validation_batch_size: 16
    metric: wikitext2_ppl
    per_group_type: true
  aa_svd:
    use_post_prune_inputs: true
    cross_covariance: true
  bcov_storage_dtype: float16
  block_refine:
    enabled: true
    epochs: 25
    batch_size: 32
    learning_rate: 0.0001
    warmup_ratio: 0.1
    weight_decay: 0.0
stage4_eora:
  per_expert: true
  compensation_budget_pct: 0.03
  eigenspace_rank_cap: 128
stage5_router_kd:
  optimizer: adamw
  learning_rate: 5.0e-05
  experts_implementation: grouped_mm
  weight_decay: 0.01
  epochs: 1
  lr_schedule: cosine
  warmup_ratio: 0.05
  lr_min_ratio: 0.1
  save_best: true
  best_metric_ema_alpha: 0.2
  early_stop_patience: 8
  shuffle_batches_each_epoch: false
  batch_size: 8
  gradient_accumulation: 1
  teacher_load_in_4bit: false
  teacher_model_repo: Qwen/Qwen3.6-35B-A3B-FP8
  max_sequence_length: 512
  kd_temperature: 1.0
  kd_seq_chunk_size: 32
  max_calibration_samples: 3000
  checkpoint_every_n_steps: 100
  trainable_name_patterns:
  - mlp.gate.weight
  frozen_name_patterns:
  - experts
  - shared_expert
  - self_attn
  - norm
  - embed
  - lm_head
  merge_repair:
    enabled: false
    mse_weight: 1.0
    merge_map_path: null
  torch_compile: true
stage6_validate:
  mode: thermometer
  thermometer:
    corpus: wikitext
    wikitext:
      dataset: wikitext
      subset: wikitext-2-raw-v1
      split: test
    num_sequences: 64
    sequence_length: 2048
    bpt_batch_size: 8
    lm_eval_batch_size: auto:8
    arc_easy_limit: 100
    hellaswag_limit: 200
    subset_weights:
      math: 0.35
      swe: 0.25
      chat: 0.25
      science: 0.15
    teacher_cache_path: /cache/ablations/_shared/thermometer_teacher_cache.json
  experts_implementation: grouped_mm
  ppl_batch_size: 8
  lm_eval_batch_size: auto:8
  gen_batch_size: 8
  torch_compile: true
  teacher_eval_cache:
    enabled: true
    cache_path: /cache/ablations/_shared/teacher_eval_cache.json
  wikitext2:
    enabled: true
    dataset: Salesforce/wikitext
    subset: wikitext-2-raw-v1
    split: test
    sequence_length: 2048
  strict_revision_pinning: true
  dataset_revisions:
    wikitext_ppl: b08601e04326c79dfdd32d625aee71d232d685c3
    humaneval: 7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544
    math500: 6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be
  zero_shot:
    enabled: true
    tasks:
    - arc_challenge
    - hellaswag
  generative:
    enabled: true
    humaneval:
      max_new_tokens: 512
    math500:
      num_samples: 500
      max_new_tokens: 1024
  thresholds:
    wikitext2_ppl_relative_max_increase: 0.03
    arc_c_absolute_max_drop: 0.015
    hellaswag_absolute_max_drop: 0.015
    humaneval_absolute_max_drop: 0.03
    math500_absolute_max_drop: 0.03
    measured_reduction_min: 0.3
  imatrix:
    enabled: false
    ngl: 99
    ctx_size: 2048
    llama_cpp_dir: null
logging:
  level: INFO
  log_every_n_steps: 50
  save_intermediate_every_n_layers: 10