{
  "model": "Novi-Nano-Base",
  "version": 2,
  "vocab_size": 8192,
  "block_size": 256,
  "n_embd": 96,
  "n_layer": 4,
  "n_head": 4,
  "n_inner": 384,
  "parameters": 1258560,
  "training_tokens": 300000000,
  "tokens_seen": 300023808,
  "optimizer_steps": 2289,
  "batch_size": 256,
  "grad_accum_steps": 2,
  "learning_rate": 0.0003,
  "min_learning_rate": 3e-05,
  "warmup_steps": 500,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "validation_tokens": 5000000,
  "final_validation_loss": 5.418698577137737,
  "final_validation_perplexity": 225.5853494508474,
  "best_validation_loss": 5.418698577137737
}