Upload folder using huggingface_hub

finemath-4plus-p0-medium/2000/checkpoint_metadata.json

@@ -0,0 +1,18 @@
+{
+  "custom_metas": null,
+  "dp": 4,
+  "metas": {
+    "consumed_train_samples": 2048000,
+    "data_stages": [
+      {
+        "consumed_train_samples": 2048000,
+        "name": "stable phase",
+        "start_training_step": 1
+      }
+    ],
+    "last_stage_idx": 0,
+    "last_train_step": 2000
+  },
+  "tp": 1,
+  "version": "1.4"
+}

finemath-4plus-p0-medium/2000/config.yaml

@@ -0,0 +1,107 @@
+checkpoints:
+  checkpoint_interval: 1000
+  checkpoints_path: /scratch/e1154543/checkpoints/finemath-4plus-p0-medium
+  checkpoints_path_is_shared_file_system: true
+  load_lr_scheduler: false
+  load_optimizer: false
+  resume_checkpoint_path: /scratch/e1154543/checkpoints/nanotron_pretrained_checkpoints/Nanotron-Llama-3.2-3B
+  save_final_state: true
+  save_initial_state: false
+data_stages:
+- data:
+    dataset:
+      dataset_folder:
+      - /scratch/e1154543/tokenized_finemath/finemath-4-plus-p0.0
+      dataset_weights: null
+    num_loading_workers: 0
+    seed: 8
+  name: stable phase
+  start_training_step: 1
+general:
+  benchmark_csv_path: null
+  consumed_train_samples: 2048000
+  ignore_sanity_checks: true
+  project: llama3-3B-CPT
+  run: finemath-4plus
+  seed: 6
+  step: 2000
+lighteval: null
+logging:
+  iteration_step_info_interval: 1
+  log_level: info
+  log_level_replica: info
+model:
+  ddp_bucket_cap_mb: 25
+  dtype: bfloat16
+  init_method:
+    std: 0.041666666666666664
+  make_vocab_size_divisible_by: 1
+  model_config:
+    bos_token_id: 128000
+    eos_token_id: 128001
+    hidden_act: silu
+    hidden_size: 3072
+    initializer_range: 0.02
+    intermediate_size: 8192
+    is_llama_config: true
+    max_position_embeddings: 4096
+    num_attention_heads: 24
+    num_hidden_layers: 28
+    num_key_value_heads: 8
+    pad_token_id: null
+    pretraining_tp: 2
+    rms_norm_eps: 1.0e-05
+    rope_interleaved: false
+    rope_scaling:
+      factor: 32.0
+      high_freq_factor: 4.0
+      low_freq_factor: 1.0
+      original_max_position_embeddings: 8192
+      rope_type: llama3
+    rope_theta: 500000.0
+    tie_word_embeddings: true
+    use_cache: true
+    vocab_size: 128256
+optimizer:
+  accumulate_grad_in_fp32: true
+  clip_grad: 1.0
+  learning_rate_scheduler:
+    learning_rate: 5.0e-05
+    lr_decay_starting_step: 50000
+    lr_decay_steps: 10000
+    lr_decay_style: linear
+    lr_warmup_steps: 1000
+    lr_warmup_style: linear
+    min_decay_lr: 0
+  optimizer_factory:
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    adam_eps: 1.0e-08
+    name: adamW
+    torch_adam_is_fused: true
+  weight_decay: 0.01
+  zero_stage: 1
+parallelism:
+  dp: 4
+  expert_parallel_size: 1
+  pp: 2
+  pp_engine: 1f1b
+  recompute_layer: true
+  tp: 1
+  tp_linear_async_communication: true
+  tp_mode: REDUCE_SCATTER
+  tp_recompute_allgather: true
+profiler: null
+s3_upload: null
+tokenizer:
+  tokenizer_max_length: null
+  tokenizer_name_or_path: meta-llama/Llama-3.2-3B
+  tokenizer_revision: null
+tokens:
+  batch_accumulation_per_replica: 32
+  limit_test_batches: 0
+  limit_val_batches: 0
+  micro_batch_size: 8
+  sequence_length: 4096
+  train_steps: 60000
+  val_check_interval: -1

finemath-4plus-p0-medium/2000/lr_scheduler/lr_scheduler_pp-0-of-2_tp-0-of-1_exp-0-of-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d15eab5443380661f6060e52f1bac0f11c98c689f99fc53acf5e95059741ebae
+size 3248

finemath-4plus-p0-medium/2000/lr_scheduler/lr_scheduler_pp-1-of-2_tp-0-of-1_exp-0-of-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dbaa202e60a75b58e94f93135cb29e0387adb214aa5748b7e9e265509f2ce40
+size 2800

finemath-4plus-p0-medium/2000/model/model/decoder/0/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b00c4115bf116ea3fb6c68c3fd44a1e7071ce3487872684f79dfbab4d49130f8
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/0/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:869bb4119765b9ccffd78a0c3eb08e270902f54f29d7eb075162e900a9b275a8
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/0/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0322b59a3a6728237ca6d827aa2f7a30bb4a89b9f679ebce1daee9a97691c174
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/0/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0e725afc54a70aa17f962249ff3eb85a1652e51ac17bbba6fb4da201d67b64
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/0/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3ad3af57bf9bd84856cf1dec1de6be9682bf7454f7f5bf2a1a666b950dbb47c
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/0/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657c90bc8cc0354905ef35dfa9b23eed900ad7712edb78e80016367340ec01dc
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/1/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c3e0c38fb9b81f400308450aa122342dfbd371b5640c8ecfb7a80eb96a0b4f
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/1/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e666655919ff31959da08fd25b11c1dc9bef94c6073f94e473e88c80f4f7f9
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/1/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00e8fd03bce8e9b0f3891a018fe2c2dcded0f75958d99f4c7a21bb51057f18cf
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/1/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a97b6e195e1cabb98af0523de0193f35709b64639b6ad0ea40e84375c96249f
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/1/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b82b9d2b4e5bf840184034b82fb98f6ebf06b66b4c53df0d79643811970b7273
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/1/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb9e656223871af4280c25e500853c4285e14e917795119f20d715ee15c4536
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/10/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c10a983b1009424943dc3e1174d6a03bcdc006c829d66d25b5586fa6bc8f18a1
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/10/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b06eed08d870503f098adb451be49745bea4650cde672c634326ae0c691afbe3
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/10/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c29871b55778b1ce6407acce92a171f1eaccb7cf6a205763596397ce4dfb0545
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/10/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5afa94521382328cc45451bbb874bebc24084768698947c37d251802496c2617
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/10/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a5500a49c161286c514994621026679531ec61bde545951000c71bad84550d
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/10/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:580847a4491e86873993c8c68450bcfd97e1ed7fe816f219d53d02374e93bc68
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/11/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54363b339cc19b71b7f6b77d70b2b87ed1abb92013e5a924572fb65a6ddc144d
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/11/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecb82a333c4bd7d7ede2188f8f6772f79e05f3ef4add7c302d4472d3826f0286
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/11/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e678d97a7f02af4483c67f43ad80803172cdd4b28aff7a2622904f8f163b8dd
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/11/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b022a10faecc558c87f6de24229b4714c8033ef7c7949545da20f8797c952f5a
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/11/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1c8cabf874aa9cd26a2578ca7fd1f5f60970c43245d7dd4efb5c5d4cb6455e7
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/11/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfed1d18895f2f149e71cae046173d19f994627173e01c0d07c1db85dbb0bc06
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/12/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f036d2d60dcc4d4e08972c8c7c0cf9eddf7168a26447283a6b46ab8ef9835b8
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/12/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10be7939784c21d1b0b9fb0e14c9e28bfcb99de4ee38bad2223656f311f1f052
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/12/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66ae234430bbd324d7f713cf117ce4aa4211653418d92283aa1c6d50b6c159a5
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/12/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b003fa7a86a2d60c3a5ae59694a6e646bdea8e2d38bc5d3700b1897d29decbd
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/12/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8a301a48249b0780db305cf8f1f39414a292920805211518611aab75fa2e2f
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/12/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e034b4d7b088f08ca538d6b101791229dc834806d0320fa91ac9c6efffff64c0
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/13/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6c526cc2b7475028a441f53004f9207077e6a00ee763243112f118eda2c741
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/13/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e052dab289cc5649f013fe32a994527ce69dda0c8152a5eafe1c22c30fd945de
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/13/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7072517242c51143e6b5ef42ec0b0d5dbca256c36a1c6f23d79be5629ac53de7
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/13/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfbaf8f1b46f6d38c07e814b737ea9b043a42ec9407673096db01933c986a7b2
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/13/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:666955b7890a10c07df467ec4d9c1394b71bd33c75e61fc2fc5843a8928fd8c0
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/13/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:443edf4972fb7b725b5c603c773e5a664bfa0d9f6fb6b926d2080f7e03c19375
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/14/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e71f545e78874d104145bbab5c35e261184c61abbbc571728e064daddd40850
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/14/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6962498f0ea437acdd01dc9b6e187fbd3dd3ba1f667f3988ac72fca5a732b028
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/14/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:131687f85fa8a0b055afe530cbc022d8af0de597697fce3c3e5c37390fc61a88
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/14/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac6dfff7c3cd65abf13575a4e22108fcead8a26382acf60c0d505b887749058d
+size 50331888

finemath-4plus-p0-medium/2000/model/model/decoder/14/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fa8589d30818469dbcc4705bb43c052f2fecf7ef038c632c3b7a7089b4a07f9
+size 100663592

finemath-4plus-p0-medium/2000/model/model/decoder/14/pp_block/post_attention_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2040aa04176724781ef74e189110c5a74e5fd56cf19ab29e0c99ac018bd3d49
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/15/pp_block/attn/o_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d6713de612bd2d5abbc5840af8cfd14f76e96218ce9b87d059edd6e3a88ec04
+size 18874608

finemath-4plus-p0-medium/2000/model/model/decoder/15/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a203b76674197c91c61d9d65e29737372bd68f84f3454524b700d1bf0e12f5b6
+size 31457632

finemath-4plus-p0-medium/2000/model/model/decoder/15/pp_block/input_layernorm/model_weight.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f11de1746f54db1c6dea734fc2da81012c8b766233bf1df9ec53ab0d6365bf1
+size 6240

finemath-4plus-p0-medium/2000/model/model/decoder/15/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-2_tp-rank-0-of-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0541d8bd010f2e69a5c6c4ba70eab4008a805ab81119ccd888f0f3183ead1d39
+size 50331888