Hugging Face's logo

duynht
/
stok-3b

Model card Files
xet
 Community
duynht commited on
Commit
8446e96
·
verified ·
1 Parent(s): fc2aa6b

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. 1000/checkpoint_metadata.json +18 -0
  2. 1000/config.yaml +107 -0
  3. 1000/lr_scheduler/lr_scheduler_pp-0-of-1_tp-0-of-1_exp-0-of-1.pt +3 -0
  4. 1000/model/model/decoder/0/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  5. 1000/model/model/decoder/0/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  6. 1000/model/model/decoder/0/pp_block/input_layernorm/model_weight.safetensors +3 -0
  7. 1000/model/model/decoder/0/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  8. 1000/model/model/decoder/0/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  9. 1000/model/model/decoder/0/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  10. 1000/model/model/decoder/1/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  11. 1000/model/model/decoder/1/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  12. 1000/model/model/decoder/1/pp_block/input_layernorm/model_weight.safetensors +3 -0
  13. 1000/model/model/decoder/1/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  14. 1000/model/model/decoder/1/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  15. 1000/model/model/decoder/1/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  16. 1000/model/model/decoder/10/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  17. 1000/model/model/decoder/10/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  18. 1000/model/model/decoder/10/pp_block/input_layernorm/model_weight.safetensors +3 -0
  19. 1000/model/model/decoder/10/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  20. 1000/model/model/decoder/10/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  21. 1000/model/model/decoder/10/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  22. 1000/model/model/decoder/11/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  23. 1000/model/model/decoder/11/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  24. 1000/model/model/decoder/11/pp_block/input_layernorm/model_weight.safetensors +3 -0
  25. 1000/model/model/decoder/11/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  26. 1000/model/model/decoder/11/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  27. 1000/model/model/decoder/11/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  28. 1000/model/model/decoder/12/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  29. 1000/model/model/decoder/12/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  30. 1000/model/model/decoder/12/pp_block/input_layernorm/model_weight.safetensors +3 -0
  31. 1000/model/model/decoder/12/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  32. 1000/model/model/decoder/12/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  33. 1000/model/model/decoder/12/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  34. 1000/model/model/decoder/13/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  35. 1000/model/model/decoder/13/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  36. 1000/model/model/decoder/13/pp_block/input_layernorm/model_weight.safetensors +3 -0
  37. 1000/model/model/decoder/13/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  38. 1000/model/model/decoder/13/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  39. 1000/model/model/decoder/13/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  40. 1000/model/model/decoder/14/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  41. 1000/model/model/decoder/14/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  42. 1000/model/model/decoder/14/pp_block/input_layernorm/model_weight.safetensors +3 -0
  43. 1000/model/model/decoder/14/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  44. 1000/model/model/decoder/14/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  45. 1000/model/model/decoder/14/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  46. 1000/model/model/decoder/15/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  47. 1000/model/model/decoder/15/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  48. 1000/model/model/decoder/15/pp_block/input_layernorm/model_weight.safetensors +3 -0
  49. 1000/model/model/decoder/15/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  50. 1000/model/model/decoder/15/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
1000/checkpoint_metadata.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "custom_metas": null,
3
+ "dp": 4,
4
+ "metas": {
5
+ "consumed_train_samples": 256000,
6
+ "data_stages": [
7
+ {
8
+ "consumed_train_samples": 256000,
9
+ "name": "stable phase",
10
+ "start_training_step": 1
11
+ }
12
+ ],
13
+ "last_stage_idx": 0,
14
+ "last_train_step": 1000
15
+ },
16
+ "tp": 1,
17
+ "version": "1.4"
18
+ }
1000/config.yaml ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ checkpoints:
2
+ checkpoint_interval: 1000
3
+ checkpoints_path: checkpoints/llama-3.2-3b-finemath-4plus-p0.1-bs256-tc60B
4
+ checkpoints_path_is_shared_file_system: true
5
+ load_lr_scheduler: false
6
+ load_optimizer: false
7
+ resume_checkpoint_path: checkpoints/nanotron_pretrained_checkpoints/Nanotron-Llama-3.2-3B
8
+ save_final_state: true
9
+ save_initial_state: false
10
+ data_stages:
11
+ - data:
12
+ dataset:
13
+ dataset_folder:
14
+ - finemath/tokenization/finemath-4-plus-p0.1
15
+ dataset_weights: null
16
+ num_loading_workers: 0
17
+ seed: 8
18
+ name: stable phase
19
+ start_training_step: 1
20
+ general:
21
+ benchmark_csv_path: null
22
+ consumed_train_samples: 256000
23
+ ignore_sanity_checks: true
24
+ project: llama3-3B-CPT
25
+ run: finemath-4plus-p0.1
26
+ seed: 6
27
+ step: 1000
28
+ lighteval: null
29
+ logging:
30
+ iteration_step_info_interval: 1
31
+ log_level: info
32
+ log_level_replica: info
33
+ model:
34
+ ddp_bucket_cap_mb: 25
35
+ dtype: bfloat16
36
+ init_method:
37
+ std: 0.041666666666666664
38
+ make_vocab_size_divisible_by: 1
39
+ model_config:
40
+ bos_token_id: 128000
41
+ eos_token_id: 128001
42
+ hidden_act: silu
43
+ hidden_size: 3072
44
+ initializer_range: 0.02
45
+ intermediate_size: 8192
46
+ is_llama_config: true
47
+ max_position_embeddings: 4096
48
+ num_attention_heads: 24
49
+ num_hidden_layers: 28
50
+ num_key_value_heads: 8
51
+ pad_token_id: null
52
+ pretraining_tp: 2
53
+ rms_norm_eps: 1.0e-05
54
+ rope_interleaved: false
55
+ rope_scaling:
56
+ factor: 32.0
57
+ high_freq_factor: 4.0
58
+ low_freq_factor: 1.0
59
+ original_max_position_embeddings: 8192
60
+ rope_type: llama3
61
+ rope_theta: 500000.0
62
+ tie_word_embeddings: true
63
+ use_cache: true
64
+ vocab_size: 128256
65
+ optimizer:
66
+ accumulate_grad_in_fp32: true
67
+ clip_grad: 1.0
68
+ learning_rate_scheduler:
69
+ learning_rate: 5.0e-05
70
+ lr_decay_starting_step: 50000
71
+ lr_decay_steps: 10000
72
+ lr_decay_style: linear
73
+ lr_warmup_steps: 1000
74
+ lr_warmup_style: linear
75
+ min_decay_lr: 0
76
+ optimizer_factory:
77
+ adam_beta1: 0.9
78
+ adam_beta2: 0.95
79
+ adam_eps: 1.0e-08
80
+ name: adamW
81
+ torch_adam_is_fused: true
82
+ weight_decay: 0.01
83
+ zero_stage: 1
84
+ parallelism:
85
+ dp: 4
86
+ expert_parallel_size: 1
87
+ pp: 1
88
+ pp_engine: 1f1b
89
+ recompute_layer: true
90
+ tp: 1
91
+ tp_linear_async_communication: true
92
+ tp_mode: REDUCE_SCATTER
93
+ tp_recompute_allgather: true
94
+ profiler: null
95
+ s3_upload: null
96
+ tokenizer:
97
+ tokenizer_max_length: null
98
+ tokenizer_name_or_path: meta-llama/Llama-3.2-3B
99
+ tokenizer_revision: null
100
+ tokens:
101
+ batch_accumulation_per_replica: 8
102
+ limit_test_batches: 0
103
+ limit_val_batches: 0
104
+ micro_batch_size: 8
105
+ sequence_length: 4096
106
+ train_steps: 60000
107
+ val_check_interval: -1
1000/lr_scheduler/lr_scheduler_pp-0-of-1_tp-0-of-1_exp-0-of-1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b5a0033906491c612619e07a1355b9767a128036eecac5003f5532c82f43f47
3
+ size 4784
1000/model/model/decoder/0/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43a274b51b8ac2d650d4acf6b071a79f60dddb73aeb6d7ab6334cc9706c0045d
3
+ size 18874608
1000/model/model/decoder/0/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4003d885395382af278675d5c9034b6b1b117684c66eb9654eed480d99b18bbe
3
+ size 31457632
1000/model/model/decoder/0/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b9a717d2d56dd0dc0ea4f21b7cd60db0688eddfe6b0372896891bac25551ab7
3
+ size 6240
1000/model/model/decoder/0/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7638b0f2b4dbb7bb0b2a548237122bef9333695974f9cfa833929e4cb471eefb
3
+ size 50331888
1000/model/model/decoder/0/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e338a1a205b95324fbde11f896feae625983bf31e5029783a2a62867410bd422
3
+ size 100663592
1000/model/model/decoder/0/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3037edbfaa7d2867cf58d0afcdc09cf4b11f2c8218a92392d54b211033f65811
3
+ size 6240
1000/model/model/decoder/1/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d55be7c0444cc7fca091787d5436ff0a32ba8c7a52a2f762798a3d84946aa05f
3
+ size 18874608
1000/model/model/decoder/1/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc7e02fcc328a42072a0fc8ffc2d3b33c730d8da7f6244671f1464050b310262
3
+ size 31457632
1000/model/model/decoder/1/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a5ae6a514591af446e8dd5625e7eafe398537f5b79d7a45215345d0cb75f811
3
+ size 6240
1000/model/model/decoder/1/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bb0c1a33804eddeb272f78fbfe73ac566e59e438fe09d3a1414944c18645499
3
+ size 50331888
1000/model/model/decoder/1/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4febd9c82a9ca83cd8ec4a3163f88866e7b3ba6b18006723128ec3d0e28a10e3
3
+ size 100663592
1000/model/model/decoder/1/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7090a6ebc777f277a0ce8737ff1d8ff724c2ae7a6fd3f54e9b806dd74f315a5f
3
+ size 6240
1000/model/model/decoder/10/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d61218cc162d188fd0882b568c236a8bc43167d718ea26d6a5a542ca2e03bead
3
+ size 18874608
1000/model/model/decoder/10/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58f0e42cb189a85810281eca63b8d6fdf465f847afac60092659ad5f660728ba
3
+ size 31457632
1000/model/model/decoder/10/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc7f9a67a335138e8946973562fbaef3c29f0e2ec55a8990c73a566a2186441d
3
+ size 6240
1000/model/model/decoder/10/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a97e73e76d601d6d1ac4c456f7a36fd66c5c58b58fc68871e1a7fae5647a3091
3
+ size 50331888
1000/model/model/decoder/10/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99e71dbb0f58e57a9d10551763bcca83ee5265073e71166bec1649fae7ef09fa
3
+ size 100663592
1000/model/model/decoder/10/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8dd48e95d616681c17729830773a368221ed96d8d85e7658b45afc2dda1daed2
3
+ size 6240
1000/model/model/decoder/11/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94e173273f2a75df20b7174d8260a2e8bebcbe786afe5a8065282eed8ad792e4
3
+ size 18874608
1000/model/model/decoder/11/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9daa4348789a21e67fcca1ea890a964e47a05128a7ae1ae2c97a3429655844fa
3
+ size 31457632
1000/model/model/decoder/11/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75ea48ea48c1892d3faa96e258305e4a345a542b2622da923291265c87dc74a0
3
+ size 6240
1000/model/model/decoder/11/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bb8b8d61c685436fdea4c89891599a705fbbeab6a2e81a228da3c6d07ed10a8
3
+ size 50331888
1000/model/model/decoder/11/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3e410aa2f5d15dd7af2791121e86c65b82969157ed8e9d9024836b05a75b9f7
3
+ size 100663592
1000/model/model/decoder/11/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da90ce6db083b96536587bfed0bfbdb39719137acb2ae014ca3e4dbadbcd8e10
3
+ size 6240
1000/model/model/decoder/12/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40ecb5aa40caa814d35b267aacb39048f06fd8d94faa3503918fddf77f8aa717
3
+ size 18874608
1000/model/model/decoder/12/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d4a2b54c12e8e5ff453acb1de16d99344f3837ad3020728736e20116fbbc2a3
3
+ size 31457632
1000/model/model/decoder/12/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1940240c09cee75f42470bd0859628ba9f2c78fc619979fbc482bed6d0c1b81b
3
+ size 6240
1000/model/model/decoder/12/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef272e59cdec932517c54cb14d6da152e4f5bfc42b7bc343a6969b583eed2f31
3
+ size 50331888
1000/model/model/decoder/12/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05bf5427d22963c6f7fe68adc897ed3d3fa87d76773c36965662decbc236bbe3
3
+ size 100663592
1000/model/model/decoder/12/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6248b8202b255e410d280d6e41b067c37a06283ea943afa49d57e8f3db0fce2
3
+ size 6240
1000/model/model/decoder/13/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:470767278c8be54d47bf53834eb04017a9d9be27716f705bca1196dba517887e
3
+ size 18874608
1000/model/model/decoder/13/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d05e6b9a92be6e6c4d04df7a1b0240e7cc669d92d2c48f4d61c7261133df25e4
3
+ size 31457632
1000/model/model/decoder/13/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d25240b311c40a22bdbc43cb30b5f53b3357306420358a1fd1c854319fe02e5
3
+ size 6240
1000/model/model/decoder/13/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27387e8ceb3b9c4124c3efd7dd3add65e0b56ec7c90551fe8ba7b892d80fe70a
3
+ size 50331888
1000/model/model/decoder/13/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e53116691cd4322ac04ec1f0f50ba5bb858b098d07cfb405c3c5acdae8b3e49a
3
+ size 100663592
1000/model/model/decoder/13/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:35a9e57b95a2d70939d248ba8be29e12400eea62df230283209fc27c4b865ed6
3
+ size 6240
1000/model/model/decoder/14/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:141696d69b8e3ee5d969d7eb9a7c21363857c918ad5d4b47b49861c73bbf7d54
3
+ size 18874608
1000/model/model/decoder/14/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cdb2587d77a938c8c0236a414bb538839fd82d28a3db993c871f638a8e998a51
3
+ size 31457632
1000/model/model/decoder/14/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82db41f7348fc0b3177b2e8e024888e455d67774fb50c97cbc4bff6c79b64360
3
+ size 6240
1000/model/model/decoder/14/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d765a5214578167031e558a7b34ac576cbb055dc0ed7775145cce4396a29890
3
+ size 50331888
1000/model/model/decoder/14/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df856c8027422ac977c0bf70fcff6dc33fe4e3b95d05915adf3496201dddc619
3
+ size 100663592
1000/model/model/decoder/14/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:188e7dd0b63b4b6318952ab6ebf26dc2f0c08eb78f3107abecf85bf4cdd83ece
3
+ size 6240
1000/model/model/decoder/15/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5c682cb5a260ee87b68b8b0913f7c3e287bb69fe95d03caac485d0e75f92262
3
+ size 18874608
1000/model/model/decoder/15/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c64ee159ac35d9618eac1399bf862142cccc23edc5d1415db3b16be146f1820
3
+ size 31457632
1000/model/model/decoder/15/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9baca8f1c7623bdfb447cbdd8a25dc54306a464f62871c1128d55a0bbd360308
3
+ size 6240
1000/model/model/decoder/15/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b0c2027d30869b85e98711e21f536237a678c4e446eb3155276daee5dfc331e
3
+ size 50331888
1000/model/model/decoder/15/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b22646c418e57649ff910e282ad1700ffed3910239cb41e478454ef10304658d
3
+ size 100663592