Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-3B-Instruct
-datasets: open-r1/OpenR1-Math-220k
 library_name: transformers
 model_name: Qwen2.5-3B-Open-R1-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen2.5-3B-Open-R1-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct) on the [open-r1/OpenR1-Math-220k](https://huggingface.co/datasets/open-r1/OpenR1-Math-220k) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/chenyukang2020-nvidia/huggingface/runs/v84ltger) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/chenyukang2020-nvidia/huggingface/runs/9wwsfr8r) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00012006735727737873,
-    "train_runtime": 1294.972,
+    "train_loss": 0.00015187089497744325,
+    "train_runtime": 1302.9313,
     "train_samples": 93733,
-    "train_samples_per_second": 72.382,
-    "train_steps_per_second": 2.263
+    "train_samples_per_second": 71.94,
+    "train_steps_per_second": 2.249
 }

config.json

@@ -22,7 +22,7 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.52.3",
-  "use_cache": true,
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:475b54e34ea27c5104c1ddd4518b6f609710ff981dc519a420d872c707c357c1
+oid sha256:40274e3ec9e8dc4d8fc7fa0b582f6f3b3ce9d41fdcacd2960d3506f7a944ed4b
 size 4957560304

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:118d6868b8fcb22c426966d8b4db5ffce230512ebc5c29e3e801ceedc92dd266
+oid sha256:a25ec01907d0215b10d3faf0841cf152610f26e87c5b5cccfd966513984b58f0
 size 1214366696

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00012006735727737873,
-    "train_runtime": 1294.972,
+    "train_loss": 0.00015187089497744325,
+    "train_runtime": 1302.9313,
     "train_samples": 93733,
-    "train_samples_per_second": 72.382,
-    "train_steps_per_second": 2.263
+    "train_samples_per_second": 71.94,
+    "train_steps_per_second": 2.249
 }

trainer_state.json

@@ -87024,7 +87024,7 @@
       "completions/min_terminated_length": 371.0,
       "epoch": 0.9903558931467099,
       "frac_reward_zero_std": 0.6875,
-      "grad_norm": 0.09052230139899971,
+      "grad_norm": 0.09051816360814072,
       "kl": 0.07421875,
       "learning_rate": 6.386247842353755e-09,
       "loss": 0.0128,
@@ -87046,23 +87046,23 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1668.0,
-      "completions/max_terminated_length": 1668.0,
-      "completions/mean_length": 811.07421875,
-      "completions/mean_terminated_length": 811.07421875,
-      "completions/min_length": 251.0,
-      "completions/min_terminated_length": 251.0,
+      "completions/max_length": 1555.0,
+      "completions/max_terminated_length": 1555.0,
+      "completions/mean_length": 802.6328125,
+      "completions/mean_terminated_length": 802.6328125,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.9906972774601007,
       "frac_reward_zero_std": 0.6875,
-      "grad_norm": 0.08715568374725409,
-      "kl": 0.0780029296875,
+      "grad_norm": 0.12488894096087517,
+      "kl": 0.0802001953125,
       "learning_rate": 5.967635461854304e-09,
-      "loss": 0.0142,
-      "num_tokens": 1391197756.0,
-      "reward": 2.07958984375,
-      "reward_std": 0.10148443281650543,
-      "rewards/accuracy_reward/mean": 0.087890625,
-      "rewards/accuracy_reward/std": 0.2834126651287079,
+      "loss": 0.0146,
+      "num_tokens": 1391193434.0,
+      "reward": 2.07763671875,
+      "reward_std": 0.09719854593276978,
+      "rewards/accuracy_reward/mean": 0.0859375,
+      "rewards/accuracy_reward/std": 0.28054583072662354,
       "rewards/format_reward/mean": 0.9921875,
       "rewards/format_reward/std": 0.08812850713729858,
       "rewards/tag_count_reward/mean": 0.99951171875,
@@ -87075,28 +87075,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1928.0,
-      "completions/max_terminated_length": 1928.0,
-      "completions/mean_length": 818.82421875,
-      "completions/mean_terminated_length": 818.82421875,
-      "completions/min_length": 155.0,
-      "completions/min_terminated_length": 155.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2002.0,
+      "completions/mean_length": 817.966796875,
+      "completions/mean_terminated_length": 815.5596923828125,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.9910386617734915,
       "frac_reward_zero_std": 0.59375,
-      "grad_norm": 0.10523394407924126,
-      "kl": 0.077392578125,
+      "grad_norm": 0.10402330505958786,
+      "kl": 0.0775146484375,
       "learning_rate": 5.563207782363078e-09,
-      "loss": 0.0098,
-      "num_tokens": 1391710914.0,
-      "reward": 2.0439453125,
-      "reward_std": 0.1362731009721756,
-      "rewards/accuracy_reward/mean": 0.056640625,
-      "rewards/accuracy_reward/std": 0.23138070106506348,
-      "rewards/format_reward/mean": 0.98828125,
-      "rewards/format_reward/std": 0.10772226005792618,
-      "rewards/tag_count_reward/mean": 0.9990234375,
-      "rewards/tag_count_reward/std": 0.015609703958034515,
+      "loss": 0.0097,
+      "num_tokens": 1391706153.0,
+      "reward": 2.048828125,
+      "reward_std": 0.13361230492591858,
+      "rewards/accuracy_reward/mean": 0.0625,
+      "rewards/accuracy_reward/std": 0.2422981858253479,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
+      "rewards/tag_count_reward/mean": 0.99609375,
+      "rewards/tag_count_reward/std": 0.04119514673948288,
       "step": 2903
     },
     {
@@ -87106,27 +87106,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1336.0,
-      "completions/max_terminated_length": 1336.0,
-      "completions/mean_length": 733.837890625,
-      "completions/mean_terminated_length": 733.837890625,
-      "completions/min_length": 288.0,
-      "completions/min_terminated_length": 288.0,
+      "completions/max_length": 1835.0,
+      "completions/max_terminated_length": 1835.0,
+      "completions/mean_length": 742.251953125,
+      "completions/mean_terminated_length": 742.251953125,
+      "completions/min_length": 220.0,
+      "completions/min_terminated_length": 220.0,
       "epoch": 0.9913800460868823,
-      "frac_reward_zero_std": 0.6875,
-      "grad_norm": 0.0962644733446894,
-      "kl": 0.080078125,
+      "frac_reward_zero_std": 0.53125,
+      "grad_norm": 0.12250308762985097,
+      "kl": 0.08251953125,
       "learning_rate": 5.172965377890915e-09,
-      "loss": 0.0052,
-      "num_tokens": 1392171551.0,
-      "reward": 2.06201171875,
-      "reward_std": 0.09075203537940979,
-      "rewards/accuracy_reward/mean": 0.06640625,
-      "rewards/accuracy_reward/std": 0.2492343932390213,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
-      "rewards/tag_count_reward/mean": 0.99951171875,
-      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "loss": 0.0111,
+      "num_tokens": 1392171098.0,
+      "reward": 2.06396484375,
+      "reward_std": 0.16945436596870422,
+      "rewards/accuracy_reward/mean": 0.0859375,
+      "rewards/accuracy_reward/std": 0.28054583072662354,
+      "rewards/format_reward/mean": 0.98046875,
+      "rewards/format_reward/std": 0.1385180652141571,
+      "rewards/tag_count_reward/mean": 0.99755859375,
+      "rewards/tag_count_reward/std": 0.03659820929169655,
       "step": 2904
     },
     {
@@ -87135,28 +87135,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1900.0,
-      "completions/mean_length": 771.89453125,
-      "completions/mean_terminated_length": 769.3972778320312,
-      "completions/min_length": 262.0,
-      "completions/min_terminated_length": 262.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1859.0,
+      "completions/max_terminated_length": 1859.0,
+      "completions/mean_length": 780.9609375,
+      "completions/mean_terminated_length": 780.9609375,
+      "completions/min_length": 309.0,
+      "completions/min_terminated_length": 309.0,
       "epoch": 0.9917214304002732,
-      "frac_reward_zero_std": 0.53125,
-      "grad_norm": 0.12188297667505647,
-      "kl": 0.0731201171875,
+      "frac_reward_zero_std": 0.5625,
+      "grad_norm": 0.09705909340956163,
+      "kl": 0.072509765625,
       "learning_rate": 4.79690880231587e-09,
-      "loss": 0.0159,
-      "num_tokens": 1392641337.0,
-      "reward": 2.05078125,
-      "reward_std": 0.16719159483909607,
-      "rewards/accuracy_reward/mean": 0.068359375,
-      "rewards/accuracy_reward/std": 0.25260838866233826,
-      "rewards/format_reward/mean": 0.984375,
-      "rewards/format_reward/std": 0.12414088100194931,
-      "rewards/tag_count_reward/mean": 0.998046875,
-      "rewards/tag_count_reward/std": 0.022032126784324646,
+      "loss": 0.0189,
+      "num_tokens": 1392645526.0,
+      "reward": 2.05029296875,
+      "reward_std": 0.15216603875160217,
+      "rewards/accuracy_reward/mean": 0.064453125,
+      "rewards/accuracy_reward/std": 0.24579854309558868,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
       "step": 2905
     },
     {
@@ -87166,27 +87166,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1612.0,
-      "completions/max_terminated_length": 1612.0,
-      "completions/mean_length": 761.6171875,
-      "completions/mean_terminated_length": 761.6171875,
-      "completions/min_length": 224.0,
-      "completions/min_terminated_length": 224.0,
+      "completions/max_length": 1650.0,
+      "completions/max_terminated_length": 1650.0,
+      "completions/mean_length": 745.669921875,
+      "completions/mean_terminated_length": 745.669921875,
+      "completions/min_length": 221.0,
+      "completions/min_terminated_length": 221.0,
       "epoch": 0.9920628147136639,
-      "frac_reward_zero_std": 0.375,
-      "grad_norm": 0.12393463207782816,
-      "kl": 0.07861328125,
+      "frac_reward_zero_std": 0.53125,
+      "grad_norm": 0.12680546398444922,
+      "kl": 0.08154296875,
       "learning_rate": 4.435038589380991e-09,
-      "loss": 0.0037,
-      "num_tokens": 1393116629.0,
-      "reward": 2.1787109375,
-      "reward_std": 0.20795820653438568,
-      "rewards/accuracy_reward/mean": 0.1953125,
-      "rewards/accuracy_reward/std": 0.3968288004398346,
-      "rewards/format_reward/mean": 0.986328125,
-      "rewards/format_reward/std": 0.1162383034825325,
-      "rewards/tag_count_reward/mean": 0.9970703125,
-      "rewards/tag_count_reward/std": 0.038198307156562805,
+      "loss": 0.0099,
+      "num_tokens": 1393112653.0,
+      "reward": 2.18994140625,
+      "reward_std": 0.16484864056110382,
+      "rewards/accuracy_reward/mean": 0.203125,
+      "rewards/accuracy_reward/std": 0.4027182459831238,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2906
     },
     {
@@ -87196,27 +87196,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1460.0,
-      "completions/max_terminated_length": 1460.0,
-      "completions/mean_length": 704.7890625,
-      "completions/mean_terminated_length": 704.7890625,
-      "completions/min_length": 310.0,
-      "completions/min_terminated_length": 310.0,
+      "completions/max_length": 1381.0,
+      "completions/max_terminated_length": 1381.0,
+      "completions/mean_length": 694.4453125,
+      "completions/mean_terminated_length": 694.4453125,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.9924041990270547,
-      "frac_reward_zero_std": 0.625,
-      "grad_norm": 0.10340246619120935,
-      "kl": 0.0853271484375,
+      "frac_reward_zero_std": 0.53125,
+      "grad_norm": 0.11603922474180073,
+      "kl": 0.0872802734375,
       "learning_rate": 4.087355252694325e-09,
-      "loss": 0.0031,
-      "num_tokens": 1393561353.0,
-      "reward": 2.15185546875,
-      "reward_std": 0.16270402073860168,
-      "rewards/accuracy_reward/mean": 0.162109375,
-      "rewards/accuracy_reward/std": 0.3689115643501282,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
-      "rewards/tag_count_reward/mean": 0.99951171875,
-      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "loss": 0.0072,
+      "num_tokens": 1393552081.0,
+      "reward": 2.12353515625,
+      "reward_std": 0.18750609457492828,
+      "rewards/accuracy_reward/mean": 0.138671875,
+      "rewards/accuracy_reward/std": 0.34594178199768066,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2907
     },
     {
@@ -87225,28 +87225,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1991.0,
-      "completions/mean_length": 815.96875,
-      "completions/mean_terminated_length": 813.5577392578125,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/max_terminated_length": 1959.0,
+      "completions/mean_length": 825.453125,
+      "completions/mean_terminated_length": 818.24755859375,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.9927455833404455,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11634370392277571,
-      "kl": 0.0787353515625,
+      "frac_reward_zero_std": 0.375,
+      "grad_norm": 0.12487904460610164,
+      "kl": 0.077392578125,
       "learning_rate": 3.753859285730022e-09,
-      "loss": 0.0138,
-      "num_tokens": 1394058921.0,
-      "reward": 2.0869140625,
-      "reward_std": 0.18937204778194427,
-      "rewards/accuracy_reward/mean": 0.109375,
-      "rewards/accuracy_reward/std": 0.31241437792778015,
-      "rewards/format_reward/mean": 0.98046875,
-      "rewards/format_reward/std": 0.1385180652141571,
-      "rewards/tag_count_reward/mean": 0.9970703125,
-      "rewards/tag_count_reward/std": 0.026930565014481544,
+      "loss": 0.0293,
+      "num_tokens": 1394054505.0,
+      "reward": 2.09375,
+      "reward_std": 0.20864024758338928,
+      "rewards/accuracy_reward/mean": 0.119140625,
+      "rewards/accuracy_reward/std": 0.32427072525024414,
+      "rewards/format_reward/mean": 0.978515625,
+      "rewards/format_reward/std": 0.14513419568538666,
+      "rewards/tag_count_reward/mean": 0.99609375,
+      "rewards/tag_count_reward/std": 0.031035220250487328,
       "step": 2908
     },
     {
@@ -87255,28 +87255,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1637.0,
-      "completions/mean_length": 776.6796875,
-      "completions/mean_terminated_length": 774.1917724609375,
-      "completions/min_length": 217.0,
-      "completions/min_terminated_length": 217.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1732.0,
+      "completions/max_terminated_length": 1732.0,
+      "completions/mean_length": 787.2265625,
+      "completions/mean_terminated_length": 787.2265625,
+      "completions/min_length": 254.0,
+      "completions/min_terminated_length": 254.0,
       "epoch": 0.9930869676538363,
-      "frac_reward_zero_std": 0.4375,
-      "grad_norm": 0.12612242780687172,
-      "kl": 0.0765380859375,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.11516269657952301,
+      "kl": 0.0797119140625,
       "learning_rate": 3.4345511618238957e-09,
-      "loss": 0.018,
-      "num_tokens": 1394542725.0,
-      "reward": 2.119140625,
-      "reward_std": 0.18908953666687012,
-      "rewards/accuracy_reward/mean": 0.12890625,
-      "rewards/accuracy_reward/std": 0.33542385697364807,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
-      "rewards/tag_count_reward/mean": 0.998046875,
-      "rewards/tag_count_reward/std": 0.03491804376244545,
+      "loss": 0.0174,
+      "num_tokens": 1394543709.0,
+      "reward": 2.09228515625,
+      "reward_std": 0.17656370997428894,
+      "rewards/accuracy_reward/mean": 0.115234375,
+      "rewards/accuracy_reward/std": 0.3196168541908264,
+      "rewards/format_reward/mean": 0.98046875,
+      "rewards/format_reward/std": 0.1385180652141571,
+      "rewards/tag_count_reward/mean": 0.99658203125,
+      "rewards/tag_count_reward/std": 0.03972800448536873,
       "step": 2909
     },
     {
@@ -87285,28 +87285,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1677.0,
-      "completions/max_terminated_length": 1677.0,
-      "completions/mean_length": 776.33203125,
-      "completions/mean_terminated_length": 776.33203125,
-      "completions/min_length": 345.0,
-      "completions/min_terminated_length": 345.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1820.0,
+      "completions/max_terminated_length": 1820.0,
+      "completions/mean_length": 778.107421875,
+      "completions/mean_terminated_length": 776.7182006835938,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
       "epoch": 0.9934283519672271,
-      "frac_reward_zero_std": 0.46875,
-      "grad_norm": 0.11521416617457238,
-      "kl": 0.0809326171875,
+      "frac_reward_zero_std": 0.4375,
+      "grad_norm": 0.3443649766769348,
+      "kl": 0.0982666015625,
       "learning_rate": 3.129431334175648e-09,
-      "loss": 0.0171,
-      "num_tokens": 1395031791.0,
-      "reward": 2.1220703125,
-      "reward_std": 0.20290058851242065,
-      "rewards/accuracy_reward/mean": 0.138671875,
-      "rewards/accuracy_reward/std": 0.34594178199768066,
-      "rewards/format_reward/mean": 0.986328125,
-      "rewards/format_reward/std": 0.1162383034825325,
-      "rewards/tag_count_reward/mean": 0.9970703125,
-      "rewards/tag_count_reward/std": 0.038198307156562805,
+      "loss": 0.0329,
+      "num_tokens": 1395033684.0,
+      "reward": 2.11474609375,
+      "reward_std": 0.20991787314414978,
+      "rewards/accuracy_reward/mean": 0.146484375,
+      "rewards/accuracy_reward/std": 0.35393697023391724,
+      "rewards/format_reward/mean": 0.97265625,
+      "rewards/format_reward/std": 0.16324250400066376,
+      "rewards/tag_count_reward/mean": 0.99560546875,
+      "rewards/tag_count_reward/std": 0.04260620102286339,
       "step": 2910
     },
     {
@@ -87315,26 +87315,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1906.0,
-      "completions/mean_length": 795.970703125,
-      "completions/mean_terminated_length": 793.5205688476562,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1908.0,
+      "completions/max_terminated_length": 1908.0,
+      "completions/mean_length": 782.6015625,
+      "completions/mean_terminated_length": 782.6015625,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.9937697362806179,
-      "frac_reward_zero_std": 0.46875,
-      "grad_norm": 0.1145051128307554,
-      "kl": 0.0772705078125,
+      "frac_reward_zero_std": 0.53125,
+      "grad_norm": 0.12301114676381447,
+      "kl": 0.0780029296875,
       "learning_rate": 2.8385002358466417e-09,
-      "loss": 0.0184,
-      "num_tokens": 1395529104.0,
-      "reward": 2.123046875,
-      "reward_std": 0.18341054022312164,
-      "rewards/accuracy_reward/mean": 0.1391129046678543,
-      "rewards/accuracy_reward/std": 0.3464137017726898,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
+      "loss": 0.0053,
+      "num_tokens": 1395524152.0,
+      "reward": 2.103515625,
+      "reward_std": 0.171902135014534,
+      "rewards/accuracy_reward/mean": 0.12096773833036423,
+      "rewards/accuracy_reward/std": 0.32641899585723877,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "rewards/tag_count_reward/mean": 0.998046875,
       "rewards/tag_count_reward/std": 0.022032126784324646,
       "step": 2911
@@ -87346,27 +87346,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1722.0,
-      "completions/max_terminated_length": 1722.0,
-      "completions/mean_length": 716.91796875,
-      "completions/mean_terminated_length": 716.91796875,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/max_length": 1561.0,
+      "completions/max_terminated_length": 1561.0,
+      "completions/mean_length": 714.876953125,
+      "completions/mean_terminated_length": 714.876953125,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.9941111205940087,
-      "frac_reward_zero_std": 0.40625,
-      "grad_norm": 0.32500195710324153,
-      "kl": 0.1021728515625,
+      "frac_reward_zero_std": 0.53125,
+      "grad_norm": 0.11373851773465948,
+      "kl": 0.08544921875,
       "learning_rate": 2.5617582797610174e-09,
-      "loss": 0.0096,
-      "num_tokens": 1395976998.0,
-      "reward": 2.08984375,
-      "reward_std": 0.19573727250099182,
-      "rewards/accuracy_reward/mean": 0.109375,
-      "rewards/accuracy_reward/std": 0.31241437792778015,
-      "rewards/format_reward/mean": 0.982421875,
-      "rewards/format_reward/std": 0.13154059648513794,
-      "rewards/tag_count_reward/mean": 0.998046875,
-      "rewards/tag_count_reward/std": 0.022032126784324646,
+      "loss": 0.026,
+      "num_tokens": 1395971001.0,
+      "reward": 2.08349609375,
+      "reward_std": 0.17116190493106842,
+      "rewards/accuracy_reward/mean": 0.1015625,
+      "rewards/accuracy_reward/std": 0.30236753821372986,
+      "rewards/format_reward/mean": 0.984375,
+      "rewards/format_reward/std": 0.12414088100194931,
+      "rewards/tag_count_reward/mean": 0.99755859375,
+      "rewards/tag_count_reward/std": 0.024608410894870758,
       "step": 2912
     },
     {
@@ -87376,27 +87376,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1732.0,
-      "completions/max_terminated_length": 1732.0,
-      "completions/mean_length": 803.76953125,
-      "completions/mean_terminated_length": 803.76953125,
-      "completions/min_length": 232.0,
-      "completions/min_terminated_length": 232.0,
+      "completions/max_length": 1902.0,
+      "completions/max_terminated_length": 1902.0,
+      "completions/mean_length": 794.6015625,
+      "completions/mean_terminated_length": 794.6015625,
+      "completions/min_length": 281.0,
+      "completions/min_terminated_length": 281.0,
       "epoch": 0.9944525049073996,
-      "frac_reward_zero_std": 0.65625,
-      "grad_norm": 0.09947710674034467,
-      "kl": 0.07958984375,
+      "frac_reward_zero_std": 0.5625,
+      "grad_norm": 0.11380928184475643,
+      "kl": 0.0804443359375,
       "learning_rate": 2.299205858702358e-09,
-      "loss": 0.0011,
-      "num_tokens": 1396464736.0,
-      "reward": 2.08056640625,
-      "reward_std": 0.13412871956825256,
-      "rewards/accuracy_reward/mean": 0.0947580635547638,
-      "rewards/accuracy_reward/std": 0.29317617416381836,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
-      "rewards/tag_count_reward/mean": 0.99853515625,
-      "rewards/tag_count_reward/std": 0.019099153578281403,
+      "loss": 0.0148,
+      "num_tokens": 1396454045.0,
+      "reward": 2.11669921875,
+      "reward_std": 0.15615960955619812,
+      "rewards/accuracy_reward/mean": 0.13709677755832672,
+      "rewards/accuracy_reward/std": 0.34429675340652466,
+      "rewards/format_reward/mean": 0.984375,
+      "rewards/format_reward/std": 0.12414088100194931,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
       "step": 2913
     },
     {
@@ -87405,28 +87405,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2024.0,
-      "completions/max_terminated_length": 2024.0,
-      "completions/mean_length": 813.1953125,
-      "completions/mean_terminated_length": 813.1953125,
-      "completions/min_length": 236.0,
-      "completions/min_terminated_length": 236.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2010.0,
+      "completions/mean_length": 835.59765625,
+      "completions/mean_terminated_length": 833.2250366210938,
+      "completions/min_length": 245.0,
+      "completions/min_terminated_length": 245.0,
       "epoch": 0.9947938892207903,
-      "frac_reward_zero_std": 0.59375,
-      "grad_norm": 0.08836543570664221,
-      "kl": 0.0740966796875,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.10149774742284243,
+      "kl": 0.074462890625,
       "learning_rate": 2.0508433453170218e-09,
-      "loss": 0.0143,
-      "num_tokens": 1396964116.0,
-      "reward": 2.1201171875,
-      "reward_std": 0.15555161237716675,
-      "rewards/accuracy_reward/mean": 0.12890625,
-      "rewards/accuracy_reward/std": 0.33542385697364807,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
-      "rewards/tag_count_reward/mean": 0.9990234375,
-      "rewards/tag_count_reward/std": 0.015609703958034515,
+      "loss": 0.0245,
+      "num_tokens": 1396964895.0,
+      "reward": 2.08154296875,
+      "reward_std": 0.1786787509918213,
+      "rewards/accuracy_reward/mean": 0.10546875,
+      "rewards/accuracy_reward/std": 0.3074568510055542,
+      "rewards/format_reward/mean": 0.98046875,
+      "rewards/format_reward/std": 0.1385180652141571,
+      "rewards/tag_count_reward/mean": 0.99560546875,
+      "rewards/tag_count_reward/std": 0.050489041954278946,
       "step": 2914
     },
     {
@@ -87436,25 +87436,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1848.0,
-      "completions/max_terminated_length": 1848.0,
-      "completions/mean_length": 815.22265625,
-      "completions/mean_terminated_length": 815.22265625,
-      "completions/min_length": 123.0,
-      "completions/min_terminated_length": 123.0,
+      "completions/max_length": 1628.0,
+      "completions/max_terminated_length": 1628.0,
+      "completions/mean_length": 808.962890625,
+      "completions/mean_terminated_length": 808.962890625,
+      "completions/min_length": 257.0,
+      "completions/min_terminated_length": 257.0,
       "epoch": 0.9951352735341811,
-      "frac_reward_zero_std": 0.71875,
-      "grad_norm": 0.08489306927332194,
-      "kl": 0.0733642578125,
+      "frac_reward_zero_std": 0.625,
+      "grad_norm": 0.09026605662138586,
+      "kl": 0.0748291015625,
       "learning_rate": 1.8166710921097008e-09,
-      "loss": 0.0036,
-      "num_tokens": 1397459142.0,
-      "reward": 2.07861328125,
-      "reward_std": 0.1113491952419281,
-      "rewards/accuracy_reward/mean": 0.08984375,
-      "rewards/accuracy_reward/std": 0.2862374484539032,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
+      "loss": 0.0103,
+      "num_tokens": 1397456716.0,
+      "reward": 2.08642578125,
+      "reward_std": 0.14233165979385376,
+      "rewards/accuracy_reward/mean": 0.099609375,
+      "rewards/accuracy_reward/std": 0.29977133870124817,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "rewards/tag_count_reward/mean": 0.99853515625,
       "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2915
@@ -87467,26 +87467,26 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1898.0,
-      "completions/mean_length": 832.837890625,
-      "completions/mean_terminated_length": 830.4598999023438,
-      "completions/min_length": 329.0,
-      "completions/min_terminated_length": 329.0,
+      "completions/max_terminated_length": 1839.0,
+      "completions/mean_length": 838.197265625,
+      "completions/mean_terminated_length": 835.8297119140625,
+      "completions/min_length": 341.0,
+      "completions/min_terminated_length": 341.0,
       "epoch": 0.9954766578475719,
       "frac_reward_zero_std": 0.59375,
-      "grad_norm": 0.09716514163001252,
-      "kl": 0.0833740234375,
+      "grad_norm": 0.0990546564513383,
+      "kl": 0.0794677734375,
       "learning_rate": 1.5966894314456416e-09,
-      "loss": 0.0136,
-      "num_tokens": 1397976707.0,
-      "reward": 2.07763671875,
-      "reward_std": 0.15054047107696533,
-      "rewards/accuracy_reward/mean": 0.091796875,
-      "rewards/accuracy_reward/std": 0.289021372795105,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
-      "rewards/tag_count_reward/mean": 0.99560546875,
-      "rewards/tag_count_reward/std": 0.04260620102286339,
+      "loss": 0.0194,
+      "num_tokens": 1397977025.0,
+      "reward": 2.06494140625,
+      "reward_std": 0.1546308547258377,
+      "rewards/accuracy_reward/mean": 0.080078125,
+      "rewards/accuracy_reward/std": 0.271679550409317,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2916
     },
     {
@@ -87496,25 +87496,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1680.0,
-      "completions/max_terminated_length": 1680.0,
-      "completions/mean_length": 727.982421875,
-      "completions/mean_terminated_length": 727.982421875,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/max_length": 1592.0,
+      "completions/max_terminated_length": 1592.0,
+      "completions/mean_length": 735.02734375,
+      "completions/mean_terminated_length": 735.02734375,
+      "completions/min_length": 160.0,
+      "completions/min_terminated_length": 160.0,
       "epoch": 0.9958180421609627,
-      "frac_reward_zero_std": 0.65625,
-      "grad_norm": 0.09975321839818838,
-      "kl": 0.087890625,
+      "frac_reward_zero_std": 0.6875,
+      "grad_norm": 0.09306644198204554,
+      "kl": 0.08642578125,
       "learning_rate": 1.3908986755473142e-09,
-      "loss": 0.0116,
-      "num_tokens": 1398431482.0,
-      "reward": 2.0810546875,
-      "reward_std": 0.1253216564655304,
-      "rewards/accuracy_reward/mean": 0.09375,
-      "rewards/accuracy_reward/std": 0.29176566004753113,
-      "rewards/format_reward/mean": 0.98828125,
-      "rewards/format_reward/std": 0.10772226005792618,
+      "loss": 0.0144,
+      "num_tokens": 1398435407.0,
+      "reward": 2.0849609375,
+      "reward_std": 0.1204758882522583,
+      "rewards/accuracy_reward/mean": 0.099609375,
+      "rewards/accuracy_reward/std": 0.29977133870124817,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
       "rewards/tag_count_reward/mean": 0.9990234375,
       "rewards/tag_count_reward/std": 0.015609703958034515,
       "step": 2917
@@ -87526,27 +87526,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1584.0,
-      "completions/max_terminated_length": 1584.0,
-      "completions/mean_length": 781.779296875,
-      "completions/mean_terminated_length": 781.779296875,
-      "completions/min_length": 312.0,
-      "completions/min_terminated_length": 312.0,
+      "completions/max_length": 1498.0,
+      "completions/max_terminated_length": 1498.0,
+      "completions/mean_length": 774.666015625,
+      "completions/mean_terminated_length": 774.666015625,
+      "completions/min_length": 360.0,
+      "completions/min_terminated_length": 360.0,
       "epoch": 0.9961594264743535,
-      "frac_reward_zero_std": 0.71875,
-      "grad_norm": 0.08017537622311298,
-      "kl": 0.0828857421875,
+      "frac_reward_zero_std": 0.5625,
+      "grad_norm": 0.11717990565862879,
+      "kl": 0.08349609375,
       "learning_rate": 1.199299116497743e-09,
-      "loss": 0.0152,
-      "num_tokens": 1398908953.0,
-      "reward": 2.05615234375,
-      "reward_std": 0.09608898311853409,
-      "rewards/accuracy_reward/mean": 0.0625,
-      "rewards/accuracy_reward/std": 0.2422981858253479,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
-      "rewards/tag_count_reward/mean": 0.99951171875,
-      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "loss": 0.0232,
+      "num_tokens": 1398909236.0,
+      "reward": 2.056640625,
+      "reward_std": 0.13060790300369263,
+      "rewards/accuracy_reward/mean": 0.0703125,
+      "rewards/accuracy_reward/std": 0.25592297315597534,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
+      "rewards/tag_count_reward/mean": 0.998046875,
+      "rewards/tag_count_reward/std": 0.022032126784324646,
       "step": 2918
     },
     {
@@ -87556,27 +87556,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1876.0,
-      "completions/max_terminated_length": 1876.0,
-      "completions/mean_length": 794.0,
-      "completions/mean_terminated_length": 794.0,
-      "completions/min_length": 268.0,
-      "completions/min_terminated_length": 268.0,
+      "completions/max_length": 1840.0,
+      "completions/max_terminated_length": 1840.0,
+      "completions/mean_length": 794.30078125,
+      "completions/mean_terminated_length": 794.30078125,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
       "epoch": 0.9965008107877443,
-      "frac_reward_zero_std": 0.40625,
-      "grad_norm": 0.11566190296325213,
-      "kl": 0.0745849609375,
+      "frac_reward_zero_std": 0.375,
+      "grad_norm": 0.11873912143398956,
+      "kl": 0.0755615234375,
       "learning_rate": 1.0218910262371762e-09,
-      "loss": 0.0017,
-      "num_tokens": 1399397769.0,
-      "reward": 2.1748046875,
-      "reward_std": 0.20263367891311646,
-      "rewards/accuracy_reward/mean": 0.181640625,
-      "rewards/accuracy_reward/std": 0.38592514395713806,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
-      "rewards/tag_count_reward/mean": 0.9990234375,
-      "rewards/tag_count_reward/std": 0.015609703958034515,
+      "loss": 0.0179,
+      "num_tokens": 1399398206.0,
+      "reward": 2.14208984375,
+      "reward_std": 0.23764190077781677,
+      "rewards/accuracy_reward/mean": 0.162109375,
+      "rewards/accuracy_reward/std": 0.3689115643501282,
+      "rewards/format_reward/mean": 0.982421875,
+      "rewards/format_reward/std": 0.13154059648513794,
+      "rewards/tag_count_reward/mean": 0.99755859375,
+      "rewards/tag_count_reward/std": 0.03659820929169655,
       "step": 2919
     },
     {
@@ -87586,25 +87586,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1765.0,
-      "completions/max_terminated_length": 1765.0,
-      "completions/mean_length": 826.298828125,
-      "completions/mean_terminated_length": 826.298828125,
-      "completions/min_length": 250.0,
-      "completions/min_terminated_length": 250.0,
+      "completions/max_length": 2016.0,
+      "completions/max_terminated_length": 2016.0,
+      "completions/mean_length": 833.984375,
+      "completions/mean_terminated_length": 833.984375,
+      "completions/min_length": 292.0,
+      "completions/min_terminated_length": 292.0,
       "epoch": 0.9968421951011351,
       "frac_reward_zero_std": 0.53125,
-      "grad_norm": 0.09891209808935245,
-      "kl": 0.0767822265625,
+      "grad_norm": 0.10545064088363902,
+      "kl": 0.0784912109375,
       "learning_rate": 8.586746565641957e-10,
-      "loss": 0.0099,
-      "num_tokens": 1399917122.0,
-      "reward": 2.13134765625,
-      "reward_std": 0.18901605904102325,
-      "rewards/accuracy_reward/mean": 0.138671875,
-      "rewards/accuracy_reward/std": 0.34594178199768066,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "loss": 0.0003,
+      "num_tokens": 1399921494.0,
+      "reward": 2.12548828125,
+      "reward_std": 0.1864640712738037,
+      "rewards/accuracy_reward/mean": 0.13671875,
+      "rewards/accuracy_reward/std": 0.3438861668109894,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
       "rewards/tag_count_reward/mean": 0.99853515625,
       "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2920
@@ -87616,27 +87616,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1708.0,
-      "completions/max_terminated_length": 1708.0,
-      "completions/mean_length": 753.177734375,
-      "completions/mean_terminated_length": 753.177734375,
-      "completions/min_length": 260.0,
-      "completions/min_terminated_length": 260.0,
+      "completions/max_length": 1390.0,
+      "completions/max_terminated_length": 1390.0,
+      "completions/mean_length": 740.46484375,
+      "completions/mean_terminated_length": 740.46484375,
+      "completions/min_length": 262.0,
+      "completions/min_terminated_length": 262.0,
       "epoch": 0.997183579414526,
       "frac_reward_zero_std": 0.59375,
-      "grad_norm": 0.10619904424152342,
-      "kl": 0.078857421875,
+      "grad_norm": 0.10125262138426178,
+      "kl": 0.0797119140625,
       "learning_rate": 7.096502391346072e-10,
-      "loss": 0.0124,
-      "num_tokens": 1400383885.0,
-      "reward": 2.08154296875,
-      "reward_std": 0.16373801231384277,
-      "rewards/accuracy_reward/mean": 0.08984375,
-      "rewards/accuracy_reward/std": 0.2862374484539032,
+      "loss": 0.0078,
+      "num_tokens": 1400381748.0,
+      "reward": 2.06103515625,
+      "reward_std": 0.15187877416610718,
+      "rewards/accuracy_reward/mean": 0.0703125,
+      "rewards/accuracy_reward/std": 0.25592297315597534,
       "rewards/format_reward/mean": 0.9921875,
       "rewards/format_reward/std": 0.08812850713729858,
-      "rewards/tag_count_reward/mean": 0.99951171875,
-      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2921
     },
     {
@@ -87646,27 +87646,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1729.0,
-      "completions/max_terminated_length": 1729.0,
-      "completions/mean_length": 793.041015625,
-      "completions/mean_terminated_length": 793.041015625,
-      "completions/min_length": 272.0,
-      "completions/min_terminated_length": 272.0,
+      "completions/max_length": 1748.0,
+      "completions/max_terminated_length": 1748.0,
+      "completions/mean_length": 813.0234375,
+      "completions/mean_terminated_length": 813.0234375,
+      "completions/min_length": 291.0,
+      "completions/min_terminated_length": 291.0,
       "epoch": 0.9975249637279167,
-      "frac_reward_zero_std": 0.46875,
-      "grad_norm": 0.10599137915386056,
-      "kl": 0.0753173828125,
+      "frac_reward_zero_std": 0.5625,
+      "grad_norm": 0.09890654551815842,
+      "kl": 0.0748291015625,
       "learning_rate": 5.748179854614399e-10,
-      "loss": 0.0149,
-      "num_tokens": 1400873378.0,
-      "reward": 2.0498046875,
-      "reward_std": 0.1855737864971161,
-      "rewards/accuracy_reward/mean": 0.072265625,
-      "rewards/accuracy_reward/std": 0.2591804563999176,
-      "rewards/format_reward/mean": 0.98046875,
-      "rewards/format_reward/std": 0.1385180652141571,
+      "loss": 0.0172,
+      "num_tokens": 1400881472.0,
+      "reward": 2.0908203125,
+      "reward_std": 0.1693429946899414,
+      "rewards/accuracy_reward/mean": 0.107421875,
+      "rewards/accuracy_reward/std": 0.30995169281959534,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
       "rewards/tag_count_reward/mean": 0.9970703125,
-      "rewards/tag_count_reward/std": 0.026930565014481544,
+      "rewards/tag_count_reward/std": 0.038198307156562805,
       "step": 2922
     },
     {
@@ -87676,27 +87676,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1602.0,
-      "completions/max_terminated_length": 1602.0,
-      "completions/mean_length": 735.794921875,
-      "completions/mean_terminated_length": 735.794921875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_length": 1460.0,
+      "completions/max_terminated_length": 1460.0,
+      "completions/mean_length": 737.50390625,
+      "completions/mean_terminated_length": 737.50390625,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.9978663480413075,
-      "frac_reward_zero_std": 0.6875,
-      "grad_norm": 0.09767874624378871,
-      "kl": 0.079833984375,
+      "frac_reward_zero_std": 0.625,
+      "grad_norm": 0.10499893723997014,
+      "kl": 0.081787109375,
       "learning_rate": 4.541780869138368e-10,
-      "loss": 0.0138,
-      "num_tokens": 1401325961.0,
-      "reward": 2.0703125,
-      "reward_std": 0.11659518629312515,
-      "rewards/accuracy_reward/mean": 0.080078125,
-      "rewards/accuracy_reward/std": 0.271679550409317,
+      "loss": 0.0114,
+      "num_tokens": 1401334930.0,
+      "reward": 2.06103515625,
+      "reward_std": 0.1196913868188858,
+      "rewards/accuracy_reward/mean": 0.072265625,
+      "rewards/accuracy_reward/std": 0.2591804563999176,
       "rewards/format_reward/mean": 0.990234375,
       "rewards/format_reward/std": 0.09843364357948303,
-      "rewards/tag_count_reward/mean": 1.0,
-      "rewards/tag_count_reward/std": 0.0,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2923
     },
     {
@@ -87706,27 +87706,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1751.0,
-      "completions/max_terminated_length": 1751.0,
-      "completions/mean_length": 797.919921875,
-      "completions/mean_terminated_length": 797.919921875,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/max_length": 1754.0,
+      "completions/max_terminated_length": 1754.0,
+      "completions/mean_length": 797.484375,
+      "completions/mean_terminated_length": 797.484375,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.9982077323546983,
-      "frac_reward_zero_std": 0.53125,
-      "grad_norm": 0.11418503362700216,
-      "kl": 0.080322265625,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.10729524502444683,
+      "kl": 0.080078125,
       "learning_rate": 3.477307147192743e-10,
-      "loss": 0.0051,
-      "num_tokens": 1401810896.0,
-      "reward": 2.154296875,
-      "reward_std": 0.19188588857650757,
-      "rewards/accuracy_reward/mean": 0.16796875,
-      "rewards/accuracy_reward/std": 0.374204158782959,
+      "loss": 0.016,
+      "num_tokens": 1401819642.0,
+      "reward": 2.13671875,
+      "reward_std": 0.1730148196220398,
+      "rewards/accuracy_reward/mean": 0.150390625,
+      "rewards/accuracy_reward/std": 0.35780346393585205,
       "rewards/format_reward/mean": 0.98828125,
       "rewards/format_reward/std": 0.10772226005792618,
       "rewards/tag_count_reward/mean": 0.998046875,
-      "rewards/tag_count_reward/std": 0.02701912261545658,
+      "rewards/tag_count_reward/std": 0.022032126784324646,
       "step": 2924
     },
     {
@@ -87736,27 +87736,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1490.0,
-      "completions/max_terminated_length": 1490.0,
-      "completions/mean_length": 795.6015625,
-      "completions/mean_terminated_length": 795.6015625,
-      "completions/min_length": 393.0,
-      "completions/min_terminated_length": 393.0,
+      "completions/max_length": 1937.0,
+      "completions/max_terminated_length": 1937.0,
+      "completions/mean_length": 811.166015625,
+      "completions/mean_terminated_length": 811.166015625,
+      "completions/min_length": 327.0,
+      "completions/min_terminated_length": 327.0,
       "epoch": 0.9985491166680891,
-      "frac_reward_zero_std": 0.40625,
-      "grad_norm": 0.12543432676122176,
-      "kl": 0.0797119140625,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.10537359711068275,
+      "kl": 0.081298828125,
       "learning_rate": 2.5547601995912216e-10,
-      "loss": 0.0177,
-      "num_tokens": 1402303908.0,
-      "reward": 2.14697265625,
-      "reward_std": 0.23765350878238678,
-      "rewards/accuracy_reward/mean": 0.166015625,
-      "rewards/accuracy_reward/std": 0.3724585771560669,
-      "rewards/format_reward/mean": 0.984375,
-      "rewards/format_reward/std": 0.12414088100194931,
-      "rewards/tag_count_reward/mean": 0.99658203125,
-      "rewards/tag_count_reward/std": 0.03972800448536873,
+      "loss": 0.0178,
+      "num_tokens": 1402320623.0,
+      "reward": 2.14892578125,
+      "reward_std": 0.21511411666870117,
+      "rewards/accuracy_reward/mean": 0.162109375,
+      "rewards/accuracy_reward/std": 0.3689115643501282,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.019099153578281403,
       "step": 2925
     },
     {
@@ -87765,28 +87765,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1363.0,
-      "completions/max_terminated_length": 1363.0,
-      "completions/mean_length": 705.349609375,
-      "completions/mean_terminated_length": 704.0626220703125,
-      "completions/min_length": 255.0,
-      "completions/min_terminated_length": 255.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1430.0,
+      "completions/max_terminated_length": 1430.0,
+      "completions/mean_length": 703.416015625,
+      "completions/mean_terminated_length": 703.416015625,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.9988905009814799,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.22959447791895354,
-      "kl": 0.2906494140625,
+      "grad_norm": 0.1300151179262908,
+      "kl": 0.0833740234375,
       "learning_rate": 1.7741413357197367e-10,
-      "loss": 0.0225,
-      "num_tokens": 1402745271.0,
-      "reward": 2.0859375,
-      "reward_std": 0.16681620478630066,
-      "rewards/accuracy_reward/mean": 0.103515625,
-      "rewards/accuracy_reward/std": 0.30492907762527466,
-      "rewards/format_reward/mean": 0.986328125,
-      "rewards/format_reward/std": 0.1162383034825325,
-      "rewards/tag_count_reward/mean": 0.99609375,
-      "rewards/tag_count_reward/std": 0.04930410906672478,
+      "loss": 0.0125,
+      "num_tokens": 1402760996.0,
+      "reward": 2.080078125,
+      "reward_std": 0.17316466569900513,
+      "rewards/accuracy_reward/mean": 0.099609375,
+      "rewards/accuracy_reward/std": 0.29977133870124817,
+      "rewards/format_reward/mean": 0.982421875,
+      "rewards/format_reward/std": 0.13154059648513794,
+      "rewards/tag_count_reward/mean": 0.998046875,
+      "rewards/tag_count_reward/std": 0.022032126784324646,
       "step": 2926
     },
     {
@@ -87796,27 +87796,27 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1917.0,
-      "completions/max_terminated_length": 1917.0,
-      "completions/mean_length": 838.294921875,
-      "completions/mean_terminated_length": 838.294921875,
-      "completions/min_length": 339.0,
-      "completions/min_terminated_length": 339.0,
+      "completions/max_length": 1723.0,
+      "completions/max_terminated_length": 1723.0,
+      "completions/mean_length": 840.955078125,
+      "completions/mean_terminated_length": 840.955078125,
+      "completions/min_length": 352.0,
+      "completions/min_terminated_length": 352.0,
       "epoch": 0.9992318852948707,
-      "frac_reward_zero_std": 0.6875,
-      "grad_norm": 0.09099736855996839,
+      "frac_reward_zero_std": 0.5625,
+      "grad_norm": 0.10865170665818756,
       "kl": 0.0772705078125,
       "learning_rate": 1.1354516635364577e-10,
-      "loss": 0.0035,
-      "num_tokens": 1403254030.0,
-      "reward": 2.048828125,
-      "reward_std": 0.09261970221996307,
-      "rewards/accuracy_reward/mean": 0.05859375,
-      "rewards/accuracy_reward/std": 0.23509246110916138,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
-      "rewards/tag_count_reward/mean": 0.998046875,
-      "rewards/tag_count_reward/std": 0.022032126784324646,
+      "loss": 0.0192,
+      "num_tokens": 1403271117.0,
+      "reward": 2.0263671875,
+      "reward_std": 0.13820995390415192,
+      "rewards/accuracy_reward/mean": 0.048828125,
+      "rewards/accuracy_reward/std": 0.2157193273305893,
+      "rewards/format_reward/mean": 0.98046875,
+      "rewards/format_reward/std": 0.1385180652141571,
+      "rewards/tag_count_reward/mean": 0.9970703125,
+      "rewards/tag_count_reward/std": 0.026930565014481544,
       "step": 2927
     },
     {
@@ -87825,28 +87825,28 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1740.0,
-      "completions/mean_length": 769.6796875,
-      "completions/mean_terminated_length": 767.1781005859375,
-      "completions/min_length": 271.0,
-      "completions/min_terminated_length": 271.0,
+      "completions/max_terminated_length": 1749.0,
+      "completions/mean_length": 767.625,
+      "completions/mean_terminated_length": 760.07861328125,
+      "completions/min_length": 279.0,
+      "completions/min_terminated_length": 279.0,
       "epoch": 0.9995732696082615,
-      "frac_reward_zero_std": 0.40625,
-      "grad_norm": 0.12120526316644997,
-      "kl": 0.077880859375,
+      "frac_reward_zero_std": 0.53125,
+      "grad_norm": 0.1098002562236282,
+      "kl": 0.0780029296875,
       "learning_rate": 6.386920895384841e-11,
-      "loss": 0.032,
-      "num_tokens": 1403723962.0,
-      "reward": 2.09521484375,
-      "reward_std": 0.20090395212173462,
-      "rewards/accuracy_reward/mean": 0.12298387289047241,
-      "rewards/accuracy_reward/std": 0.32875028252601624,
+      "loss": 0.0139,
+      "num_tokens": 1403739997.0,
+      "reward": 2.08935546875,
+      "reward_std": 0.16179436445236206,
+      "rewards/accuracy_reward/mean": 0.11895161122083664,
+      "rewards/accuracy_reward/std": 0.3240584135055542,
       "rewards/format_reward/mean": 0.98046875,
       "rewards/format_reward/std": 0.1385180652141571,
-      "rewards/tag_count_reward/mean": 0.99560546875,
-      "rewards/tag_count_reward/std": 0.050489041954278946,
+      "rewards/tag_count_reward/mean": 0.99365234375,
+      "rewards/tag_count_reward/std": 0.05493048578500748,
       "step": 2928
     },
     {
@@ -87856,42 +87856,42 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1504.0,
-      "completions/max_terminated_length": 1504.0,
-      "completions/mean_length": 782.98828125,
-      "completions/mean_terminated_length": 782.98828125,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "completions/max_length": 1440.0,
+      "completions/max_terminated_length": 1440.0,
+      "completions/mean_length": 793.716796875,
+      "completions/mean_terminated_length": 793.716796875,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.9999146539216524,
-      "frac_reward_zero_std": 0.5625,
-      "grad_norm": 0.10721380774650786,
-      "kl": 0.0792236328125,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.11492651391052233,
+      "kl": 0.0770263671875,
       "learning_rate": 2.838633187729478e-11,
-      "loss": 0.0172,
-      "num_tokens": 1404212612.0,
-      "reward": 2.09619140625,
-      "reward_std": 0.18561691045761108,
-      "rewards/accuracy_reward/mean": 0.12109375,
-      "rewards/accuracy_reward/std": 0.3265552520751953,
-      "rewards/format_reward/mean": 0.978515625,
-      "rewards/format_reward/std": 0.14513419568538666,
-      "rewards/tag_count_reward/mean": 0.99658203125,
-      "rewards/tag_count_reward/std": 0.029059575870633125,
+      "loss": 0.0089,
+      "num_tokens": 1404234140.0,
+      "reward": 2.09912109375,
+      "reward_std": 0.16686061024665833,
+      "rewards/accuracy_reward/mean": 0.115234375,
+      "rewards/accuracy_reward/std": 0.3196168541908264,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
+      "rewards/tag_count_reward/mean": 0.99755859375,
+      "rewards/tag_count_reward/std": 0.024608410894870758,
       "step": 2929
     },
     {
       "epoch": 0.9999146539216524,
       "step": 2929,
       "total_flos": 0.0,
-      "train_loss": 0.00012006735727737873,
-      "train_runtime": 1294.972,
-      "train_samples_per_second": 72.382,
-      "train_steps_per_second": 2.263
+      "train_loss": 0.00015187089497744325,
+      "train_runtime": 1302.9313,
+      "train_samples_per_second": 71.94,
+      "train_steps_per_second": 2.249
     }
   ],
   "logging_steps": 1,
   "max_steps": 2930,
-  "num_input_tokens_seen": 1404212612,
+  "num_input_tokens_seen": 1404234140,
   "num_train_epochs": 1,
   "save_steps": 100,
   "stateful_callbacks": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce4599805e4a46a67aa4e3afb0862b0625c1ee9e61ed3b82c608ce522b65f009
+oid sha256:df64e4c9b10422927491885e2146cbeed575ec0206ed018c170dc3bb7d57cf3c
 size 8504