Update model

eval_results.json

@@ -1,4 +1,4 @@
 {
-    "eval_accuracy": 0.7494515776634216,
-    "eval_loss": 0.8329918384552002
+    "eval_accuracy": 0.7501800656318665,
+    "eval_loss": 0.8296337723731995
 }

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6adf5dfa55da480dfa0a8c47f53d1cb4cee581f3f22e017c7591c3717ef2416
+oid sha256:2f5a8b116ac6d7591d0dfac6bbdb45dfd26d32bce65d4e385d2a346ad7957fda
 size 1198558445

run_byt5_small-mc4.sh

@@ -12,7 +12,6 @@ export MODEL_PATH="${HOME}/data/${HF_PROJECT}" # Path to the model
 mkdir -p "${MODEL_PATH}"
 
 python ../train/run_t5_mlm_flax_pmap.py \
-    --resume_from_checkpoint="${MODEL_PATH}" \
     --output_dir="${MODEL_PATH}" \
     --model_type="t5" \
     --config_name="${CONFIG_NAME}" \
@@ -26,7 +25,8 @@ python ../train/run_t5_mlm_flax_pmap.py \
     --per_device_eval_batch_size="16" \
     --gradient_accumulation_steps="8" \
     --mean_noise_span_length="20" \
-    --dtype="float32" \
+    --dtype="bfloat16" \
+    --z_loss="1e-4" \
     --optim="adafactor" \
     --learning_rate="0.0034" \
     --lr_decay="linear" \
@@ -45,6 +45,7 @@ python ../train/run_t5_mlm_flax_pmap.py \
 #    --max_eval_samples="1000" \
 
 #    --model_name_or_path="${MODEL_PATH}" \
+#    --resume_from_checkpoint="${MODEL_PATH}" \
 
 #     \