diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..4225adc83ff31902091dc9173a805554a2d2c696 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,132 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__0_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__1_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__2_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__3_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__4_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__5_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__6_9.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_1.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_10.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_11.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_12.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_13.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_14.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_15.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_2.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_3.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_4.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_5.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_6.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_7.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_8.distcp filter=lfs diff=lfs merge=lfs -text
+model_and_optim/__7_9.distcp filter=lfs diff=lfs merge=lfs -text
diff --git a/.metadata.json b/.metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..9009f04abdf177415721bc1c689b7f71d47f5380
--- /dev/null
+++ b/.metadata.json
@@ -0,0 +1 @@
+{"version": "2.0.0"}
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..ebb966a107bca442c9767a94afd24a2716e83125
--- /dev/null
+++ b/config.json
@@ -0,0 +1 @@
+{"run_name": "caia_olmo2_1b_d3", "launch": {"name": "caia_olmo2_1b_d3-train-95a8ba94", "cmd": ["src/scripts/train/OLMoE-2x1B-anneal.py", "train", "caia_olmo2_1b_d3", "ai2/jupiter-cirrascale-2", "--launch.num_nodes=1", "--launch.workspace=OLMo-modular", "--launch.priority=urgent", "--launch.beaker_image=petew/olmo-core-tch260cu124", "--trainer.callbacks.wandb.enabled=True", "--trainer.callbacks.comet.enabled=False", "--trainer.max_duration.value=200_000_000", "--trainer.max_duration.unit=tokens", "--dataset.mix_base_dir=/weka/oe-training-default/ai2-llm/preprocessed", "--dataset.mix=caia", "--trainer.load_path=/weka/oe-training-default/ai2-llm/checkpoints/swj0419/model/merge/olmo2_1b/moe2_random", "--train_module.dp_config.num_replicas=4", "--train_module.scheduler.warmup_steps=2000", "--train_module.optim.lr=4e-5", "--model.block.feed_forward_moe.router.top_k=2", "--train_module.rank_microbatch_size=4096", "--train_module.ep_config.degree=2"], "budget": "ai2/oe-base", "task_name": "train", "workspace": "OLMo-modular", "setup_steps": ["conda install gh --channel conda-forge", "gh repo clone \"$REPO_URL\" .", "git checkout \"$GIT_REF\"", "git submodule update --init --recursive", "conda shell.bash activate base", "pip install -e '.[dev,beaker,wandb,train]'", "pip freeze", "mkdir -p ~/.aws", "printenv AWS_CONFIG > ~/.aws/config", "printenv AWS_CREDENTIALS > ~/.aws/credentials"], "beaker_image": "petew/olmo-core-tch260cu124", "num_nodes": 1, "num_gpus": 8, "shared_memory": "10GiB", "clusters": ["ai2/jupiter-cirrascale-2"], "shared_filesystem": true, "priority": "urgent", "preemptible": true, "env_vars": [{"name": "NCCL_DEBUG", "value": "WARN", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvVar"}, {"name": "CUDA_LAUNCH_BLOCKING", "value": "0", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvVar"}], "env_secrets": [{"name": "GITHUB_TOKEN", "secret": "weijias_GITHUB_TOKEN", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "BEAKER_TOKEN", "secret": "weijias_BEAKER_TOKEN", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "WANDB_API_KEY", "secret": "weijias_WANDB_API_KEY", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "COMET_API_KEY", "secret": "weijias_COMET_API_KEY", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "AWS_CONFIG", "secret": "weijias_AWS_CONFIG", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "AWS_CREDENTIALS", "secret": "weijias_AWS_CREDENTIALS", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "R2_ENDPOINT_URL", "secret": "R2_ENDPOINT_URL", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "WEKA_ENDPOINT_URL", "secret": "WEKA_ENDPOINT_URL", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}, {"name": "SLACK_WEBHOOK_URL", "secret": "SLACK_WEBHOOK_URL", "_CLASS_": "olmo_core.launch.beaker.BeakerEnvSecret"}], "nfs": false, "weka_buckets": [{"bucket": "oe-training-default", "mount": "/weka/oe-training-default", "_CLASS_": "olmo_core.launch.beaker.BeakerWekaBucket"}], "allow_dirty": false, "_CLASS_": "olmo_core.launch.beaker.BeakerLaunchConfig"}, "model": {"d_model": 2048, "vocab_size": 100352, "n_layers": 16, "block": {"attention": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "use_flash": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.attention.AttentionConfig"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward_moe": {"name": "default", "num_experts": 2, "hidden_size": 8192, "capacity_factor": 1.2, "router": {"name": "default", "top_k": 2, "uniform_expert_assignment": false, "_CLASS_": "olmo_core.nn.moe.router.MoERouterConfig"}, "lb_loss_weight": 0.0, "z_loss_weight": 0.001, "dtype": "float32", "_CLASS_": "olmo_core.nn.moe.moe.MoEConfig"}, "name": "moe_reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "moe", "dtype": "float32", "init_method": "normal", "init_seed": 0, "freeze_params": ["embeddings.*", "blocks.*.attention*", "blocks.*.feed_forward_norm.*", "lm_head.*"], "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "name": "fsl", "sequence_length": 4096, "max_target_sequence_length": 8192, "mix": "caia", "mix_base_dir": "/weka/oe-training-default/ai2-llm", "include_instance_metadata": true, "generate_doc_lengths": false, "expand_glob": false, "work_dir": "/weka/oe-training-default/ai2-llm/checkpoints/weijias/caia_olmo2_1b_d3/dataset-cache", "_CLASS_": "olmo_core.data.numpy_dataset.NumpyDatasetConfig"}, "data_loader": {"global_batch_size": 4194304, "seed": 34521, "num_workers": 4, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig"}, "train_module": {"rank_microbatch_size": 4096, "max_sequence_length": 4096, "optim": {"compile": false, "fixed_fields": ["initial_lr"], "lr": 4e-05, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.0, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "warmup_steps": 2000, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "num_replicas": 4, "wrapping_strategy": "fine_grained", "prefetch_factor": 0, "_CLASS_": "olmo_core.train.train_module.transformer.TransformerDataParallelConfig"}, "ep_config": {"degree": 2, "_CLASS_": "olmo_core.train.train_module.transformer.TransformerExpertParallelConfig"}, "z_loss_multiplier": 1e-05, "label_ignore_index": -100, "_CLASS_": "olmo_modular.train.train_module.transfomer.FreezeTransformerTrainModuleConfig"}, "trainer": {"save_folder": "/weka/oe-training-default/ai2-llm/checkpoints/weijias/caia_olmo2_1b_d3", "load_path": "/weka/oe-training-default/ai2-llm/checkpoints/swj0419/model/merge/olmo2_1b/moe2_random", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": true, "max_duration": {"value": 200000000, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 1, "metrics_collect_interval": 10, "callbacks": {"downstream_evaluator": {"tasks": ["piqa"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 500, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "enabled": true, "_CLASS_": "olmo_modular.eval.evaluator_callback.DownstreamEvaluatorUpdatedCallbackConfig"}, "checkpointer": {"save_interval": 10000, "ephemeral_save_interval": 250, "save_async": true, "remove": "ephemeral_only", "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "comet": {"enabled": false, "name": "caia_olmo2_1b_d3", "project": "OLMo-modular", "workspace": "ai2", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "notifications": "none", "failure_tag": "failed", "_CLASS_": "olmo_core.train.callbacks.comet.CometCallback"}, "wandb": {"enabled": true, "name": "caia_olmo2_1b_d3", "project": "OLMo-modular", "entity": "ai2-llm", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "profiler": {"skip_first": 0, "wait": 1, "warmup": 5, "active": 3, "repeat": 1, "enabled": false, "_CLASS_": "olmo_core.train.callbacks.profiler.ProfilerCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "slack_notifier": {"name": "caia_olmo2_1b_d3", "notifications": "end_only", "enabled": false, "_CLASS_": "olmo_core.train.callbacks.slack_notifier.SlackNotifierCallback"}, "beaker": {"enabled": true, "_CLASS_": "olmo_core.train.callbacks.beaker.BeakerCallback"}, "gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "name": "padded_fsl", "sequence_length": 4096, "mix": "v3-small-ppl-validation", "mix_base_dir": "/weka/oe-training-default/ai2-llm", "include_instance_metadata": true, "generate_doc_lengths": false, "expand_glob": false, "work_dir": "/weka/oe-training-default/ai2-llm/checkpoints/weijias/dataset-cache", "_CLASS_": "olmo_core.data.numpy_dataset.NumpyDatasetConfig"}, "eval_interval": 1000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}}, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 12536, "_CLASS_": "olmo_modular.internal.freeze_experiment.ExperimentConfig"}
\ No newline at end of file
diff --git a/data_paths.txt b/data_paths.txt
new file mode 100644
index 0000000000000000000000000000000000000000..996f3c4cfe3cf1bc5e912c2a07c648fcc1879ff2
--- /dev/null
+++ b/data_paths.txt
@@ -0,0 +1,2 @@
+/weka/oe-training-default/ai2-llm/caia_olmo2/combine_tokenized/part-0-00000.npy
+/weka/oe-training-default/ai2-llm/caia_olmo2/combine_tokenized/part-1-00000.npy
\ No newline at end of file
diff --git a/model_and_optim/.metadata b/model_and_optim/.metadata
new file mode 100644
index 0000000000000000000000000000000000000000..7c13eadc0a1f25cd510c4bd91620a3aec5b010af
--- /dev/null
+++ b/model_and_optim/.metadata
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c25f5639792ad24246cb78b1ba2920fa504ea59100266013bddaeb02692e5869
+size 616662
diff --git a/model_and_optim/__0_0.distcp b/model_and_optim/__0_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..485b7947703e8980fb670e59661a266294d8998f
--- /dev/null
+++ b/model_and_optim/__0_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61e7bb41da152f4f948b5f82b5f92041ee4435e242c7eed7a3996a6b253c417
+size 545314056
diff --git a/model_and_optim/__0_1.distcp b/model_and_optim/__0_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..3f8bc9ca50c9b8a413e56da6216656b81dbb7ba3
--- /dev/null
+++ b/model_and_optim/__0_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a75c7c4aa1231c20ede8227af30b38a6fe86d2d3103f729520935745ad5da9a1
+size 545314056
diff --git a/model_and_optim/__0_10.distcp b/model_and_optim/__0_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..72518778209bd9f617ee96643bed63742fb911a7
--- /dev/null
+++ b/model_and_optim/__0_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af457da84aafb48dc8b4a798a1893334b446276549c31807d8f0dea7f78b31ca
+size 184666668
diff --git a/model_and_optim/__0_11.distcp b/model_and_optim/__0_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f6ba600bf86ddb7116806f17217fb878cce932e0
--- /dev/null
+++ b/model_and_optim/__0_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3946ea81f7c32e03f2c8597b052ac370a3b32005032a71fbee38a86accedb767
+size 184666668
diff --git a/model_and_optim/__0_12.distcp b/model_and_optim/__0_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5a8acda75371c22738f4e6e21db94ef501aa0abb
--- /dev/null
+++ b/model_and_optim/__0_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0febd277fa5a17bcb0fa905d5857e7791b1cb73adb483371e2bbb2cd311957a6
+size 184666668
diff --git a/model_and_optim/__0_13.distcp b/model_and_optim/__0_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..93f70dd8cbfb30ea84c8725cbd4112e2e6d60e72
--- /dev/null
+++ b/model_and_optim/__0_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec449b3c76a0b6292152d7d3d450a9be541240bfbbac3e55bc599ffe47d44f8b
+size 184666668
diff --git a/model_and_optim/__0_14.distcp b/model_and_optim/__0_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..a9f4bec741c9af3c4fc549f2459d2186bca856f7
--- /dev/null
+++ b/model_and_optim/__0_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fdcd263934ff2e9860dce95862a41df865989b914a3f05a449f2d9b3c774b81
+size 235006576
diff --git a/model_and_optim/__0_15.distcp b/model_and_optim/__0_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..c7341ebe914e537cb38db68a457b6101bf1d3f94
--- /dev/null
+++ b/model_and_optim/__0_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a84ae79e526b03fffe0b28886e07f38e1be925dd487a6ba03179461a00567da4
+size 235006576
diff --git a/model_and_optim/__0_2.distcp b/model_and_optim/__0_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..afd75c11b8eb66e7e9ef7d33f9a9b32783c6b287
--- /dev/null
+++ b/model_and_optim/__0_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a50244048048de753ab58f829e8b366ba5d8a8cbc0d535dbc430851c95d97f6
+size 184660144
diff --git a/model_and_optim/__0_3.distcp b/model_and_optim/__0_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..6ce81265cff0e96bf7b29deebe8a024aa74e8e74
--- /dev/null
+++ b/model_and_optim/__0_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38a3e7ae6e4015d4f157bdbe2babe2fa9c296a2e9a5970919994235734c23b5b
+size 184660144
diff --git a/model_and_optim/__0_4.distcp b/model_and_optim/__0_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..d4e9d5ba2c56c4ef63d357fab0e7b3f23feddf35
--- /dev/null
+++ b/model_and_optim/__0_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7caa231a1072b75e5e34c631e1e160e7a77b8c7660c17b87edfeb5edb86d0ed
+size 184660144
diff --git a/model_and_optim/__0_5.distcp b/model_and_optim/__0_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..bc6e467a29a23962a37d771a446b235e4369d92d
--- /dev/null
+++ b/model_and_optim/__0_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20598cf5e65d5cfe49aa39708d0245a856e41ac18cfb2a1c95b655ffd3a78c52
+size 184660144
diff --git a/model_and_optim/__0_6.distcp b/model_and_optim/__0_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f6b8825040ab26efb1f59d1cdded152ec70df88c
--- /dev/null
+++ b/model_and_optim/__0_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0a5e29880353df214007ef10ffd4146ac4a6f84117feaf22d169f7508dd80f5
+size 184660144
diff --git a/model_and_optim/__0_7.distcp b/model_and_optim/__0_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ad0e6609d8aa00d2e73358b458886df54a0ca89b
--- /dev/null
+++ b/model_and_optim/__0_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0acf275433ddda4c1dc73a8fbf3cb19b6a2c5e40093d27efad768532f3c8ddf3
+size 184660144
diff --git a/model_and_optim/__0_8.distcp b/model_and_optim/__0_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..6b8870600b0e14df3a61f3110fb30a664d20c4eb
--- /dev/null
+++ b/model_and_optim/__0_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:246dcfaf8a4ac8c8209a04b7fb625157fc26cd9f89d2791b63841687e92a0151
+size 184661324
diff --git a/model_and_optim/__0_9.distcp b/model_and_optim/__0_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..672958cb6d9e62db861c6dd3c6356ebc690b9544
--- /dev/null
+++ b/model_and_optim/__0_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73ab491220f462d7171250d3ff9532ec073e7512941f66927ce4f18ee6df8330
+size 184667848
diff --git a/model_and_optim/__1_0.distcp b/model_and_optim/__1_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f9da70a51e453bdb7b9698509bfd0f7f8658c951
--- /dev/null
+++ b/model_and_optim/__1_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:727a8e4eb5e0548618ba92c313fb5b16a35ebb20477516b5cbd8e7d7e7d7de10
+size 545276040
diff --git a/model_and_optim/__1_1.distcp b/model_and_optim/__1_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ef241a1386112b992e984ee9c92bb4bef4e9308a
--- /dev/null
+++ b/model_and_optim/__1_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1d910b74fba7d81ddd887b4ef43e430a69f236e7d9a40415b07484668ce5d39
+size 545276040
diff --git a/model_and_optim/__1_10.distcp b/model_and_optim/__1_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..289be77588ea9e9d23364b130a1bce71068af018
--- /dev/null
+++ b/model_and_optim/__1_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb0f518e97d119a9c0bde3c96bcfef8d807e27830bcd3402c5454f4f5523c858
+size 184618032
diff --git a/model_and_optim/__1_11.distcp b/model_and_optim/__1_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..0d8b9e8e1984d992047886879313ad4142c82bf6
--- /dev/null
+++ b/model_and_optim/__1_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc928c2e4ca56cdbcf80ac36da987fa719ae5da64760fb31c3ff135b8241b44b
+size 184618032
diff --git a/model_and_optim/__1_12.distcp b/model_and_optim/__1_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8579a2567fbcf15c80b28b0d471667c92d2ae8b5
--- /dev/null
+++ b/model_and_optim/__1_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73f1f684e4e1c957c70ecfa7411ec25de96d85f2ac9d4a7945f696a8c4775c2e
+size 184618032
diff --git a/model_and_optim/__1_13.distcp b/model_and_optim/__1_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..1b15f58097aa248e89cae3a1b3901971f8a5609e
--- /dev/null
+++ b/model_and_optim/__1_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2244bc3d52fec82f8229ff9f45cd29c42c97eab40d94ae59a6041da518afa310
+size 184618032
diff --git a/model_and_optim/__1_14.distcp b/model_and_optim/__1_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..53b16fa7c389115b4269868349c30d6675661cf2
--- /dev/null
+++ b/model_and_optim/__1_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3bc1fab016d42eb40630d23e3e3ba3b5b4948410f46445ec22294287c7f07e3
+size 234957940
diff --git a/model_and_optim/__1_15.distcp b/model_and_optim/__1_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..a8e254205cb6ad387e7fd2ec0606bc53d30e582b
--- /dev/null
+++ b/model_and_optim/__1_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d2e18d5bd5832c507789e8771ffff5b91e181efecd5c855c22a7868d96bad97
+size 234957940
diff --git a/model_and_optim/__1_2.distcp b/model_and_optim/__1_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..51b75394e9e954d3afbf7924f839150a0425a6a0
--- /dev/null
+++ b/model_and_optim/__1_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34e289e5dca974468e39067b6fc97e698ba9441cc500537d0fa9b77b01bfec86
+size 184622128
diff --git a/model_and_optim/__1_3.distcp b/model_and_optim/__1_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..294975f27d61befa58172b4f9d83eb1c09358ccb
--- /dev/null
+++ b/model_and_optim/__1_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f26a272b661a76c8837848843031cd81bcfd17494dadf86d7ba5de5382c7ece
+size 184622128
diff --git a/model_and_optim/__1_4.distcp b/model_and_optim/__1_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..943306974c39d1324eaa0c1a46db1f7d82b418ad
--- /dev/null
+++ b/model_and_optim/__1_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:019559b5433399fd8108a958951bdd315492e3b956e5e4b2023bc05eb8bcb9d8
+size 184622128
diff --git a/model_and_optim/__1_5.distcp b/model_and_optim/__1_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..01f1a26f946501955141644ec128f53bad452708
--- /dev/null
+++ b/model_and_optim/__1_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:683a1a58a844675429ff1de10aaaa75a2c79b7e2c53198329d172aa5309a6cf2
+size 184622128
diff --git a/model_and_optim/__1_6.distcp b/model_and_optim/__1_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..7ab7959a74fd6ca05f2f144530166429dbf8b22b
--- /dev/null
+++ b/model_and_optim/__1_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09b6e0f15b468a7acfb5c0a06720d53997dfce700c1b91d0488d1cc43885a503
+size 184622128
diff --git a/model_and_optim/__1_7.distcp b/model_and_optim/__1_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..729ef15bd951e60740ab239502000eaadd58a1b6
--- /dev/null
+++ b/model_and_optim/__1_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cf048ff61144716923c36c9a69d7abcbea3733c6e1cb500f9e89238d8f974ec
+size 184622128
diff --git a/model_and_optim/__1_8.distcp b/model_and_optim/__1_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..937a236925a37d002b00276ac375575a30131fab
--- /dev/null
+++ b/model_and_optim/__1_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e3bb846150594185392d790dfbdeb61d40011a304818aa9e7a610f933b3877b
+size 184623308
diff --git a/model_and_optim/__1_9.distcp b/model_and_optim/__1_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..dcb87a51a214cb02abb6b26bf14e700f86759082
--- /dev/null
+++ b/model_and_optim/__1_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:595cdfc5038ec6b9754ebca4f704559680391ab60a409779a1d176504f4330f4
+size 184618032
diff --git a/model_and_optim/__2_0.distcp b/model_and_optim/__2_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..73fa5b50c1d630769137e1d0fac33eb31f4507d2
--- /dev/null
+++ b/model_and_optim/__2_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7328b4eb7fdc6050c7b37ab651d85f735d4315aa1b5f2a9c35ae7db9edb88f38
+size 151005564
diff --git a/model_and_optim/__2_1.distcp b/model_and_optim/__2_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..9f0f7db2484b2555c146d169605e8a19ee370456
--- /dev/null
+++ b/model_and_optim/__2_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e3f749f7b34918f8156c0c503cf23a71b7502bfc1d89d351d99a696948a0dc1
+size 151005564
diff --git a/model_and_optim/__2_10.distcp b/model_and_optim/__2_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ee634e3ebbc28f3a3575412f39afabde5a205e64
--- /dev/null
+++ b/model_and_optim/__2_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d564b0fb22af7caf17399fe6a9a64f2ac19efdf2655af10f593c6dc066189b71
+size 151005564
diff --git a/model_and_optim/__2_11.distcp b/model_and_optim/__2_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..b44f6927d16f2c95fdaf14734f5e3ec8024f8aa8
--- /dev/null
+++ b/model_and_optim/__2_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d6b2ad31fd84ac2905aa293c33467f57ae7c531d6eb1de751c9e146d75d0b04
+size 151005564
diff --git a/model_and_optim/__2_12.distcp b/model_and_optim/__2_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..868da24a563812e0160a6c38cb3515b85e794646
--- /dev/null
+++ b/model_and_optim/__2_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:395aa7f87518b3fe6cf251917cc0001f0e4d83948f370ba7ec4198ccfff04d1f
+size 151005564
diff --git a/model_and_optim/__2_13.distcp b/model_and_optim/__2_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..4247bd4019cf894db80bb69a8e18e32097a2df20
--- /dev/null
+++ b/model_and_optim/__2_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a740ef150903500ca40e37e57521dd1fad4b76b0601f51bda4ce4562e1c3dee2
+size 151005564
diff --git a/model_and_optim/__2_14.distcp b/model_and_optim/__2_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..66822b066be3b1c9e871a9fb10890c89642718a0
--- /dev/null
+++ b/model_and_optim/__2_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32444cc5580db282db3ca17eceaa46cff65a943d31396ebeaafbf47b7fa9c308
+size 151005564
diff --git a/model_and_optim/__2_15.distcp b/model_and_optim/__2_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f7c8716ded86c240d0f996469a9979f392cf0104
--- /dev/null
+++ b/model_and_optim/__2_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1706d91f254a427d5e635aa9749eceb50ff41db3f27c4109e0e5cc48bb6386b6
+size 151005564
diff --git a/model_and_optim/__2_2.distcp b/model_and_optim/__2_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..b7a0f4d9186af4563f6eab54c0ed1541e94283e7
--- /dev/null
+++ b/model_and_optim/__2_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7250b55553443793b734bdcf443da683be2b8e42c00c9fe407e45c507c82f642
+size 151005564
diff --git a/model_and_optim/__2_3.distcp b/model_and_optim/__2_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..93f79ade61e6e376729e71b5e2770a9681d34298
--- /dev/null
+++ b/model_and_optim/__2_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b44ad99fa2f99a8a6838c085c19b5bef7a3b723cd80c35ee764b09db7bb9ff17
+size 151005564
diff --git a/model_and_optim/__2_4.distcp b/model_and_optim/__2_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..013b5b70e7a52c9af9f998d9438cf607090bbac7
--- /dev/null
+++ b/model_and_optim/__2_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0614ca129c14258e6d73ecd31faf26fb4b6ecf8e4d1fbd0eed63a0c8f5e31962
+size 151005564
diff --git a/model_and_optim/__2_5.distcp b/model_and_optim/__2_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..4132a8458498cfefbecbf5aef13b813d64f1458f
--- /dev/null
+++ b/model_and_optim/__2_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:406971458d744bfe10bed6068409a36c87242c2e30c64552dee52098739a7bd6
+size 151005564
diff --git a/model_and_optim/__2_6.distcp b/model_and_optim/__2_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..454ae2be23d74a8c0ffada9480b9d206b2b7f4d5
--- /dev/null
+++ b/model_and_optim/__2_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90d551a440c35b04a61c20f4aac8983b567f86215ed7e05905dabfc35c74f599
+size 151005564
diff --git a/model_and_optim/__2_7.distcp b/model_and_optim/__2_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5becc8de36ec33a5fe9ffaeabcb8bc603c7b3af0
--- /dev/null
+++ b/model_and_optim/__2_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cd0c8534c83050192ae77ff4cc2c257df92f0d5e8312ba4f1095b657523ffcd
+size 151005564
diff --git a/model_and_optim/__2_8.distcp b/model_and_optim/__2_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8a905ef741935ceda94d69ef72f7dbf1c962cbe3
--- /dev/null
+++ b/model_and_optim/__2_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78b0ce01852a9532c4a0f744120f97c71949a55ff2060d4b3eb61dfbe65b6b94
+size 151005564
diff --git a/model_and_optim/__2_9.distcp b/model_and_optim/__2_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8337187b19db5421703ec95f1fce6bc215ad0947
--- /dev/null
+++ b/model_and_optim/__2_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5ef3a573620a16b63cc9cfe18a345cd282c4b691387970a4d868dbe0eea5f09
+size 151005564
diff --git a/model_and_optim/__3_0.distcp b/model_and_optim/__3_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..816c1fd4265795916ed895469344d796771a2837
--- /dev/null
+++ b/model_and_optim/__3_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bdb4a0b8ec0cea87e139fef2f5e29bf2fc369df806d3981af558488c4d77c52
+size 151005564
diff --git a/model_and_optim/__3_1.distcp b/model_and_optim/__3_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..eea9cc6995ce12e68503c3a3cf399959cd5d6157
--- /dev/null
+++ b/model_and_optim/__3_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69acb17786ccd43f3cf6f7dbbde867410f125c67fb12955c9bd667e9a40eb767
+size 151005564
diff --git a/model_and_optim/__3_10.distcp b/model_and_optim/__3_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..c076c5167c749dbcf87f13a71f6b570880b43f0a
--- /dev/null
+++ b/model_and_optim/__3_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debc55f49ab7ddd927f463a5fd5cbe224d50f00aa1e901cec279ff6c3ee9a998
+size 151005564
diff --git a/model_and_optim/__3_11.distcp b/model_and_optim/__3_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..7acc0ffabfc9b6d360ff8a08a34831385967ee3e
--- /dev/null
+++ b/model_and_optim/__3_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab86e0bfbaa8cc041f764453c0f029f44117af03850448c4e650c0f2ac0a6188
+size 151005564
diff --git a/model_and_optim/__3_12.distcp b/model_and_optim/__3_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8b927daae498bc2ea1f0e13a98326d38b415cd40
--- /dev/null
+++ b/model_and_optim/__3_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4cb52268d7d2449f0cf496474c65abe22fbaf7918a17539215570759ea08eff
+size 151005564
diff --git a/model_and_optim/__3_13.distcp b/model_and_optim/__3_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..e5d8e7a7e4181e4b8f5c1ae9b6485115ef35df45
--- /dev/null
+++ b/model_and_optim/__3_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fb1b32c6563e643059f6d9fc6ca62d3cc2696ea262fc767a2802b148b1b9516
+size 151005564
diff --git a/model_and_optim/__3_14.distcp b/model_and_optim/__3_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..cf5adff25ef1265631209a02d972caf31bf5250d
--- /dev/null
+++ b/model_and_optim/__3_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8b07a6a18d319b0a3b418622d035cfaca1944f39259b970499d0172058b6875
+size 151005564
diff --git a/model_and_optim/__3_15.distcp b/model_and_optim/__3_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5b66d65f3bf6fe3b93b5fa25c4d89a6617b282e4
--- /dev/null
+++ b/model_and_optim/__3_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb41b88263e5aafffdc941e16eea4883974c11f2e4880d99ab5b09758fa5c33a
+size 151005564
diff --git a/model_and_optim/__3_2.distcp b/model_and_optim/__3_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..1387fa47390594951c7b93c28e123bf1e3993504
--- /dev/null
+++ b/model_and_optim/__3_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0019f488dfd90196220ed123802b5170b995469c156d6d2f2e59ebf8f94149f0
+size 151005564
diff --git a/model_and_optim/__3_3.distcp b/model_and_optim/__3_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..016e4c02513a4035b1015c334ada8c217a9c582f
--- /dev/null
+++ b/model_and_optim/__3_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0ff84d8782e80ac1f53a5049ee0682774d7c4d24a756e273fcb6bb261d85a5f
+size 151005564
diff --git a/model_and_optim/__3_4.distcp b/model_and_optim/__3_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..44e29f012f6d1de036747b32c5533c7654bf92a6
--- /dev/null
+++ b/model_and_optim/__3_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f14e89ac319acfbfd62326a9d3781d459ae8d2dbc3655bca38030440beb2cb8b
+size 151005564
diff --git a/model_and_optim/__3_5.distcp b/model_and_optim/__3_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..631a681aa81db222474cf7833cee6008126597e9
--- /dev/null
+++ b/model_and_optim/__3_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ce955eff7b2ca6b8dbc2f3600865c4e71802117308b24403206a198ac85b141
+size 151005564
diff --git a/model_and_optim/__3_6.distcp b/model_and_optim/__3_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5107ac2529e46b70dc5daa273d2b51e012b017a5
--- /dev/null
+++ b/model_and_optim/__3_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bb334ecfca34c26ab700d0e7c1dd8c53fd4e30482a6633e79556d09646baf28
+size 151005564
diff --git a/model_and_optim/__3_7.distcp b/model_and_optim/__3_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..73ee3b6558530f533f0c631d6b704f0ad8b3339c
--- /dev/null
+++ b/model_and_optim/__3_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e53131435c60a8f8d6cb516332c51d701471e4a680367441b5f92bc8c35b0441
+size 151005564
diff --git a/model_and_optim/__3_8.distcp b/model_and_optim/__3_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..57414a31b24f4f717cb29c37aef25db5b083d6a0
--- /dev/null
+++ b/model_and_optim/__3_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2717053fab6192a441842db4203e701f7907230bc36b75c524a85cba6a60ee62
+size 151005564
diff --git a/model_and_optim/__3_9.distcp b/model_and_optim/__3_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..42de59e9152ac6cb38c3821648845227d849199c
--- /dev/null
+++ b/model_and_optim/__3_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df3d771b4fa9c05609c2bb02ead69930d10a75feb5502ff94ced814cdb70d491
+size 151005564
diff --git a/model_and_optim/__4_0.distcp b/model_and_optim/__4_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..bb46031e0474a30fe40398c8b75c3ca0f63244af
--- /dev/null
+++ b/model_and_optim/__4_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0211d30598e1c210f0df942eaa6f809700ff6788201ad6503cbadb7af8ca8209
+size 151005564
diff --git a/model_and_optim/__4_1.distcp b/model_and_optim/__4_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8586edaf7ad10645ff2f06401854fce7f2f5b556
--- /dev/null
+++ b/model_and_optim/__4_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d553018f3823f4f5df414f11d1ec2abcb4e9dff44d0f568597fdcfea728fe507
+size 151005564
diff --git a/model_and_optim/__4_10.distcp b/model_and_optim/__4_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5d585231be85d999b97b5d0e1bf825477d799b20
--- /dev/null
+++ b/model_and_optim/__4_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21816697713759e8df59e1fc6b7a235c43357973f84240fe7fe6b7a2f31f7358
+size 151005564
diff --git a/model_and_optim/__4_11.distcp b/model_and_optim/__4_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ab358818a669e50be9d9cd52d63fd182ab3f0d30
--- /dev/null
+++ b/model_and_optim/__4_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:552625d3ed2aff6f15692add62fa0746c2280b1f47b0688642c24ec39ab047c9
+size 151005564
diff --git a/model_and_optim/__4_12.distcp b/model_and_optim/__4_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..3e25271c4123fbd36d9aa668d27ea2b384cf78d4
--- /dev/null
+++ b/model_and_optim/__4_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a863824a56f7da06a26f2b30024671636ea28b288d3bc2fae9a84f3078857796
+size 151005564
diff --git a/model_and_optim/__4_13.distcp b/model_and_optim/__4_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..b1025dc677f703661d8a6312e0cb5cfa696f0470
--- /dev/null
+++ b/model_and_optim/__4_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e5e0d8ec142ee80342c43324f4ff457b712d3581c073a601ac00a08b8f33723
+size 151005564
diff --git a/model_and_optim/__4_14.distcp b/model_and_optim/__4_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..70b0d9a8455c22910919b98266ef483b1545e00c
--- /dev/null
+++ b/model_and_optim/__4_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1095197e8a93f133e40aff44ff22e91fbe6ea000f56f7c4e03be3b51211982bd
+size 151005564
diff --git a/model_and_optim/__4_15.distcp b/model_and_optim/__4_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5cfd1b9c74c75a34ee565aae87b93a66dd823c07
--- /dev/null
+++ b/model_and_optim/__4_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c90b494ae974d9a40111bd0781bcb90d60466647168a5d2b5fd98a33ee8b1fd
+size 151005564
diff --git a/model_and_optim/__4_2.distcp b/model_and_optim/__4_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..900ebd96423974ec80ad8a209824342a6956fc66
--- /dev/null
+++ b/model_and_optim/__4_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:284f8db00a084d105dc827266325875f301df9db2a507cdf803e20f92774d01e
+size 151005564
diff --git a/model_and_optim/__4_3.distcp b/model_and_optim/__4_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ab96826316b0c9766f00372ecb56a489f4af9da0
--- /dev/null
+++ b/model_and_optim/__4_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a298855e97f189af9ffc7a636c28269047d8a9f85188479469366a3e1e53bdb
+size 151005564
diff --git a/model_and_optim/__4_4.distcp b/model_and_optim/__4_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..9d84e16d1e9055eaf63eb2afe406dc9ef9280181
--- /dev/null
+++ b/model_and_optim/__4_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a54e7e96f5303c792ba197365a5bccb35d99fdebd5be010710638d88b4c2362
+size 151005564
diff --git a/model_and_optim/__4_5.distcp b/model_and_optim/__4_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..df5774b32b23c908f435d275c4a7b566af02e8bf
--- /dev/null
+++ b/model_and_optim/__4_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23143b98f7b55bf2ae39a4fb966a13cd8109fd7ce0cb475b36fba6df9f747f94
+size 151005564
diff --git a/model_and_optim/__4_6.distcp b/model_and_optim/__4_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..0bc5fb12e80e999866bb2e1f292a118ecd4f6877
--- /dev/null
+++ b/model_and_optim/__4_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca9115279991d9e5e95a9afaeafcd492e62b9ad4e6ecf855235df1f99a8555d5
+size 151005564
diff --git a/model_and_optim/__4_7.distcp b/model_and_optim/__4_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..7092215a593cecb69fec48da4eb01e75a6e198ca
--- /dev/null
+++ b/model_and_optim/__4_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e32140cecd8a31b12ea2aac28131b01f66dd858571d6e36627427a02ab64276
+size 151005564
diff --git a/model_and_optim/__4_8.distcp b/model_and_optim/__4_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..078dc440392aab2ff17eb2906b3aa1b1ae7d30f7
--- /dev/null
+++ b/model_and_optim/__4_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be04a4b94c80ee68c772ac130ff77816c3e1d935a6602a56cc37d0ef5f0ee4b
+size 151005564
diff --git a/model_and_optim/__4_9.distcp b/model_and_optim/__4_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..caf7616926e5497cd701e7bbe62d63407cce768a
--- /dev/null
+++ b/model_and_optim/__4_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b47b70d6ea87295d76bc7d8d682c257d1c2e9f288a2d279f4fde39e39d2925a
+size 151005564
diff --git a/model_and_optim/__5_0.distcp b/model_and_optim/__5_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ee3dd375b7fd7f58372b4727b53f5950bda146d2
--- /dev/null
+++ b/model_and_optim/__5_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7735ecbbd0f2b3bf5076a93190fa4c0659705acaafffc4283d39bda6b24cf548
+size 151005564
diff --git a/model_and_optim/__5_1.distcp b/model_and_optim/__5_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..b98b03297d997ff59bdfd626258039371211b57a
--- /dev/null
+++ b/model_and_optim/__5_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd2e06443b63047a2becf6671a359ec15a5d0781aaf83b4f11a3a03d597eb526
+size 151005564
diff --git a/model_and_optim/__5_10.distcp b/model_and_optim/__5_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..e68c52003e9fe5599cd4bcbbfa4ca486d909a207
--- /dev/null
+++ b/model_and_optim/__5_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ece6292ed5e77c722f84f39b8731fae73ca8bf1ec88c4fca53c9ab9f8eca9af1
+size 151005564
diff --git a/model_and_optim/__5_11.distcp b/model_and_optim/__5_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..03465d81605fedb47d856730f54f139975712701
--- /dev/null
+++ b/model_and_optim/__5_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c1d8479a5f7df922ab1e4bfa28d41c65ea3e206c22ce35ccdee3e11d0594327
+size 151005564
diff --git a/model_and_optim/__5_12.distcp b/model_and_optim/__5_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..278de4b9641be8676bfef55eeb18c47e9db0734f
--- /dev/null
+++ b/model_and_optim/__5_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb5f80f6a3aa2ad92cb57c1736565c33fc8b0b89c11575bd3aa167b15b1c68c
+size 151005564
diff --git a/model_and_optim/__5_13.distcp b/model_and_optim/__5_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..779811824609043493442b119a0d8b7865470d83
--- /dev/null
+++ b/model_and_optim/__5_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d64102c127cc580587aa1b6faa40838fe52e69240b97ee96fdb34cc3dce663af
+size 151005564
diff --git a/model_and_optim/__5_14.distcp b/model_and_optim/__5_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8182d2355ec7eb38eb616977e061a4178cd83ec9
--- /dev/null
+++ b/model_and_optim/__5_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9541a0f2f9120e3e2518fad56a7fc9c841bac2f307c92e29ea47d914f7e72337
+size 151005564
diff --git a/model_and_optim/__5_15.distcp b/model_and_optim/__5_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..649c145450291e74399fd74b74c660eb5e30b3a8
--- /dev/null
+++ b/model_and_optim/__5_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6301a8fea27c1d20d1ed3aeb335450a804a4b3af97a558e88a41df693b04aed8
+size 151005564
diff --git a/model_and_optim/__5_2.distcp b/model_and_optim/__5_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..0f45b9ac19e49a2c3140e59c04413f82a0db6bde
--- /dev/null
+++ b/model_and_optim/__5_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79bbf652b8a9fef7d1e6a219c688507f585f61ccba505da4718dd8e2f44699a5
+size 151005564
diff --git a/model_and_optim/__5_3.distcp b/model_and_optim/__5_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..209ef99b4f02cf9aacaf25ca8e7d24b65f408b20
--- /dev/null
+++ b/model_and_optim/__5_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c449769e54746448cff61cacc363488d0573a4941858372bb7c1432baab541c
+size 151005564
diff --git a/model_and_optim/__5_4.distcp b/model_and_optim/__5_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..39747c597f062d40ddd1c446c1aac462739e144e
--- /dev/null
+++ b/model_and_optim/__5_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c824f69135a15979abdb2863cf3685b904db396584f80fcd408fa6a872a918b8
+size 151005564
diff --git a/model_and_optim/__5_5.distcp b/model_and_optim/__5_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..d5535bbbf7fa20ab72dc5239d1191c42ce21c38d
--- /dev/null
+++ b/model_and_optim/__5_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b57d7e0a4c6cc56441d109dd4871b771b7504ad446f41f4128ae7f23c840dc1d
+size 151005564
diff --git a/model_and_optim/__5_6.distcp b/model_and_optim/__5_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..98605cce479d07f7ea19ec18885271f137e455cc
--- /dev/null
+++ b/model_and_optim/__5_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37176d34cdbe1fa5531c642402f43df7c944d82646a3dd2393ef00d228fec855
+size 151005564
diff --git a/model_and_optim/__5_7.distcp b/model_and_optim/__5_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8eeb57f4265211b291f99d988009635d3d23e33e
--- /dev/null
+++ b/model_and_optim/__5_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ada26688943dfcf2c48ddfd9cc300c2bf5c73da0668cde659360c181947a68e9
+size 151005564
diff --git a/model_and_optim/__5_8.distcp b/model_and_optim/__5_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..9554626e9b926a156aa48b01655ed6a1ac667888
--- /dev/null
+++ b/model_and_optim/__5_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1a14d5e86be2f1a6e0d16fdb2e34cbd97d35c2ab55ed4e3156ae474c43224a0
+size 151005564
diff --git a/model_and_optim/__5_9.distcp b/model_and_optim/__5_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..a5fde6bb407059bb9ffda28168afb25d8fb3860a
--- /dev/null
+++ b/model_and_optim/__5_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd97f4c338d5e82d97eb83c34e75010f1833075001f14cd69cfdd5024909f285
+size 151005564
diff --git a/model_and_optim/__6_0.distcp b/model_and_optim/__6_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..db854002d824346526c3e7ab6f8a15ff54314efd
--- /dev/null
+++ b/model_and_optim/__6_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29a387aeadcb395956604687917149841e68c3b4658526a00fde144938eaebb5
+size 151005564
diff --git a/model_and_optim/__6_1.distcp b/model_and_optim/__6_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..3e2a261e94a9aa0aebf8e9609c4cadfe3e178ffe
--- /dev/null
+++ b/model_and_optim/__6_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d719bbd5da6ca54d4b1ff3b3cb646c28d73e3e60425d0869597a3b48157e878
+size 151005564
diff --git a/model_and_optim/__6_10.distcp b/model_and_optim/__6_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f06530ca1a39e37968bb18810b9e812b6a09d363
--- /dev/null
+++ b/model_and_optim/__6_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:985f80303ea6fd625007c50cda7804d9785fcf3ddc4bd89969997c412f8ecbc3
+size 151005564
diff --git a/model_and_optim/__6_11.distcp b/model_and_optim/__6_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..9f59d19660b0d522de04a96454aa15533af8beca
--- /dev/null
+++ b/model_and_optim/__6_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b37ce8c5e11bffe929024666ba09c4a138c04d9fd65a4ee880972bb5ad647be
+size 151005564
diff --git a/model_and_optim/__6_12.distcp b/model_and_optim/__6_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..5f3aaf3891e65f6052ccdd0cd0b39d23219fed59
--- /dev/null
+++ b/model_and_optim/__6_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c4fa7a5b62459e18a5d516878733b3900e7eef11da85eb6897aaffbc1c3484e
+size 151005564
diff --git a/model_and_optim/__6_13.distcp b/model_and_optim/__6_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..6b916fa5d15bbf18048901ef1757f9e6311045ad
--- /dev/null
+++ b/model_and_optim/__6_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b25207f7a40208d045573c2fc6fc6426cc62d9b993d7b65af8e158f0df27703e
+size 151005564
diff --git a/model_and_optim/__6_14.distcp b/model_and_optim/__6_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..ae02116842a23689ecd7f67fb7198790c32ff036
--- /dev/null
+++ b/model_and_optim/__6_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e046e3360579df740b312f1bd8e5597d7f43ad7e6fb6f4b5e9cb0af7987aa0
+size 151005564
diff --git a/model_and_optim/__6_15.distcp b/model_and_optim/__6_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..99c812160df47d2a1b33755d2851ff59829459d5
--- /dev/null
+++ b/model_and_optim/__6_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:100c8bddd03e95cbf4a538838220f5e73b14f1101d574d8417bfe3a19ce335e0
+size 151005564
diff --git a/model_and_optim/__6_2.distcp b/model_and_optim/__6_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..0efcfad1616b5f47874322f90e06b3903625b2e1
--- /dev/null
+++ b/model_and_optim/__6_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14f761d4703ed3842d4ab89795400e99ccc545f339a2d0d114526cd48f4dd6ae
+size 151005564
diff --git a/model_and_optim/__6_3.distcp b/model_and_optim/__6_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..7481f5e9e64c68e5a87ec44e33bda52a5df8acb0
--- /dev/null
+++ b/model_and_optim/__6_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab23897330ca18e74c97a7e222745b09d98dcc765db27baa62919acf4892335a
+size 151005564
diff --git a/model_and_optim/__6_4.distcp b/model_and_optim/__6_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..79ee7c65fafa3a22c062e1d29675ad4570805c32
--- /dev/null
+++ b/model_and_optim/__6_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cc2a0c6ac52ed3f9a665ed700e0fcf71a7e2a8db9bda7778779fe84d08f8ba1
+size 151005564
diff --git a/model_and_optim/__6_5.distcp b/model_and_optim/__6_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..bf28c77185773070a160dc591a82cf5f26330836
--- /dev/null
+++ b/model_and_optim/__6_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d90debf928b65f7a36b2b37a697e431d3cb4620d5bcefbbc77893587a2c9f6
+size 151005564
diff --git a/model_and_optim/__6_6.distcp b/model_and_optim/__6_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..4cb99945f3b2e2032908069bee212049644df447
--- /dev/null
+++ b/model_and_optim/__6_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:091ff9c558f7228d8845f1d8163571b71d0a48c443be663c47fccccf35738428
+size 151005564
diff --git a/model_and_optim/__6_7.distcp b/model_and_optim/__6_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..cf246d25567bdf826afadbe5c0e954f43791b8f7
--- /dev/null
+++ b/model_and_optim/__6_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9582e9dd7eb2b8010d9b29c43ae6f62674fc36712cee686ccfb457a76f73c6d
+size 151005564
diff --git a/model_and_optim/__6_8.distcp b/model_and_optim/__6_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..881059cc57472288c667432493c61c46a07f2c42
--- /dev/null
+++ b/model_and_optim/__6_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9522e3edc3ae1791ed60e7f682704e04a3ab88f02681e0f8dd2917496c812aaa
+size 151005564
diff --git a/model_and_optim/__6_9.distcp b/model_and_optim/__6_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..557ba4efb933c140f6972e2ce0005bf38810da9b
--- /dev/null
+++ b/model_and_optim/__6_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57830857b785069203375057f1cdede00a86901f4c7685662481ec9d89a8e4df
+size 151005564
diff --git a/model_and_optim/__7_0.distcp b/model_and_optim/__7_0.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..b7853ade1c8510a69c22b3adcd56e8d415d083ea
--- /dev/null
+++ b/model_and_optim/__7_0.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:269d9e4c04ee029b7a8a42a55de3ac73b9d62857d983fe09be92ac1b031b67bd
+size 151005564
diff --git a/model_and_optim/__7_1.distcp b/model_and_optim/__7_1.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f45662028546028aaf5018c06f3c15c2d17ab1d5
--- /dev/null
+++ b/model_and_optim/__7_1.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149de73e33a0e523d46c2f5a9d3c7ef5ae9fe5b64d289611d7b4880be6c22cc8
+size 151005564
diff --git a/model_and_optim/__7_10.distcp b/model_and_optim/__7_10.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..0eaf4f12b21dc0ea64f2bfa9f3c72bc8ad3333ff
--- /dev/null
+++ b/model_and_optim/__7_10.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd178f7e52dde43b2752468572993b3d19567269e821eee51e5057a3dd378ac
+size 151005564
diff --git a/model_and_optim/__7_11.distcp b/model_and_optim/__7_11.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..cece3979a5950600eeb5c19b31a2a472d6736ebb
--- /dev/null
+++ b/model_and_optim/__7_11.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fa01101253af4042fb73bd995befe5fe4fd4e318c0c82013f69f76254994474
+size 151005564
diff --git a/model_and_optim/__7_12.distcp b/model_and_optim/__7_12.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8859490da8cd5ed5a200c2a230f23fc4787d2b0f
--- /dev/null
+++ b/model_and_optim/__7_12.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5213974cfafde5573d49b97d0f0b17eb61ecd37e3bcf00f85081e2eb38bf12d2
+size 151005564
diff --git a/model_and_optim/__7_13.distcp b/model_and_optim/__7_13.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f5b08706d21b4751ce3068226148b8da75296f24
--- /dev/null
+++ b/model_and_optim/__7_13.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ff9dbeba68db005951b8dff6ee51e050a4b927a17a7fdf72a9a091a56349ff9
+size 151005564
diff --git a/model_and_optim/__7_14.distcp b/model_and_optim/__7_14.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..2070aa5adf64dd8b51449a046b71cc0c605870ce
--- /dev/null
+++ b/model_and_optim/__7_14.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:535625c4ac82e51542c2469551b7365f30bdf3baaadc033cb969384e85120933
+size 151005564
diff --git a/model_and_optim/__7_15.distcp b/model_and_optim/__7_15.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..0aeb4e92a6930c1da477441344e136a8144220dc
--- /dev/null
+++ b/model_and_optim/__7_15.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfd36f26e6049faad80cbbc5a1993d46ad302bb8206524b1474063afa5d7050b
+size 151005564
diff --git a/model_and_optim/__7_2.distcp b/model_and_optim/__7_2.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..fac47b29773468639b882a1158f77389eac49294
--- /dev/null
+++ b/model_and_optim/__7_2.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2bc8d0221cd258b5f23a667f2196376aecf8e52f99f11d8049a448504940011
+size 151005564
diff --git a/model_and_optim/__7_3.distcp b/model_and_optim/__7_3.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..f7797bf8f0563ad2257feb9310ea1fd43d31afe1
--- /dev/null
+++ b/model_and_optim/__7_3.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaafe3714b2e7d52fb7faa35f66536931f95ea8b5a2b3a48ae725a714410aa86
+size 151005564
diff --git a/model_and_optim/__7_4.distcp b/model_and_optim/__7_4.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..eac9fca52b7b929efb1b375d5092d455f17e22d2
--- /dev/null
+++ b/model_and_optim/__7_4.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8725af11d9db353b99eae01c2401e7c57bb0483d059cbbbe78bb49f8c6e6e499
+size 151005564
diff --git a/model_and_optim/__7_5.distcp b/model_and_optim/__7_5.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..1212d5b6d5c5ab32b7f84bd92830cb28849cc8da
--- /dev/null
+++ b/model_and_optim/__7_5.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a74713d14dc0b1ab732d62ca64d1769ec7eee9da4be70f4d9a106989c5eaeee
+size 151005564
diff --git a/model_and_optim/__7_6.distcp b/model_and_optim/__7_6.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..8fab7b2e4f0e898b554d670c86110f1197f3e1d4
--- /dev/null
+++ b/model_and_optim/__7_6.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84af4d150c3feab6f82b7dd29544155b6792a46db476b9d259849bc4e6593c88
+size 151005564
diff --git a/model_and_optim/__7_7.distcp b/model_and_optim/__7_7.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..36d6d2e0035d8a6f323af73a6ee2116ab5426457
--- /dev/null
+++ b/model_and_optim/__7_7.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43463e21eea4a477f71ff46d331aec025a9904ac49a6a3a69dc5cff1abf062cc
+size 151005564
diff --git a/model_and_optim/__7_8.distcp b/model_and_optim/__7_8.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..56d9dcca8127ac504eaa02ad885f0dfe692ad5ad
--- /dev/null
+++ b/model_and_optim/__7_8.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a717b44fde8776a8e127c5273e3b76768d2ae3ef922caf2fe319c3d501a2beee
+size 151005564
diff --git a/model_and_optim/__7_9.distcp b/model_and_optim/__7_9.distcp
new file mode 100644
index 0000000000000000000000000000000000000000..656ab4f0a4fd77091ecd8a3eeef025c0c7059347
--- /dev/null
+++ b/model_and_optim/__7_9.distcp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ec9041b4a2fadc58d61d119280e23a05ab304a562f7d016a23cf144ae95308f
+size 151005564
diff --git a/train/rank0.pt b/train/rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d2cc709ed21c97df9f3abc10e80fa509215221a
--- /dev/null
+++ b/train/rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94d75ac90ed9e86b1a8fd5986387d74ee66687e6b271d9c852904236d825c849
+size 15052
diff --git a/train/rank1.pt b/train/rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c168eb8535ebb29cd650876db56f1cbeb984b6a7
--- /dev/null
+++ b/train/rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b95170252bf4d42b03f20eb08ea47a6771a298ba07696bb022daeb7d2de451f5
+size 15052
diff --git a/train/rank2.pt b/train/rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..119922e8ac8f377b1a78dae8e4e8187129495a3b
--- /dev/null
+++ b/train/rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cf9694aebfb97e28c7e147435c731ac2e2589c7d0d2275820fd07caaa771021
+size 15052
diff --git a/train/rank3.pt b/train/rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c34a770484ea04fede80158feca6286f58a6732
--- /dev/null
+++ b/train/rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d304ee52399a501b18d435da1d8b1f929061cd27b4a00d2afde76c7366099a9c
+size 15052
diff --git a/train/rank4.pt b/train/rank4.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22517fda2c2111345cd2097a624295229195de74
--- /dev/null
+++ b/train/rank4.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b9745b833a54b919f3eabe39b7cdd76d54124f6ca282c3d427b2d03d1af6cd4
+size 15052
diff --git a/train/rank5.pt b/train/rank5.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b2339b6460a7046966c73c9fde275db3ce681f9
--- /dev/null
+++ b/train/rank5.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abd22070430b1cbfc3c571ad01ac78b22480d0404bf03ca4cef8f765da95167c
+size 15052
diff --git a/train/rank6.pt b/train/rank6.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38bb72a9dd2546eb7f50da94fd40cf387123e877
--- /dev/null
+++ b/train/rank6.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9af38dbb8cc62447c4a47f8eba5a2a34f0c97d3cc06123051fef30989b40f23c
+size 15052
diff --git a/train/rank7.pt b/train/rank7.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b624eac96469fc2f1f495d4f763385cb3b630fa
--- /dev/null
+++ b/train/rank7.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e510509bc665e9049775364995549fcc27877fce12db3f4daddabcd2125084d
+size 15052