diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c9ac16702ab86299cca44a2c1411e497923eaa1d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb filter=lfs diff=lfs merge=lfs -text +train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/run-fhdgxxjz.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/train/act_dice_art_run1/checkpoints/020000/pretrained_model/config.json b/train/act_dice_art_run1/checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eed17d34a03c9896fc7beb8bf154021d20848c32 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,64 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/020000/pretrained_model/model.safetensors b/train/act_dice_art_run1/checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..411ee6aebaa3fcebfc807402bb9ef5c083fdd1a1 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3debbbbdc15ac24f2faf211ed999b8521d89fb2932125b067b08cfb77945cb86 +size 206701064 diff --git a/train/act_dice_art_run1/checkpoints/020000/pretrained_model/train_config.json b/train/act_dice_art_run1/checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c100a9681ad3eb0bdcc2eaa670550836b3d7c49e --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,170 @@ +{ + "dataset": { + "repo_id": "BoredMachinist/dice-art-run1", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/act_dice_art_run1", + "job_name": "act_dice_art_run1", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "fhdgxxjz", + "mode": null + } +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_param_groups.json b/train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_state.safetensors b/train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d573498ff36540ba3dfea946d12fda17d93a7be0 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ecb7bf1376a08e87a23aa01f2d51a65997f0bda2a544f19ac22e147babcfe8 +size 412817652 diff --git a/train/act_dice_art_run1/checkpoints/020000/training_state/rng_state.safetensors b/train/act_dice_art_run1/checkpoints/020000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f83463a629ab219f3d3408b45d3278cccdd5d072 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1ba88f4abbb0f4a60c28ce5166e14f1de15326d12c483f0def3d50e7c105a13 +size 15708 diff --git a/train/act_dice_art_run1/checkpoints/020000/training_state/training_step.json b/train/act_dice_art_run1/checkpoints/020000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9bb47026c5d5237ca6fc5dbff6020dd122ea05 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/020000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 20000 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/040000/pretrained_model/config.json b/train/act_dice_art_run1/checkpoints/040000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eed17d34a03c9896fc7beb8bf154021d20848c32 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/pretrained_model/config.json @@ -0,0 +1,64 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/040000/pretrained_model/model.safetensors b/train/act_dice_art_run1/checkpoints/040000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ea9f4c8c9337813839b1d7dd41c9580d55aa9f4e --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:792c237d0a9e0b491fac49010bb7e7861279cb0420934bb46fdd550f8cf7e6f0 +size 206701064 diff --git a/train/act_dice_art_run1/checkpoints/040000/pretrained_model/train_config.json b/train/act_dice_art_run1/checkpoints/040000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c100a9681ad3eb0bdcc2eaa670550836b3d7c49e --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/pretrained_model/train_config.json @@ -0,0 +1,170 @@ +{ + "dataset": { + "repo_id": "BoredMachinist/dice-art-run1", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/act_dice_art_run1", + "job_name": "act_dice_art_run1", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "fhdgxxjz", + "mode": null + } +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_param_groups.json b/train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_state.safetensors b/train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ee148fca6dbc4624f21b085390b20a15588f1e8d --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746228ecd60295447cdae7097efd335d0499dbb811eb7c26d694e61c9e937598 +size 412817652 diff --git a/train/act_dice_art_run1/checkpoints/040000/training_state/rng_state.safetensors b/train/act_dice_art_run1/checkpoints/040000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc1156b05ca22484d005cbb235d4647f4c975be9 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88ccf929509382665c94e8858d70d7af7646ca677014ead574a2440444698955 +size 15708 diff --git a/train/act_dice_art_run1/checkpoints/040000/training_state/training_step.json b/train/act_dice_art_run1/checkpoints/040000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..fe40d8ff4312c7e5a8fba9bcc932a43a1384ba77 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/040000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 40000 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/060000/pretrained_model/config.json b/train/act_dice_art_run1/checkpoints/060000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3821eba50196cb643fafa4e90a2e8f8912b3d9c1 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/pretrained_model/config.json @@ -0,0 +1,64 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/060000/pretrained_model/model.safetensors b/train/act_dice_art_run1/checkpoints/060000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b473a1640ec3738e9229d84a8ad6deef025fcc2 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83d53866360c1dbf05f9a3ebd73d3ab77dadfbee6dcca054c480b5bb4b83524 +size 206701064 diff --git a/train/act_dice_art_run1/checkpoints/060000/pretrained_model/train_config.json b/train/act_dice_art_run1/checkpoints/060000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..294833815ce9be6f03322780bea9d0228f72868f --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/pretrained_model/train_config.json @@ -0,0 +1,170 @@ +{ + "dataset": { + "repo_id": "BoredMachinist/dice-art-run1", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/act_dice_art_run1", + "job_name": "act_dice_art_run1", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "fhdgxxjz", + "mode": null + } +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_param_groups.json b/train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_state.safetensors b/train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d7f2c46180506b899595c02fb6b97d34169ad4f --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb566c648187f7eccdbee8d3807fc64c73381d396bbc5554457617b6c290513c +size 412817652 diff --git a/train/act_dice_art_run1/checkpoints/060000/training_state/rng_state.safetensors b/train/act_dice_art_run1/checkpoints/060000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2e311b08edb31d386f66ce366c7e405ba2455566 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da386b82654b6929f5157bdf6e8fcef1d9f55e70ddbb0d97a5b6432abd607501 +size 15708 diff --git a/train/act_dice_art_run1/checkpoints/060000/training_state/training_step.json b/train/act_dice_art_run1/checkpoints/060000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d360b6037f1c8c2c85d38e951160eb02ace507dd --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/060000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 60000 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/080000/pretrained_model/config.json b/train/act_dice_art_run1/checkpoints/080000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3821eba50196cb643fafa4e90a2e8f8912b3d9c1 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/pretrained_model/config.json @@ -0,0 +1,64 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/080000/pretrained_model/model.safetensors b/train/act_dice_art_run1/checkpoints/080000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4ffaa60b2b1cda2b4e41c56ec9d303534a84ec6c --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee379243296b72fed0528a86a041e05dcca13533c0dbd482784ea97da3e543c +size 206701064 diff --git a/train/act_dice_art_run1/checkpoints/080000/pretrained_model/train_config.json b/train/act_dice_art_run1/checkpoints/080000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..294833815ce9be6f03322780bea9d0228f72868f --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/pretrained_model/train_config.json @@ -0,0 +1,170 @@ +{ + "dataset": { + "repo_id": "BoredMachinist/dice-art-run1", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/act_dice_art_run1", + "job_name": "act_dice_art_run1", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "fhdgxxjz", + "mode": null + } +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_param_groups.json b/train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_state.safetensors b/train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9d58bebd6a2b7b9349848e1c88d06ab3748b0e38 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39a5ae696e01a4b70ac917d1295cef22a61c3f432b5943573aa4de5b89d48ca +size 412817652 diff --git a/train/act_dice_art_run1/checkpoints/080000/training_state/rng_state.safetensors b/train/act_dice_art_run1/checkpoints/080000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb8adf6737687dd1efbb12c5430155c8c5401f7a --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d91358c7f3d433e71afcdfebe0a2c8a4d876563ececf5fd9939d8e950b4ca8 +size 15708 diff --git a/train/act_dice_art_run1/checkpoints/080000/training_state/training_step.json b/train/act_dice_art_run1/checkpoints/080000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..384eace4ecc2a6cba352aa7cf27f04405b7319c3 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/080000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 80000 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/100000/pretrained_model/config.json b/train/act_dice_art_run1/checkpoints/100000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3821eba50196cb643fafa4e90a2e8f8912b3d9c1 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/pretrained_model/config.json @@ -0,0 +1,64 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/100000/pretrained_model/model.safetensors b/train/act_dice_art_run1/checkpoints/100000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..862c21ec45405354274fc2ae9d545c024dd1174f --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a9f83a2296309d6e87d2f62b202909cfc48794f29b82cf943333c0c5211945 +size 206701064 diff --git a/train/act_dice_art_run1/checkpoints/100000/pretrained_model/train_config.json b/train/act_dice_art_run1/checkpoints/100000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..294833815ce9be6f03322780bea9d0228f72868f --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/pretrained_model/train_config.json @@ -0,0 +1,170 @@ +{ + "dataset": { + "repo_id": "BoredMachinist/dice-art-run1", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.arm": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.world": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 100, + "n_action_steps": 100, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/act_dice_art_run1", + "job_name": "act_dice_art_run1", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "fhdgxxjz", + "mode": null + } +} \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_param_groups.json b/train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_state.safetensors b/train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8e01cb6eaa5c5aeb6d3efa39c09cb65a37d6df86 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a44c4e5523045bb226a12412edb707541d251e609999ad307432db6ec677981c +size 412817652 diff --git a/train/act_dice_art_run1/checkpoints/100000/training_state/rng_state.safetensors b/train/act_dice_art_run1/checkpoints/100000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..776caceed648d5029049cb4cd5e0fbe861e2ea5b --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6edd2b0eb5ffb0556ac59e5b8b208d9ec2a740d79c3c8bc18c8d0fc37afcde74 +size 15708 diff --git a/train/act_dice_art_run1/checkpoints/100000/training_state/training_step.json b/train/act_dice_art_run1/checkpoints/100000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8eb73c13d28bca88058c08796abbf931c3f9b012 --- /dev/null +++ b/train/act_dice_art_run1/checkpoints/100000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 100000 +} \ No newline at end of file diff --git a/train/act_dice_art_run1/wandb/debug-internal.log b/train/act_dice_art_run1/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cf0c80cfe098b6969067e156afefc2c3a29c843d --- /dev/null +++ b/train/act_dice_art_run1/wandb/debug-internal.log @@ -0,0 +1,136 @@ +{"time":"2025-06-15T01:25:25.992015864-07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-core.log"} +{"time":"2025-06-15T01:25:26.634032319-07:00","level":"INFO","msg":"stream: created new stream","id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634083631-07:00","level":"INFO","msg":"stream: started","id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634215949-07:00","level":"INFO","msg":"writer: Do: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634288668-07:00","level":"INFO","msg":"handler: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634389474-07:00","level":"INFO","msg":"sender: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:27.320758871-07:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-06-15T01:25:46.568983039-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.568997155-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.568999143-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569000545-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569001878-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569003382-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569004827-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569006116-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569007422-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569008882-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569048108-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:26:02.367793397-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372335004-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372337971-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372339512-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372340951-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372342409-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372343769-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372345043-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.3723463-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372348347-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372349688-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:17.685821042-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685849081-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685857431-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685858732-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685859866-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685861139-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685862255-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685863394-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685864601-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.68586591-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685867074-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:32.130793723-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130813145-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130815054-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130816448-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130817763-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130819273-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130820563-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130821885-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130823187-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.13082474-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130826044-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:46.825477529-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825530193-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825558155-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825582435-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825603491-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825629646-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825656804-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825682964-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825707833-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825737502-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825760265-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:27:02.16185199-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161878645-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161884483-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161888247-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161891557-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161894938-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161898154-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161901259-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161904335-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161907587-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161910751-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:16.797906839-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.7979248-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797928888-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797955435-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797965742-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797986483-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798010312-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798035412-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798056372-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798085178-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798107623-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:31.191814532-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.19183601-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191838508-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.19184059-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.19184258-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191850218-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191851983-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191854215-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191855837-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191857573-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191859143-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:45.982376932-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982418361-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982429595-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982466168-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982474892-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982495778-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982520859-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982542092-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982564589-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982593443-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982610475-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:28:01.229859045-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229889059-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229894087-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229897505-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229900368-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229903215-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229905919-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229908651-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229911346-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.22991437-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229917398-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:15.6350119-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635042909-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635078209-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635093855-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635112045-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635150996-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.6351606-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635184442-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.63520897-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635243681-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635260056-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T02:36:42.45420484-07:00","level":"INFO","msg":"stream: closing","id":"fhdgxxjz"} +{"time":"2025-06-15T02:36:42.454249855-07:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-06-15T02:36:42.454306083-07:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-06-15T02:37:31.874312646-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-06-15T02:37:32.062233809-07:00","level":"INFO","msg":"handler: closed","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T02:37:32.062277604-07:00","level":"INFO","msg":"writer: Close: closed","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T02:37:32.062318929-07:00","level":"INFO","msg":"sender: closed","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T02:37:32.06237074-07:00","level":"INFO","msg":"stream: closed","id":"fhdgxxjz"} diff --git a/train/act_dice_art_run1/wandb/debug.log b/train/act_dice_art_run1/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a5429fcf7dccd45ca90aa13016821371ae582889 --- /dev/null +++ b/train/act_dice_art_run1/wandb/debug.log @@ -0,0 +1,23 @@ +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1 +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Configure stats pid to 4339 +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/.config/wandb/settings +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/wandb/settings +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug.log +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-internal.log +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():831] calling init triggers +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():836] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'BoredMachinist/dice-art-run1', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {'observation.state': {'type': , 'shape': [6]}, 'observation.images.arm': {'type': , 'shape': [3, 480, 640]}, 'observation.images.world': {'type': , 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': , 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': 0, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_dice_art_run1', 'job_name': 'act_dice_art_run1', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'fhdgxxjz', 'mode': None}, '_wandb': {}} +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():872] starting backend +2025-06-15 01:25:25,979 INFO MainThread:4339 [wandb_init.py:init():875] sending inform_init request +2025-06-15 01:25:25,986 INFO MainThread:4339 [wandb_init.py:init():883] backend started and connected +2025-06-15 01:25:25,989 INFO MainThread:4339 [wandb_init.py:init():956] updated telemetry +2025-06-15 01:25:26,005 INFO MainThread:4339 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout +2025-06-15 01:25:27,275 INFO MainThread:4339 [wandb_init.py:init():1027] run resumed +2025-06-15 01:25:27,279 INFO MainThread:4339 [wandb_init.py:init():1032] starting run threads in backend +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_console_start():2453] atexit reg +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2301] redirect: wrap_raw +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2370] Wrapping output streams. +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2393] Redirects installed. +2025-06-15 01:25:27,406 INFO MainThread:4339 [wandb_init.py:init():1078] run started, returning control to user process +2025-06-15 02:36:42,453 INFO MsgRouterThr:4339 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 3 handles. diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/config.yaml b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0ffa4e8fa6a2d13306d915b2eee2318006156a4 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/config.yaml @@ -0,0 +1,161 @@ +_wandb: + value: + cli_version: 0.20.1 + m: [] + python_version: 3.10.13 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 105 + "2": + - 1 + - 41 + - 49 + - 51 + - 105 + "3": + - 13 + - 15 + - 16 + - 55 + - 61 + "4": 3.10.13 + "5": 0.20.1 + "10": + - 21 + "12": 0.20.1 + "13": linux-x86_64 +batch_size: + value: 8 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: BoredMachinist/dice-art-run1 + revision: null + root: null + use_imagenet_stats: true + video_backend: pyav +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: act_dice_art_run1 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/act_dice_art_run1 +policy: + value: + chunk_size: 100 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + replace_final_stride_with_dilation: false + temporal_ensemble_coeff: null + type: act + use_amp: false + use_vae: true + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: null +seed: + value: 1000 +steps: + value: 100000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/output.log b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1c137094a0946135d1f35e0188d6d72730de8a3b --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/output.log @@ -0,0 +1,301 @@ +Logs will be synced with wandb. +INFO 2025-06-14 20:09:52 db_utils.py:103 Track this run --> https://wandb.ai/ametind-o/lerobot/runs/fhdgxxjz +INFO 2025-06-14 20:09:52 ts/train.py:127 Creating dataset +INFO 2025-06-14 20:09:52 ts/train.py:138 Creating policy +INFO 2025-06-14 20:09:53 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-06-14 20:09:53 ts/train.py:156 Output dir: outputs/train/act_dice_art_run1 +INFO 2025-06-14 20:09:53 ts/train.py:159 cfg.steps=100000 (100K) +INFO 2025-06-14 20:09:53 ts/train.py:160 dataset.num_frames=2244 (2K) +INFO 2025-06-14 20:09:53 ts/train.py:161 dataset.num_episodes=9 +INFO 2025-06-14 20:09:53 ts/train.py:162 num_learnable_params=51597190 (52M) +INFO 2025-06-14 20:09:53 ts/train.py:163 num_total_params=51597238 (52M) +INFO 2025-06-14 20:09:53 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-06-14 20:10:09 ts/train.py:232 step:200 smpl:2K ep:6 epch:0.71 loss:6.515 grdn:149.668 lr:1.0e-05 updt_s:0.079 data_s:0.002 +INFO 2025-06-14 20:10:25 ts/train.py:232 step:400 smpl:3K ep:13 epch:1.43 loss:2.868 grdn:82.381 lr:1.0e-05 updt_s:0.076 data_s:0.002 +INFO 2025-06-14 20:10:40 ts/train.py:232 step:600 smpl:5K ep:19 epch:2.14 loss:2.408 grdn:72.525 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:10:55 ts/train.py:232 step:800 smpl:6K ep:26 epch:2.85 loss:2.109 grdn:67.732 lr:1.0e-05 updt_s:0.074 data_s:0.000 +INFO 2025-06-14 20:11:10 ts/train.py:232 step:1K smpl:8K ep:32 epch:3.57 loss:1.891 grdn:65.419 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:11:25 ts/train.py:232 step:1K smpl:10K ep:39 epch:4.28 loss:1.706 grdn:62.129 lr:1.0e-05 updt_s:0.074 data_s:0.001 +INFO 2025-06-14 20:11:40 ts/train.py:232 step:1K smpl:11K ep:45 epch:4.99 loss:1.534 grdn:58.982 lr:1.0e-05 updt_s:0.074 data_s:0.000 +INFO 2025-06-14 20:11:55 ts/train.py:232 step:2K smpl:13K ep:51 epch:5.70 loss:1.392 grdn:56.874 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:12:10 ts/train.py:232 step:2K smpl:14K ep:58 epch:6.42 loss:1.269 grdn:55.147 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:12:26 ts/train.py:232 step:2K smpl:16K ep:64 epch:7.13 loss:1.161 grdn:52.125 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:12:41 ts/train.py:232 step:2K smpl:18K ep:71 epch:7.84 loss:1.062 grdn:50.324 lr:1.0e-05 updt_s:0.074 data_s:0.000 +INFO 2025-06-14 20:12:56 ts/train.py:232 step:2K smpl:19K ep:77 epch:8.56 loss:0.960 grdn:47.189 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:13:11 ts/train.py:232 step:3K smpl:21K ep:83 epch:9.27 loss:0.870 grdn:45.187 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:13:26 ts/train.py:232 step:3K smpl:22K ep:90 epch:9.98 loss:0.793 grdn:42.965 lr:1.0e-05 updt_s:0.075 data_s:0.000 +INFO 2025-06-14 20:13:42 ts/train.py:232 step:3K smpl:24K ep:96 epch:10.70 loss:0.725 grdn:42.298 lr:1.0e-05 updt_s:0.075 data_s:0.002 +INFO 2025-06-14 20:13:57 ts/train.py:232 step:3K smpl:26K ep:103 epch:11.41 loss:0.659 grdn:40.367 lr:1.0e-05 updt_s:0.075 data_s:0.002 +INFO 2025-06-14 20:14:12 ts/train.py:232 step:3K smpl:27K ep:109 epch:12.12 loss:0.604 grdn:37.984 lr:1.0e-05 updt_s:0.075 data_s:0.002 +INFO 2025-06-14 20:14:27 ts/train.py:232 step:4K smpl:29K ep:116 epch:12.83 loss:0.555 grdn:36.595 lr:1.0e-05 updt_s:0.075 data_s:0.000 +INFO 2025-06-14 20:14:43 ts/train.py:232 step:4K smpl:30K ep:122 epch:13.55 loss:0.511 grdn:35.474 lr:1.0e-05 updt_s:0.075 data_s:0.002 +INFO 2025-06-14 20:14:58 ts/train.py:232 step:4K smpl:32K ep:128 epch:14.26 loss:0.473 grdn:33.822 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-14 20:15:13 ts/train.py:232 step:4K smpl:34K ep:135 epch:14.97 loss:0.440 grdn:34.830 lr:1.0e-05 updt_s:0.075 data_s:0.000 +INFO 2025-06-14 20:15:28 ts/train.py:232 step:4K smpl:35K ep:141 epch:15.69 loss:0.411 grdn:30.655 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-14 20:15:43 ts/train.py:232 step:5K smpl:37K ep:148 epch:16.40 loss:0.388 grdn:30.633 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:15:57 ts/train.py:232 step:5K smpl:38K ep:154 epch:17.11 loss:0.364 grdn:29.262 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:16:11 ts/train.py:232 step:5K smpl:40K ep:160 epch:17.83 loss:0.338 grdn:28.092 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:16:26 ts/train.py:232 step:5K smpl:42K ep:167 epch:18.54 loss:0.324 grdn:27.827 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:16:40 ts/train.py:232 step:5K smpl:43K ep:173 epch:19.25 loss:0.311 grdn:28.218 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:16:54 ts/train.py:232 step:6K smpl:45K ep:180 epch:19.96 loss:0.294 grdn:26.132 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:17:08 ts/train.py:232 step:6K smpl:46K ep:186 epch:20.68 loss:0.284 grdn:26.785 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:17:23 ts/train.py:232 step:6K smpl:48K ep:193 epch:21.39 loss:0.274 grdn:24.581 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:17:38 ts/train.py:232 step:6K smpl:50K ep:199 epch:22.10 loss:0.262 grdn:24.783 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:17:52 ts/train.py:232 step:6K smpl:51K ep:205 epch:22.82 loss:0.256 grdn:24.414 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-14 20:18:07 ts/train.py:232 step:7K smpl:53K ep:212 epch:23.53 loss:0.247 grdn:23.247 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-14 20:18:22 ts/train.py:232 step:7K smpl:54K ep:218 epch:24.24 loss:0.236 grdn:23.459 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-14 20:18:37 ts/train.py:232 step:7K smpl:56K ep:225 epch:24.96 loss:0.230 grdn:22.174 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:18:51 ts/train.py:232 step:7K smpl:58K ep:231 epch:25.67 loss:0.222 grdn:22.406 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:19:06 ts/train.py:232 step:7K smpl:59K ep:237 epch:26.38 loss:0.220 grdn:22.310 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:19:20 ts/train.py:232 step:8K smpl:61K ep:244 epch:27.09 loss:0.212 grdn:21.087 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:19:34 ts/train.py:232 step:8K smpl:62K ep:250 epch:27.81 loss:0.210 grdn:21.643 lr:1.0e-05 updt_s:0.071 data_s:0.000 +INFO 2025-06-14 20:19:49 ts/train.py:232 step:8K smpl:64K ep:257 epch:28.52 loss:0.205 grdn:20.799 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:20:03 ts/train.py:232 step:8K smpl:66K ep:263 epch:29.23 loss:0.201 grdn:21.118 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:20:17 ts/train.py:232 step:8K smpl:67K ep:270 epch:29.95 loss:0.193 grdn:20.314 lr:1.0e-05 updt_s:0.071 data_s:0.000 +INFO 2025-06-14 20:20:32 ts/train.py:232 step:9K smpl:69K ep:276 epch:30.66 loss:0.188 grdn:19.721 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:20:46 ts/train.py:232 step:9K smpl:70K ep:282 epch:31.37 loss:0.187 grdn:20.583 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:21:01 ts/train.py:232 step:9K smpl:72K ep:289 epch:32.09 loss:0.181 grdn:20.067 lr:1.0e-05 updt_s:0.071 data_s:0.001 +INFO 2025-06-14 20:21:15 ts/train.py:232 step:9K smpl:74K ep:295 epch:32.80 loss:0.181 grdn:19.900 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:21:30 ts/train.py:232 step:9K smpl:75K ep:302 epch:33.51 loss:0.175 grdn:19.306 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:21:44 ts/train.py:232 step:10K smpl:77K ep:308 epch:34.22 loss:0.175 grdn:18.802 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:21:58 ts/train.py:232 step:10K smpl:78K ep:314 epch:34.94 loss:0.171 grdn:18.738 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:22:13 ts/train.py:232 step:10K smpl:80K ep:321 epch:35.65 loss:0.166 grdn:18.788 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:22:27 ts/train.py:232 step:10K smpl:82K ep:327 epch:36.36 loss:0.165 grdn:17.641 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:22:42 ts/train.py:232 step:10K smpl:83K ep:334 epch:37.08 loss:0.164 grdn:19.923 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:22:56 ts/train.py:232 step:11K smpl:85K ep:340 epch:37.79 loss:0.163 grdn:19.528 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:23:10 ts/train.py:232 step:11K smpl:86K ep:347 epch:38.50 loss:0.158 grdn:17.752 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:23:25 ts/train.py:232 step:11K smpl:88K ep:353 epch:39.22 loss:0.157 grdn:17.456 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:23:39 ts/train.py:232 step:11K smpl:90K ep:359 epch:39.93 loss:0.153 grdn:17.560 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:23:53 ts/train.py:232 step:11K smpl:91K ep:366 epch:40.64 loss:0.153 grdn:17.448 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:24:07 ts/train.py:232 step:12K smpl:93K ep:372 epch:41.35 loss:0.150 grdn:17.562 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:24:21 ts/train.py:232 step:12K smpl:94K ep:379 epch:42.07 loss:0.148 grdn:17.039 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:24:36 ts/train.py:232 step:12K smpl:96K ep:385 epch:42.78 loss:0.146 grdn:16.688 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-14 20:24:50 ts/train.py:232 step:12K smpl:98K ep:391 epch:43.49 loss:0.145 grdn:17.900 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:25:04 ts/train.py:232 step:12K smpl:99K ep:398 epch:44.21 loss:0.143 grdn:16.788 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:25:18 ts/train.py:232 step:13K smpl:101K ep:404 epch:44.92 loss:0.141 grdn:16.224 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:25:32 ts/train.py:232 step:13K smpl:102K ep:411 epch:45.63 loss:0.140 grdn:16.345 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:25:47 ts/train.py:232 step:13K smpl:104K ep:417 epch:46.35 loss:0.139 grdn:16.433 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:26:01 ts/train.py:232 step:13K smpl:106K ep:424 epch:47.06 loss:0.137 grdn:16.710 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:26:15 ts/train.py:232 step:13K smpl:107K ep:430 epch:47.77 loss:0.135 grdn:15.966 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:26:29 ts/train.py:232 step:14K smpl:109K ep:436 epch:48.48 loss:0.137 grdn:16.677 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:26:43 ts/train.py:232 step:14K smpl:110K ep:443 epch:49.20 loss:0.131 grdn:15.750 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:26:57 ts/train.py:232 step:14K smpl:112K ep:449 epch:49.91 loss:0.129 grdn:15.559 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:27:12 ts/train.py:232 step:14K smpl:114K ep:456 epch:50.62 loss:0.130 grdn:15.826 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:27:26 ts/train.py:232 step:14K smpl:115K ep:462 epch:51.34 loss:0.128 grdn:15.333 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:27:40 ts/train.py:232 step:15K smpl:117K ep:468 epch:52.05 loss:0.126 grdn:14.943 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:27:54 ts/train.py:232 step:15K smpl:118K ep:475 epch:52.76 loss:0.126 grdn:15.906 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:28:09 ts/train.py:232 step:15K smpl:120K ep:481 epch:53.48 loss:0.123 grdn:15.554 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:28:23 ts/train.py:232 step:15K smpl:122K ep:488 epch:54.19 loss:0.125 grdn:15.625 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:28:37 ts/train.py:232 step:15K smpl:123K ep:494 epch:54.90 loss:0.122 grdn:14.467 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:28:51 ts/train.py:232 step:16K smpl:125K ep:501 epch:55.61 loss:0.121 grdn:14.501 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:29:05 ts/train.py:232 step:16K smpl:126K ep:507 epch:56.33 loss:0.121 grdn:15.215 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:29:19 ts/train.py:232 step:16K smpl:128K ep:513 epch:57.04 loss:0.118 grdn:14.298 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:29:34 ts/train.py:232 step:16K smpl:130K ep:520 epch:57.75 loss:0.119 grdn:14.430 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:29:48 ts/train.py:232 step:16K smpl:131K ep:526 epch:58.47 loss:0.115 grdn:14.649 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:30:02 ts/train.py:232 step:17K smpl:133K ep:533 epch:59.18 loss:0.114 grdn:14.037 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:30:16 ts/train.py:232 step:17K smpl:134K ep:539 epch:59.89 loss:0.114 grdn:14.603 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:30:30 ts/train.py:232 step:17K smpl:136K ep:545 epch:60.61 loss:0.114 grdn:13.872 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:30:45 ts/train.py:232 step:17K smpl:138K ep:552 epch:61.32 loss:0.112 grdn:14.225 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:30:59 ts/train.py:232 step:17K smpl:139K ep:558 epch:62.03 loss:0.114 grdn:14.691 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:31:13 ts/train.py:232 step:18K smpl:141K ep:565 epch:62.75 loss:0.111 grdn:13.451 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:31:27 ts/train.py:232 step:18K smpl:142K ep:571 epch:63.46 loss:0.110 grdn:14.047 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:31:41 ts/train.py:232 step:18K smpl:144K ep:578 epch:64.17 loss:0.108 grdn:13.818 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:31:55 ts/train.py:232 step:18K smpl:146K ep:584 epch:64.88 loss:0.108 grdn:13.908 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:32:10 ts/train.py:232 step:18K smpl:147K ep:590 epch:65.60 loss:0.107 grdn:13.497 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:32:24 ts/train.py:232 step:19K smpl:149K ep:597 epch:66.31 loss:0.107 grdn:13.179 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:32:38 ts/train.py:232 step:19K smpl:150K ep:603 epch:67.02 loss:0.107 grdn:13.837 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:32:52 ts/train.py:232 step:19K smpl:152K ep:610 epch:67.74 loss:0.104 grdn:13.106 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:33:07 ts/train.py:232 step:19K smpl:154K ep:616 epch:68.45 loss:0.104 grdn:12.692 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-14 20:33:21 ts/train.py:232 step:19K smpl:155K ep:622 epch:69.16 loss:0.103 grdn:13.085 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:33:35 ts/train.py:232 step:20K smpl:157K ep:629 epch:69.88 loss:0.102 grdn:12.937 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:33:49 ts/train.py:232 step:20K smpl:158K ep:635 epch:70.59 loss:0.101 grdn:12.779 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:34:04 ts/train.py:232 step:20K smpl:160K ep:642 epch:71.30 loss:0.100 grdn:12.397 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:34:04 ts/train.py:241 Checkpoint policy after step 20000 +INFO 2025-06-14 20:34:20 ts/train.py:232 step:20K smpl:162K ep:648 epch:72.01 loss:0.102 grdn:13.082 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-14 20:34:34 ts/train.py:232 step:20K smpl:163K ep:655 epch:72.73 loss:0.101 grdn:13.668 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:34:49 ts/train.py:232 step:21K smpl:165K ep:661 epch:73.44 loss:0.098 grdn:12.644 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:35:04 ts/train.py:232 step:21K smpl:166K ep:667 epch:74.15 loss:0.098 grdn:12.694 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:35:18 ts/train.py:232 step:21K smpl:168K ep:674 epch:74.87 loss:0.095 grdn:11.722 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:35:32 ts/train.py:232 step:21K smpl:170K ep:680 epch:75.58 loss:0.096 grdn:12.236 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:35:47 ts/train.py:232 step:21K smpl:171K ep:687 epch:76.29 loss:0.096 grdn:12.080 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-14 20:36:02 ts/train.py:232 step:22K smpl:173K ep:693 epch:77.01 loss:0.097 grdn:12.912 lr:1.0e-05 updt_s:0.075 data_s:0.000 +INFO 2025-06-14 20:36:18 ts/train.py:232 step:22K smpl:174K ep:699 epch:77.72 loss:0.096 grdn:12.410 lr:1.0e-05 updt_s:0.075 data_s:0.002 +INFO 2025-06-14 20:36:33 ts/train.py:232 step:22K smpl:176K ep:706 epch:78.43 loss:0.094 grdn:12.101 lr:1.0e-05 updt_s:0.074 data_s:0.001 +INFO 2025-06-14 20:36:48 ts/train.py:232 step:22K smpl:178K ep:712 epch:79.14 loss:0.093 grdn:11.906 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:37:02 ts/train.py:232 step:22K smpl:179K ep:719 epch:79.86 loss:0.094 grdn:11.773 lr:1.0e-05 updt_s:0.071 data_s:0.000 +INFO 2025-06-14 20:37:17 ts/train.py:232 step:23K smpl:181K ep:725 epch:80.57 loss:0.092 grdn:12.889 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:37:32 ts/train.py:232 step:23K smpl:182K ep:732 epch:81.28 loss:0.093 grdn:11.871 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:37:46 ts/train.py:232 step:23K smpl:184K ep:738 epch:82.00 loss:0.089 grdn:11.595 lr:1.0e-05 updt_s:0.071 data_s:0.000 +INFO 2025-06-14 20:38:01 ts/train.py:232 step:23K smpl:186K ep:744 epch:82.71 loss:0.092 grdn:12.292 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:38:15 ts/train.py:232 step:23K smpl:187K ep:751 epch:83.42 loss:0.090 grdn:12.134 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-14 20:38:30 ts/train.py:232 step:24K smpl:189K ep:757 epch:84.14 loss:0.089 grdn:11.247 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-14 20:38:45 ts/train.py:232 step:24K smpl:190K ep:764 epch:84.85 loss:0.087 grdn:11.122 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-14 20:39:00 ts/train.py:232 step:24K smpl:192K ep:770 epch:85.56 loss:0.089 grdn:11.528 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:39:15 ts/train.py:232 step:24K smpl:194K ep:776 epch:86.27 loss:0.088 grdn:12.144 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:39:29 ts/train.py:232 step:24K smpl:195K ep:783 epch:86.99 loss:0.086 grdn:10.970 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-14 20:39:44 ts/train.py:232 step:25K smpl:197K ep:789 epch:87.70 loss:0.086 grdn:11.029 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-14 20:39:59 ts/train.py:232 step:25K smpl:198K ep:796 epch:88.41 loss:0.086 grdn:11.238 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:40:13 ts/train.py:232 step:25K smpl:200K ep:802 epch:89.13 loss:0.086 grdn:10.944 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-14 20:40:28 ts/train.py:232 step:25K smpl:202K ep:809 epch:89.84 loss:0.085 grdn:11.207 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:40:42 ts/train.py:232 step:25K smpl:203K ep:815 epch:90.55 loss:0.086 grdn:11.160 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:40:56 ts/train.py:232 step:26K smpl:205K ep:821 epch:91.27 loss:0.085 grdn:11.252 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:41:10 ts/train.py:232 step:26K smpl:206K ep:828 epch:91.98 loss:0.082 grdn:10.840 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:41:25 ts/train.py:232 step:26K smpl:208K ep:834 epch:92.69 loss:0.084 grdn:10.840 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:41:39 ts/train.py:232 step:26K smpl:210K ep:841 epch:93.40 loss:0.081 grdn:11.170 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:41:53 ts/train.py:232 step:26K smpl:211K ep:847 epch:94.12 loss:0.083 grdn:10.977 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:42:08 ts/train.py:232 step:27K smpl:213K ep:853 epch:94.83 loss:0.083 grdn:11.122 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:42:22 ts/train.py:232 step:27K smpl:214K ep:860 epch:95.54 loss:0.082 grdn:10.623 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:42:36 ts/train.py:232 step:27K smpl:216K ep:866 epch:96.26 loss:0.081 grdn:10.906 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:42:51 ts/train.py:232 step:27K smpl:218K ep:873 epch:96.97 loss:0.079 grdn:10.305 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:43:05 ts/train.py:232 step:27K smpl:219K ep:879 epch:97.68 loss:0.080 grdn:10.587 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:43:19 ts/train.py:232 step:28K smpl:221K ep:886 epch:98.40 loss:0.078 grdn:9.818 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:43:33 ts/train.py:232 step:28K smpl:222K ep:892 epch:99.11 loss:0.081 grdn:10.856 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:43:48 ts/train.py:232 step:28K smpl:224K ep:898 epch:99.82 loss:0.079 grdn:10.619 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:44:02 ts/train.py:232 step:28K smpl:226K ep:905 epch:100.53 loss:0.078 grdn:10.213 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:44:16 ts/train.py:232 step:28K smpl:227K ep:911 epch:101.25 loss:0.078 grdn:10.132 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:44:31 ts/train.py:232 step:29K smpl:229K ep:918 epch:101.96 loss:0.078 grdn:10.275 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:44:45 ts/train.py:232 step:29K smpl:230K ep:924 epch:102.67 loss:0.076 grdn:9.912 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:44:59 ts/train.py:232 step:29K smpl:232K ep:930 epch:103.39 loss:0.077 grdn:9.954 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:45:13 ts/train.py:232 step:29K smpl:234K ep:937 epch:104.10 loss:0.076 grdn:9.673 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:45:28 ts/train.py:232 step:29K smpl:235K ep:943 epch:104.81 loss:0.075 grdn:9.531 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:45:42 ts/train.py:232 step:30K smpl:237K ep:950 epch:105.53 loss:0.076 grdn:10.207 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:45:56 ts/train.py:232 step:30K smpl:238K ep:956 epch:106.24 loss:0.076 grdn:9.426 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-14 20:46:10 ts/train.py:232 step:30K smpl:240K ep:963 epch:106.95 loss:0.076 grdn:10.054 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:46:25 ts/train.py:232 step:30K smpl:242K ep:969 epch:107.66 loss:0.076 grdn:9.912 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:46:39 ts/train.py:232 step:30K smpl:243K ep:975 epch:108.38 loss:0.074 grdn:9.162 lr:1.0e-05 updt_s:0.071 data_s:0.001 +INFO 2025-06-14 20:46:54 ts/train.py:232 step:31K smpl:245K ep:982 epch:109.09 loss:0.074 grdn:10.086 lr:1.0e-05 updt_s:0.071 data_s:0.000 +INFO 2025-06-14 20:47:08 ts/train.py:232 step:31K smpl:246K ep:988 epch:109.80 loss:0.073 grdn:9.689 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:47:23 ts/train.py:232 step:31K smpl:248K ep:995 epch:110.52 loss:0.074 grdn:9.903 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:47:37 ts/train.py:232 step:31K smpl:250K ep:1K epch:111.23 loss:0.072 grdn:9.448 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:47:51 ts/train.py:232 step:31K smpl:251K ep:1K epch:111.94 loss:0.073 grdn:9.866 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:48:06 ts/train.py:232 step:32K smpl:253K ep:1K epch:112.66 loss:0.073 grdn:9.879 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:48:20 ts/train.py:232 step:32K smpl:254K ep:1K epch:113.37 loss:0.072 grdn:9.268 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:48:35 ts/train.py:232 step:32K smpl:256K ep:1K epch:114.08 loss:0.072 grdn:9.532 lr:1.0e-05 updt_s:0.071 data_s:0.000 +INFO 2025-06-14 20:48:49 ts/train.py:232 step:32K smpl:258K ep:1K epch:114.80 loss:0.071 grdn:9.071 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:49:03 ts/train.py:232 step:32K smpl:259K ep:1K epch:115.51 loss:0.071 grdn:9.304 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:49:18 ts/train.py:232 step:33K smpl:261K ep:1K epch:116.22 loss:0.071 grdn:9.134 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:49:32 ts/train.py:232 step:33K smpl:262K ep:1K epch:116.93 loss:0.071 grdn:9.155 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:49:46 ts/train.py:232 step:33K smpl:264K ep:1K epch:117.65 loss:0.071 grdn:9.559 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:50:01 ts/train.py:232 step:33K smpl:266K ep:1K epch:118.36 loss:0.071 grdn:9.151 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:50:15 ts/train.py:232 step:33K smpl:267K ep:1K epch:119.07 loss:0.071 grdn:9.337 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:50:29 ts/train.py:232 step:34K smpl:269K ep:1K epch:119.79 loss:0.070 grdn:8.993 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:50:44 ts/train.py:232 step:34K smpl:270K ep:1K epch:120.50 loss:0.069 grdn:9.001 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:50:58 ts/train.py:232 step:34K smpl:272K ep:1K epch:121.21 loss:0.068 grdn:9.136 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:51:13 ts/train.py:232 step:34K smpl:274K ep:1K epch:121.93 loss:0.068 grdn:9.193 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:51:27 ts/train.py:232 step:34K smpl:275K ep:1K epch:122.64 loss:0.069 grdn:9.335 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:51:42 ts/train.py:232 step:35K smpl:277K ep:1K epch:123.35 loss:0.068 grdn:9.611 lr:1.0e-05 updt_s:0.071 data_s:0.002 +INFO 2025-06-14 20:51:56 ts/train.py:232 step:35K smpl:278K ep:1K epch:124.06 loss:0.068 grdn:9.089 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:52:10 ts/train.py:232 step:35K smpl:280K ep:1K epch:124.78 loss:0.068 grdn:9.464 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:52:24 ts/train.py:232 step:35K smpl:282K ep:1K epch:125.49 loss:0.068 grdn:8.916 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:52:38 ts/train.py:232 step:35K smpl:283K ep:1K epch:126.20 loss:0.067 grdn:8.464 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:52:52 ts/train.py:232 step:36K smpl:285K ep:1K epch:126.92 loss:0.067 grdn:9.242 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:53:07 ts/train.py:232 step:36K smpl:286K ep:1K epch:127.63 loss:0.068 grdn:8.885 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:53:21 ts/train.py:232 step:36K smpl:288K ep:1K epch:128.34 loss:0.065 grdn:8.411 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-14 20:53:35 ts/train.py:232 step:36K smpl:290K ep:1K epch:129.06 loss:0.065 grdn:9.023 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:53:49 ts/train.py:232 step:36K smpl:291K ep:1K epch:129.77 loss:0.066 grdn:8.840 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:54:04 ts/train.py:232 step:37K smpl:293K ep:1K epch:130.48 loss:0.065 grdn:8.796 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:54:18 ts/train.py:232 step:37K smpl:294K ep:1K epch:131.19 loss:0.066 grdn:9.151 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:54:32 ts/train.py:232 step:37K smpl:296K ep:1K epch:131.91 loss:0.065 grdn:8.574 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:54:46 ts/train.py:232 step:37K smpl:298K ep:1K epch:132.62 loss:0.065 grdn:8.830 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:55:00 ts/train.py:232 step:37K smpl:299K ep:1K epch:133.33 loss:0.064 grdn:8.787 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:55:14 ts/train.py:232 step:38K smpl:301K ep:1K epch:134.05 loss:0.066 grdn:8.757 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:55:29 ts/train.py:232 step:38K smpl:302K ep:1K epch:134.76 loss:0.064 grdn:8.601 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:55:43 ts/train.py:232 step:38K smpl:304K ep:1K epch:135.47 loss:0.064 grdn:8.619 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:55:57 ts/train.py:232 step:38K smpl:306K ep:1K epch:136.19 loss:0.064 grdn:8.578 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:56:11 ts/train.py:232 step:38K smpl:307K ep:1K epch:136.90 loss:0.064 grdn:8.725 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-14 20:56:26 ts/train.py:232 step:39K smpl:309K ep:1K epch:137.61 loss:0.064 grdn:8.557 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:56:40 ts/train.py:232 step:39K smpl:310K ep:1K epch:138.32 loss:0.064 grdn:8.590 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:56:54 ts/train.py:232 step:39K smpl:312K ep:1K epch:139.04 loss:0.063 grdn:8.656 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:57:08 ts/train.py:232 step:39K smpl:314K ep:1K epch:139.75 loss:0.063 grdn:8.554 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:57:22 ts/train.py:232 step:39K smpl:315K ep:1K epch:140.46 loss:0.061 grdn:7.760 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 20:57:36 ts/train.py:232 step:40K smpl:317K ep:1K epch:141.18 loss:0.062 grdn:8.565 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:57:51 ts/train.py:232 step:40K smpl:318K ep:1K epch:141.89 loss:0.064 grdn:8.273 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:58:05 ts/train.py:232 step:40K smpl:320K ep:1K epch:142.60 loss:0.062 grdn:8.144 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-14 20:58:05 ts/train.py:241 Checkpoint policy after step 40000 +INFO 2025-06-14 20:58:20 ts/train.py:232 step:40K smpl:322K ep:1K epch:143.32 loss:0.063 grdn:8.797 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:58:35 ts/train.py:232 step:40K smpl:323K ep:1K epch:144.03 loss:0.061 grdn:8.157 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-14 20:58:49 ts/train.py:232 step:41K smpl:325K ep:1K epch:144.74 loss:0.063 grdn:8.521 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:59:03 ts/train.py:232 step:41K smpl:326K ep:1K epch:145.45 loss:0.062 grdn:8.052 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 20:59:17 ts/train.py:232 step:41K smpl:328K ep:1K epch:146.17 loss:0.060 grdn:8.172 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 20:59:31 ts/train.py:232 step:41K smpl:330K ep:1K epch:146.88 loss:0.061 grdn:8.706 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-14 20:59:46 ts/train.py:232 step:41K smpl:331K ep:1K epch:147.59 loss:0.062 grdn:8.565 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-14 21:00:00 ts/train.py:232 step:42K smpl:333K ep:1K epch:148.31 loss:0.060 grdn:8.123 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 21:00:14 ts/train.py:232 step:42K smpl:334K ep:1K epch:149.02 loss:0.060 grdn:8.211 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-14 21:00:28 ts/train.py:232 step:42K smpl:336K ep:1K epch:149.73 loss:0.062 grdn:8.367 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-14 21:00:43 ts/train.py:232 step:42K smpl:338K ep:1K epch:150.45 loss:0.060 grdn:8.225 lr:1.0e-05 updt_s:0.073 data_s:0.002 +Traceback (most recent call last): + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/configs/parser.py", line 226, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 212, in train + train_tracker, output_dict = update_policy( + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 71, in update_policy + loss, output_dict = policy.forward(batch) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 153, in forward + actions_hat, (mu_hat, log_sigma_x2_hat) = self.model(batch) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 452, in forward + cls_token_out = self.vae_encoder( + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 546, in forward + x = layer(x, pos_embed=pos_embed, key_padding_mask=key_padding_mask) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 584, in forward + x = skip + self.dropout2(x) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/dropout.py", line 70, in forward + return F.dropout(input, self.p, self.training, self.inplace) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/functional.py", line 1425, in dropout + _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training) +KeyboardInterrupt +Traceback (most recent call last): + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/configs/parser.py", line 226, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 212, in train + train_tracker, output_dict = update_policy( + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 71, in update_policy + loss, output_dict = policy.forward(batch) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 153, in forward + actions_hat, (mu_hat, log_sigma_x2_hat) = self.model(batch) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 452, in forward + cls_token_out = self.vae_encoder( + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 546, in forward + x = layer(x, pos_embed=pos_embed, key_padding_mask=key_padding_mask) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 584, in forward + x = skip + self.dropout2(x) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/dropout.py", line 70, in forward + return F.dropout(input, self.p, self.training, self.inplace) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/functional.py", line 1425, in dropout + _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training) +KeyboardInterrupt diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/requirements.txt b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f3a5ff904f10fb0d11caed35d12b4260e408e2a6 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/requirements.txt @@ -0,0 +1,167 @@ +starlette==0.46.2 +httpx-sse==0.4.0 +regex==2024.11.6 +markdown-it-py==3.0.0 +mdurl==0.1.2 +groovy==0.1.2 +tqdm==4.67.1 +joblib==1.5.1 +httpcore==1.0.9 +typer==0.16.0 +pydantic_core==2.33.2 +typing_extensions==4.14.0 +mcp==1.9.0 +aiofiles==24.1.0 +ruff==0.11.13 +packaging==25.0 +pydantic-settings==2.9.1 +exceptiongroup==1.3.0 +pandas==2.3.0 +Jinja2==3.1.6 +nltk==3.9.1 +websockets==15.0.1 +tzdata==2025.2 +semantic-version==2.10.0 +fastapi==0.115.12 +orjson==3.10.18 +textblob==0.19.0 +h11==0.16.0 +python-dateutil==2.9.0.post0 +pydub==0.25.1 +sse-starlette==2.3.6 +tomlkit==0.13.3 +Pygments==2.19.1 +anyio==4.9.0 +shellingham==1.5.4 +typing-inspection==0.4.1 +pydantic==2.11.5 +rich==14.0.0 +numpy==2.2.6 +uvicorn==0.34.3 +huggingface-hub==0.32.4 +annotated-types==0.7.0 +python-dotenv==1.1.0 +safehttpx==0.1.6 +ffmpy==0.6.0 +httpx==0.28.1 +gradio_client==1.10.2 +python-multipart==0.0.20 +gradio==5.33.0 +filelock==3.18.0 +hf-xet==1.1.3 +sniffio==1.3.1 +itsdangerous==2.2.0 +toml==0.10.2 +multidict==6.4.4 +mpmath==1.3.0 +protobuf==6.31.1 +cloudpickle==3.1.1 +torchvision==0.22.1 +prompt_toolkit==3.0.51 +blinker==1.9.0 +PySocks==1.7.1 +beautifulsoup4==4.13.4 +nvidia-cuda-runtime-cu12==12.6.77 +sentry-sdk==2.30.0 +nvidia-cuda-cupti-cu12==12.6.80 +h5py==3.14.0 +aiohttp==3.12.13 +gitdb==4.0.12 +setproctitle==1.3.6 +GitPython==3.1.44 +six==1.17.0 +einops==0.8.1 +Farama-Notifications==0.0.4 +diffusers==0.33.1 +cffi==1.17.1 +pyarrow==20.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +zipp==3.23.0 +pycparser==2.22 +nvidia-nccl-cu12==2.26.2 +zarr==2.18.3 +antlr4-python3-runtime==4.9.3 +numcodecs==0.13.1 +MarkupSafe==3.0.2 +pip==25.1 +mergedeep==1.3.4 +certifi==2025.6.15 +soupsieve==2.7 +numba==0.61.2 +safetensors==0.5.3 +gymnasium==0.29.1 +multiprocess==0.70.16 +idna==3.10 +orderly-set==5.4.1 +networkx==3.4.2 +sympy==1.14.0 +xxhash==3.5.0 +pillow==11.2.1 +draccus==0.10.0 +platformdirs==4.3.8 +pyserial==3.5 +requests==2.32.4 +fasteners==0.19 +pyzmq==27.0.0 +nvidia-cudnn-cu12==9.5.1.17 +torch==2.7.1 +nvidia-cusparse-cu12==12.5.4.2 +psutil==7.0.0 +scipy==1.15.3 +async-timeout==5.0.1 +urllib3==2.4.0 +nvidia-curand-cu12==10.3.7.77 +av==14.4.0 +attrs==25.3.0 +smmap==5.0.2 +wandb==0.20.1 +hf_transfer==0.1.9 +propcache==0.3.2 +setuptools==78.1.1 +imageio-ffmpeg==0.6.0 +pynput==1.8.1 +pytz==2025.2 +pfzy==0.3.4 +torchcodec==0.4.0 +imageio==2.37.0 +nvidia-cublas-cu12==12.6.4.1 +triton==3.3.1 +pymunk==6.11.1 +wcwidth==0.2.13 +jsonlines==4.0.0 +python-xlib==0.33 +mypy_extensions==1.1.0 +cmake==4.0.3 +evdev==1.9.2 +omegaconf==2.3.0 +asciitree==0.3.3 +nvidia-cufft-cu12==11.3.0.4 +charset-normalizer==3.4.2 +deepdiff==8.5.0 +click==8.2.1 +termcolor==3.1.0 +rerun-sdk==0.23.3 +aiosignal==1.3.2 +llvmlite==0.44.0 +PyYAML==6.0.2 +importlib_metadata==8.7.0 +lerobot==0.1.0 +frozenlist==1.7.0 +Werkzeug==3.1.3 +nvidia-nvjitlink-cu12==12.6.85 +typing-inspect==0.9.0 +nvidia-cusolver-cu12==11.7.1.2 +dill==0.3.8 +pyyaml-include==1.4.1 +gdown==5.2.0 +datasets==3.6.0 +fsspec==2025.3.0 +yarl==1.20.1 +inquirerpy==0.3.4 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.1 +aiohappyeyeballs==2.6.1 +opencv-python-headless==4.11.0.86 +wheel==0.45.1 +nvidia-cufile-cu12==1.11.1.6 +nvidia-cusparselt-cu12==0.6.3 diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-metadata.json b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6082505ce19c1edfa3316a425884b5ee80cf1636 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-metadata.json @@ -0,0 +1,51 @@ +{ + "os": "Linux-6.8.0-60-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.13", + "startedAt": "2025-06-15T03:09:51.326969Z", + "args": [ + "--dataset.repo_id=BoredMachinist/dice-art-run1", + "--dataset.video_backend=pyav", + "--policy.type=act", + "--output_dir=outputs/train/act_dice_art_run1", + "--job_name=act_dice_art_run1", + "--policy.device=cuda", + "--wandb.enable=true" + ], + "program": "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "b8637c09ec93ca30d7901f5fdcf47aa4020cfe89" + }, + "root": "outputs/train/act_dice_art_run1", + "host": "hongyi-J", + "executable": "/home/hshi-j-4090/miniconda3/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 24, + "cpu_count_logical": 32, + "gpu": "NVIDIA GeForce RTX 4090", + "gpu_count": 1, + "disk": { + "/": { + "total": "3739429154816", + "used": "152715747328" + } + }, + "memory": { + "total": "67168518144" + }, + "cpu": { + "count": 24, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 4090", + "memoryTotal": "25757220864", + "cudaCores": 16384, + "architecture": "Ada", + "uuid": "GPU-5b318751-a95f-5805-6c0e-a4b86dcd442d" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-summary.json b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c2fb8f7852a733d0e9d25699e55b27d57f302e55 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/steps":42200,"train/episodes":1354.0106951871658,"_timestamp":1.7499600435914388e+09,"train/kld_loss":0.0003022812306880951,"_runtime":3059.972341236,"train/samples":337600,"train/grad_norm":8.225247728824616,"train/l1_loss":0.05051584169268608,"train/dataloading_s":0.0016573248049689938,"_step":42200,"train/update_s":0.07279860169501717,"train/loss":0.059818110205233095,"train/lr":1.0000000000000021e-05,"_wandb":{"runtime":3059},"train/epochs":150.44563279857397} \ No newline at end of file diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..7942bb46e19877d66a16d9be1efe182dc853f968 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log @@ -0,0 +1,11 @@ +{"time":"2025-06-14T20:09:51.124778953-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_d1l9fr6/port-11388.txt","pid":11388,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-06-14T20:09:51.126955456-07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37013,"Zone":""}} +{"time":"2025-06-14T20:09:51.127005411-07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":11388} +{"time":"2025-06-14T20:09:51.318865438-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:41090"} +{"time":"2025-06-14T20:09:51.330293507-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"fhdgxxjz","id":"127.0.0.1:41090"} +{"time":"2025-06-14T20:09:51.792331002-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fhdgxxjz","id":"127.0.0.1:41090"} +{"time":"2025-06-14T21:00:51.299091378-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:41090"} +{"time":"2025-06-14T21:00:51.299209042-07:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-06-14T21:00:51.299173348-07:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:41090"} +{"time":"2025-06-14T21:00:51.29937174-07:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:41090"} +{"time":"2025-06-14T21:00:52.297381201-07:00","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..edc1f0f61f17008eb258703ddba08f120aad3c01 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-06-14T20:09:51.33052201-07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"outputs/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log"} +{"time":"2025-06-14T20:09:51.792313834-07:00","level":"INFO","msg":"stream: created new stream","id":"fhdgxxjz"} +{"time":"2025-06-14T20:09:51.792328297-07:00","level":"INFO","msg":"stream: started","id":"fhdgxxjz"} +{"time":"2025-06-14T20:09:51.792427231-07:00","level":"INFO","msg":"sender: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-14T20:09:51.792424884-07:00","level":"INFO","msg":"writer: Do: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-14T20:09:51.792446588-07:00","level":"INFO","msg":"handler: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-14T20:09:52.128559874-07:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-06-14T21:00:51.299276318-07:00","level":"INFO","msg":"stream: closing","id":"fhdgxxjz"} +{"time":"2025-06-14T21:00:51.299350521-07:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-06-14T21:00:51.299409681-07:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-06-14T21:00:51.872219071-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a95255bb47238095950f27779843bc9d4de105d6 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log @@ -0,0 +1,886 @@ +2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1 +2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Configure stats pid to 11388 +2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/.config/wandb/settings +2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/wandb/settings +2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log +2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log +2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:init():831] calling init triggers +2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:init():836] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'BoredMachinist/dice-art-run1', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_dice_art_run1', 'job_name': 'act_dice_art_run1', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:init():872] starting backend +2025-06-14 20:09:51,319 INFO MainThread:11388 [wandb_init.py:init():875] sending inform_init request +2025-06-14 20:09:51,326 INFO MainThread:11388 [wandb_init.py:init():883] backend started and connected +2025-06-14 20:09:51,331 INFO MainThread:11388 [wandb_init.py:init():956] updated telemetry +2025-06-14 20:09:51,343 INFO MainThread:11388 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout +2025-06-14 20:09:52,123 INFO MainThread:11388 [wandb_init.py:init():1032] starting run threads in backend +2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_console_start():2453] atexit reg +2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_redirect():2301] redirect: wrap_raw +2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_redirect():2370] Wrapping output streams. +2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_redirect():2393] Redirects installed. +2025-06-14 20:09:52,193 INFO MainThread:11388 [wandb_init.py:init():1078] run started, returning control to user process +2025-06-14 21:00:51,298 INFO MsgRouterThr:11388 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. +2025-06-14 21:00:51,942 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,946 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,946 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,948 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,949 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,950 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,950 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,950 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,951 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,951 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,952 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,952 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,953 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,954 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,955 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,955 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,955 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,956 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,957 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,957 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,957 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,958 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,959 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,959 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,959 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,960 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,960 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,960 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe +2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback +Traceback (most recent call last): + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in + lambda data: self._console_raw_callback("stderr", data), + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn + return func(self, *args, **kwargs) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw + self._publish_output_raw(o) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) +BrokenPipeError: [Errno 32] Broken pipe diff --git a/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ef11271b2fb6fb064cbfefb57beda9894bb724c9 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72581254d769e402eaf31adb31ea26d538f83dc36f07ad52a2b7d640ed4eec45 +size 425984 diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/config.yaml b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce444c6682a354260ae5d26a7387ef594bc1eb65 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/config.yaml @@ -0,0 +1,186 @@ +_wandb: + value: + cli_version: 0.20.1 + m: [] + python_version: 3.10.13 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 105 + "2": + - 1 + - 41 + - 49 + - 51 + - 105 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 55 + - 61 + - 62 + "4": 3.10.13 + "5": 0.20.1 + "10": + - 21 + "12": 0.20.1 + "13": linux-x86_64 +batch_size: + value: 8 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: BoredMachinist/dice-art-run1 + revision: null + root: null + use_imagenet_stats: true + video_backend: pyav +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: act_dice_art_run1 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/act_dice_art_run1 +policy: + value: + chunk_size: 100 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + input_features: + observation.images.arm: + shape: + - 3 + - 480 + - 640 + type: VISUAL + observation.images.world: + shape: + - 3 + - 480 + - 640 + type: VISUAL + observation.state: + shape: + - 6 + type: STATE + kl_weight: 10 + latent_dim: 32 + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + output_features: + action: + shape: + - 6 + type: ACTION + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + replace_final_stride_with_dilation: 0 + temporal_ensemble_coeff: null + type: act + use_amp: false + use_vae: true + vision_backbone: resnet18 +resume: + value: true +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: null +seed: + value: 1000 +steps: + value: 100000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: fhdgxxjz diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/output.log b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..112622742a9db2b89328c4ec289a5aff56d258de --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/output.log @@ -0,0 +1,317 @@ +Logs will be synced with wandb. +INFO 2025-06-15 01:25:27 db_utils.py:103 Track this run --> https://wandb.ai/ametind-o/lerobot/runs/fhdgxxjz +INFO 2025-06-15 01:25:27 ts/train.py:127 Creating dataset +INFO 2025-06-15 01:25:28 ts/train.py:138 Creating policy +Loading weights from local directory +INFO 2025-06-15 01:25:28 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-06-15 01:25:29 ts/train.py:156 Output dir: outputs/train/act_dice_art_run1 +INFO 2025-06-15 01:25:29 ts/train.py:159 cfg.steps=100000 (100K) +INFO 2025-06-15 01:25:29 ts/train.py:160 dataset.num_frames=2244 (2K) +INFO 2025-06-15 01:25:29 ts/train.py:161 dataset.num_episodes=9 +INFO 2025-06-15 01:25:29 ts/train.py:162 num_learnable_params=51597190 (52M) +INFO 2025-06-15 01:25:29 ts/train.py:163 num_total_params=51597238 (52M) +INFO 2025-06-15 01:25:29 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-06-15 01:25:46 ts/train.py:232 step:40K smpl:322K ep:1K epch:143.32 loss:0.062 grdn:8.178 lr:1.0e-05 updt_s:0.084 data_s:0.002 +INFO 2025-06-15 01:26:02 ts/train.py:232 step:40K smpl:323K ep:1K epch:144.03 loss:0.062 grdn:8.049 lr:1.0e-05 updt_s:0.077 data_s:0.002 +INFO 2025-06-15 01:26:17 ts/train.py:232 step:41K smpl:325K ep:1K epch:144.74 loss:0.061 grdn:7.868 lr:1.0e-05 updt_s:0.075 data_s:0.001 +INFO 2025-06-15 01:26:32 ts/train.py:232 step:41K smpl:326K ep:1K epch:145.45 loss:0.060 grdn:8.094 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-15 01:26:46 ts/train.py:232 step:41K smpl:328K ep:1K epch:146.17 loss:0.063 grdn:8.501 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-15 01:27:02 ts/train.py:232 step:41K smpl:330K ep:1K epch:146.88 loss:0.060 grdn:7.931 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-15 01:27:16 ts/train.py:232 step:41K smpl:331K ep:1K epch:147.59 loss:0.060 grdn:8.136 lr:1.0e-05 updt_s:0.073 data_s:0.000 +INFO 2025-06-15 01:27:31 ts/train.py:232 step:42K smpl:333K ep:1K epch:148.31 loss:0.060 grdn:8.360 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:27:45 ts/train.py:232 step:42K smpl:334K ep:1K epch:149.02 loss:0.062 grdn:8.511 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-15 01:28:01 ts/train.py:232 step:42K smpl:336K ep:1K epch:149.73 loss:0.060 grdn:8.141 lr:1.0e-05 updt_s:0.074 data_s:0.002 +INFO 2025-06-15 01:28:15 ts/train.py:232 step:42K smpl:338K ep:1K epch:150.45 loss:0.059 grdn:7.811 lr:1.0e-05 updt_s:0.072 data_s:0.000 +INFO 2025-06-15 01:28:30 ts/train.py:232 step:42K smpl:339K ep:1K epch:151.16 loss:0.059 grdn:7.760 lr:1.0e-05 updt_s:0.073 data_s:0.002 +INFO 2025-06-15 01:28:45 ts/train.py:232 step:43K smpl:341K ep:1K epch:151.87 loss:0.061 grdn:8.595 lr:1.0e-05 updt_s:0.072 data_s:0.002 +INFO 2025-06-15 01:28:59 ts/train.py:232 step:43K smpl:342K ep:1K epch:152.58 loss:0.059 grdn:8.167 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:29:13 ts/train.py:232 step:43K smpl:344K ep:1K epch:153.30 loss:0.060 grdn:8.204 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:29:28 ts/train.py:232 step:43K smpl:346K ep:1K epch:154.01 loss:0.058 grdn:7.866 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:29:42 ts/train.py:232 step:43K smpl:347K ep:1K epch:154.72 loss:0.059 grdn:8.584 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:29:56 ts/train.py:232 step:44K smpl:349K ep:1K epch:155.44 loss:0.059 grdn:7.636 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:30:10 ts/train.py:232 step:44K smpl:350K ep:1K epch:156.15 loss:0.059 grdn:8.275 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:30:25 ts/train.py:232 step:44K smpl:352K ep:1K epch:156.86 loss:0.058 grdn:8.258 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:30:39 ts/train.py:232 step:44K smpl:354K ep:1K epch:157.58 loss:0.059 grdn:8.812 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:30:53 ts/train.py:232 step:44K smpl:355K ep:1K epch:158.29 loss:0.059 grdn:8.157 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:31:07 ts/train.py:232 step:45K smpl:357K ep:1K epch:159.00 loss:0.059 grdn:7.981 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:31:22 ts/train.py:232 step:45K smpl:358K ep:1K epch:159.71 loss:0.058 grdn:8.005 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:31:36 ts/train.py:232 step:45K smpl:360K ep:1K epch:160.43 loss:0.057 grdn:8.034 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:31:50 ts/train.py:232 step:45K smpl:362K ep:1K epch:161.14 loss:0.057 grdn:7.830 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:32:04 ts/train.py:232 step:45K smpl:363K ep:1K epch:161.85 loss:0.057 grdn:8.148 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:32:18 ts/train.py:232 step:46K smpl:365K ep:1K epch:162.57 loss:0.057 grdn:7.569 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:32:33 ts/train.py:232 step:46K smpl:366K ep:1K epch:163.28 loss:0.056 grdn:7.397 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:32:47 ts/train.py:232 step:46K smpl:368K ep:1K epch:163.99 loss:0.058 grdn:8.293 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:33:01 ts/train.py:232 step:46K smpl:370K ep:1K epch:164.71 loss:0.056 grdn:7.867 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:33:15 ts/train.py:232 step:46K smpl:371K ep:1K epch:165.42 loss:0.056 grdn:7.686 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:33:29 ts/train.py:232 step:47K smpl:373K ep:1K epch:166.13 loss:0.056 grdn:7.390 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:33:44 ts/train.py:232 step:47K smpl:374K ep:2K epch:166.84 loss:0.057 grdn:7.584 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:33:58 ts/train.py:232 step:47K smpl:376K ep:2K epch:167.56 loss:0.056 grdn:7.881 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:34:12 ts/train.py:232 step:47K smpl:378K ep:2K epch:168.27 loss:0.055 grdn:7.482 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:34:27 ts/train.py:232 step:47K smpl:379K ep:2K epch:168.98 loss:0.056 grdn:7.879 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:34:41 ts/train.py:232 step:48K smpl:381K ep:2K epch:169.70 loss:0.056 grdn:8.160 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:34:55 ts/train.py:232 step:48K smpl:382K ep:2K epch:170.41 loss:0.056 grdn:8.133 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:35:09 ts/train.py:232 step:48K smpl:384K ep:2K epch:171.12 loss:0.055 grdn:7.320 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:35:23 ts/train.py:232 step:48K smpl:386K ep:2K epch:171.84 loss:0.054 grdn:7.045 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:35:37 ts/train.py:232 step:48K smpl:387K ep:2K epch:172.55 loss:0.055 grdn:7.458 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:35:52 ts/train.py:232 step:49K smpl:389K ep:2K epch:173.26 loss:0.056 grdn:8.055 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:36:06 ts/train.py:232 step:49K smpl:390K ep:2K epch:173.98 loss:0.054 grdn:7.239 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:36:20 ts/train.py:232 step:49K smpl:392K ep:2K epch:174.69 loss:0.055 grdn:7.588 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:36:34 ts/train.py:232 step:49K smpl:394K ep:2K epch:175.40 loss:0.054 grdn:7.054 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:36:48 ts/train.py:232 step:49K smpl:395K ep:2K epch:176.11 loss:0.054 grdn:7.456 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:37:03 ts/train.py:232 step:50K smpl:397K ep:2K epch:176.83 loss:0.053 grdn:7.103 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:37:17 ts/train.py:232 step:50K smpl:398K ep:2K epch:177.54 loss:0.055 grdn:7.994 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:37:31 ts/train.py:232 step:50K smpl:400K ep:2K epch:178.25 loss:0.054 grdn:7.748 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:37:45 ts/train.py:232 step:50K smpl:402K ep:2K epch:178.97 loss:0.054 grdn:7.213 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:37:59 ts/train.py:232 step:50K smpl:403K ep:2K epch:179.68 loss:0.052 grdn:7.467 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:38:13 ts/train.py:232 step:51K smpl:405K ep:2K epch:180.39 loss:0.054 grdn:7.982 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:38:28 ts/train.py:232 step:51K smpl:406K ep:2K epch:181.11 loss:0.053 grdn:7.853 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:38:42 ts/train.py:232 step:51K smpl:408K ep:2K epch:181.82 loss:0.053 grdn:7.114 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:38:56 ts/train.py:232 step:51K smpl:410K ep:2K epch:182.53 loss:0.054 grdn:7.753 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:39:10 ts/train.py:232 step:51K smpl:411K ep:2K epch:183.24 loss:0.054 grdn:7.507 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:39:24 ts/train.py:232 step:52K smpl:413K ep:2K epch:183.96 loss:0.053 grdn:7.477 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:39:38 ts/train.py:232 step:52K smpl:414K ep:2K epch:184.67 loss:0.053 grdn:7.647 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:39:53 ts/train.py:232 step:52K smpl:416K ep:2K epch:185.38 loss:0.052 grdn:6.969 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:40:07 ts/train.py:232 step:52K smpl:418K ep:2K epch:186.10 loss:0.053 grdn:7.590 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:40:21 ts/train.py:232 step:52K smpl:419K ep:2K epch:186.81 loss:0.053 grdn:7.318 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:40:35 ts/train.py:232 step:53K smpl:421K ep:2K epch:187.52 loss:0.052 grdn:7.186 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:40:49 ts/train.py:232 step:53K smpl:422K ep:2K epch:188.24 loss:0.053 grdn:7.403 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:41:04 ts/train.py:232 step:53K smpl:424K ep:2K epch:188.95 loss:0.051 grdn:7.168 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:41:18 ts/train.py:232 step:53K smpl:426K ep:2K epch:189.66 loss:0.052 grdn:7.206 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:41:32 ts/train.py:232 step:53K smpl:427K ep:2K epch:190.37 loss:0.052 grdn:7.283 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:41:46 ts/train.py:232 step:54K smpl:429K ep:2K epch:191.09 loss:0.052 grdn:7.463 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:42:00 ts/train.py:232 step:54K smpl:430K ep:2K epch:191.80 loss:0.051 grdn:7.277 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:42:14 ts/train.py:232 step:54K smpl:432K ep:2K epch:192.51 loss:0.052 grdn:7.309 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:42:29 ts/train.py:232 step:54K smpl:434K ep:2K epch:193.23 loss:0.051 grdn:6.976 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:42:43 ts/train.py:232 step:54K smpl:435K ep:2K epch:193.94 loss:0.051 grdn:7.092 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:42:57 ts/train.py:232 step:55K smpl:437K ep:2K epch:194.65 loss:0.050 grdn:6.810 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:43:11 ts/train.py:232 step:55K smpl:438K ep:2K epch:195.37 loss:0.051 grdn:7.106 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:43:25 ts/train.py:232 step:55K smpl:440K ep:2K epch:196.08 loss:0.051 grdn:6.918 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:43:40 ts/train.py:232 step:55K smpl:442K ep:2K epch:196.79 loss:0.050 grdn:6.835 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:43:54 ts/train.py:232 step:55K smpl:443K ep:2K epch:197.50 loss:0.050 grdn:6.746 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:44:08 ts/train.py:232 step:56K smpl:445K ep:2K epch:198.22 loss:0.051 grdn:7.479 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:44:22 ts/train.py:232 step:56K smpl:446K ep:2K epch:198.93 loss:0.051 grdn:7.710 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:44:36 ts/train.py:232 step:56K smpl:448K ep:2K epch:199.64 loss:0.049 grdn:6.914 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:44:50 ts/train.py:232 step:56K smpl:450K ep:2K epch:200.36 loss:0.051 grdn:7.331 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:45:05 ts/train.py:232 step:56K smpl:451K ep:2K epch:201.07 loss:0.050 grdn:6.918 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:45:19 ts/train.py:232 step:57K smpl:453K ep:2K epch:201.78 loss:0.050 grdn:6.781 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:45:33 ts/train.py:232 step:57K smpl:454K ep:2K epch:202.50 loss:0.049 grdn:6.989 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:45:47 ts/train.py:232 step:57K smpl:456K ep:2K epch:203.21 loss:0.050 grdn:7.546 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:46:01 ts/train.py:232 step:57K smpl:458K ep:2K epch:203.92 loss:0.050 grdn:7.045 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:46:15 ts/train.py:232 step:57K smpl:459K ep:2K epch:204.63 loss:0.050 grdn:7.347 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:46:30 ts/train.py:232 step:58K smpl:461K ep:2K epch:205.35 loss:0.050 grdn:7.318 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:46:44 ts/train.py:232 step:58K smpl:462K ep:2K epch:206.06 loss:0.049 grdn:7.083 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:46:58 ts/train.py:232 step:58K smpl:464K ep:2K epch:206.77 loss:0.049 grdn:6.753 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:47:12 ts/train.py:232 step:58K smpl:466K ep:2K epch:207.49 loss:0.049 grdn:7.162 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:47:26 ts/train.py:232 step:58K smpl:467K ep:2K epch:208.20 loss:0.049 grdn:7.053 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:47:41 ts/train.py:232 step:59K smpl:469K ep:2K epch:208.91 loss:0.049 grdn:6.821 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:47:55 ts/train.py:232 step:59K smpl:470K ep:2K epch:209.63 loss:0.048 grdn:6.729 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:48:09 ts/train.py:232 step:59K smpl:472K ep:2K epch:210.34 loss:0.048 grdn:7.147 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:48:23 ts/train.py:232 step:59K smpl:474K ep:2K epch:211.05 loss:0.050 grdn:6.870 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:48:38 ts/train.py:232 step:59K smpl:475K ep:2K epch:211.76 loss:0.047 grdn:6.563 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:48:51 ts/train.py:232 step:60K smpl:477K ep:2K epch:212.48 loss:0.048 grdn:6.476 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:49:06 ts/train.py:232 step:60K smpl:478K ep:2K epch:213.19 loss:0.047 grdn:6.635 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:49:20 ts/train.py:232 step:60K smpl:480K ep:2K epch:213.90 loss:0.047 grdn:6.311 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:49:20 ts/train.py:241 Checkpoint policy after step 60000 +INFO 2025-06-15 01:49:35 ts/train.py:232 step:60K smpl:482K ep:2K epch:214.62 loss:0.048 grdn:6.895 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:49:49 ts/train.py:232 step:60K smpl:483K ep:2K epch:215.33 loss:0.048 grdn:6.920 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:50:04 ts/train.py:232 step:61K smpl:485K ep:2K epch:216.04 loss:0.047 grdn:6.346 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:50:18 ts/train.py:232 step:61K smpl:486K ep:2K epch:216.76 loss:0.048 grdn:6.918 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:50:32 ts/train.py:232 step:61K smpl:488K ep:2K epch:217.47 loss:0.048 grdn:6.664 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:50:46 ts/train.py:232 step:61K smpl:490K ep:2K epch:218.18 loss:0.046 grdn:6.951 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:51:01 ts/train.py:232 step:61K smpl:491K ep:2K epch:218.89 loss:0.048 grdn:7.243 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:51:15 ts/train.py:232 step:62K smpl:493K ep:2K epch:219.61 loss:0.048 grdn:6.737 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:51:29 ts/train.py:232 step:62K smpl:494K ep:2K epch:220.32 loss:0.046 grdn:6.675 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:51:43 ts/train.py:232 step:62K smpl:496K ep:2K epch:221.03 loss:0.047 grdn:6.878 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:51:57 ts/train.py:232 step:62K smpl:498K ep:2K epch:221.75 loss:0.047 grdn:6.595 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:52:11 ts/train.py:232 step:62K smpl:499K ep:2K epch:222.46 loss:0.048 grdn:6.651 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:52:26 ts/train.py:232 step:63K smpl:501K ep:2K epch:223.17 loss:0.048 grdn:7.065 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:52:40 ts/train.py:232 step:63K smpl:502K ep:2K epch:223.89 loss:0.047 grdn:6.398 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:52:54 ts/train.py:232 step:63K smpl:504K ep:2K epch:224.60 loss:0.046 grdn:6.428 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:53:08 ts/train.py:232 step:63K smpl:506K ep:2K epch:225.31 loss:0.048 grdn:6.853 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:53:22 ts/train.py:232 step:63K smpl:507K ep:2K epch:226.02 loss:0.045 grdn:6.491 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:53:36 ts/train.py:232 step:64K smpl:509K ep:2K epch:226.74 loss:0.046 grdn:6.330 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:53:51 ts/train.py:232 step:64K smpl:510K ep:2K epch:227.45 loss:0.047 grdn:7.094 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:54:05 ts/train.py:232 step:64K smpl:512K ep:2K epch:228.16 loss:0.046 grdn:6.896 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:54:19 ts/train.py:232 step:64K smpl:514K ep:2K epch:228.88 loss:0.046 grdn:6.510 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:54:33 ts/train.py:232 step:64K smpl:515K ep:2K epch:229.59 loss:0.047 grdn:6.431 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:54:47 ts/train.py:232 step:65K smpl:517K ep:2K epch:230.30 loss:0.046 grdn:6.902 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:55:02 ts/train.py:232 step:65K smpl:518K ep:2K epch:231.02 loss:0.046 grdn:7.001 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:55:16 ts/train.py:232 step:65K smpl:520K ep:2K epch:231.73 loss:0.045 grdn:6.760 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:55:30 ts/train.py:232 step:65K smpl:522K ep:2K epch:232.44 loss:0.046 grdn:6.784 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:55:44 ts/train.py:232 step:65K smpl:523K ep:2K epch:233.16 loss:0.046 grdn:6.602 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:55:59 ts/train.py:232 step:66K smpl:525K ep:2K epch:233.87 loss:0.045 grdn:6.246 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:56:13 ts/train.py:232 step:66K smpl:526K ep:2K epch:234.58 loss:0.045 grdn:6.372 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:56:27 ts/train.py:232 step:66K smpl:528K ep:2K epch:235.29 loss:0.045 grdn:6.747 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:56:41 ts/train.py:232 step:66K smpl:530K ep:2K epch:236.01 loss:0.046 grdn:6.872 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:56:55 ts/train.py:232 step:66K smpl:531K ep:2K epch:236.72 loss:0.045 grdn:6.459 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:57:09 ts/train.py:232 step:67K smpl:533K ep:2K epch:237.43 loss:0.044 grdn:6.369 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:57:24 ts/train.py:232 step:67K smpl:534K ep:2K epch:238.15 loss:0.046 grdn:7.069 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 01:57:38 ts/train.py:232 step:67K smpl:536K ep:2K epch:238.86 loss:0.046 grdn:6.989 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:57:52 ts/train.py:232 step:67K smpl:538K ep:2K epch:239.57 loss:0.044 grdn:6.302 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:58:06 ts/train.py:232 step:67K smpl:539K ep:2K epch:240.29 loss:0.044 grdn:6.276 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:58:20 ts/train.py:232 step:68K smpl:541K ep:2K epch:241.00 loss:0.044 grdn:6.225 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 01:58:34 ts/train.py:232 step:68K smpl:542K ep:2K epch:241.71 loss:0.044 grdn:6.253 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 01:58:49 ts/train.py:232 step:68K smpl:544K ep:2K epch:242.42 loss:0.044 grdn:6.556 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:59:03 ts/train.py:232 step:68K smpl:546K ep:2K epch:243.14 loss:0.045 grdn:6.488 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:59:17 ts/train.py:232 step:68K smpl:547K ep:2K epch:243.85 loss:0.044 grdn:6.332 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 01:59:31 ts/train.py:232 step:69K smpl:549K ep:2K epch:244.56 loss:0.044 grdn:6.291 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 01:59:45 ts/train.py:232 step:69K smpl:550K ep:2K epch:245.28 loss:0.044 grdn:6.523 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-15 02:00:00 ts/train.py:232 step:69K smpl:552K ep:2K epch:245.99 loss:0.044 grdn:6.346 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:00:14 ts/train.py:232 step:69K smpl:554K ep:2K epch:246.70 loss:0.045 grdn:6.513 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:00:28 ts/train.py:232 step:69K smpl:555K ep:2K epch:247.42 loss:0.043 grdn:6.302 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:00:42 ts/train.py:232 step:70K smpl:557K ep:2K epch:248.13 loss:0.044 grdn:6.421 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:00:56 ts/train.py:232 step:70K smpl:558K ep:2K epch:248.84 loss:0.044 grdn:6.630 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:01:10 ts/train.py:232 step:70K smpl:560K ep:2K epch:249.55 loss:0.043 grdn:6.248 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:01:25 ts/train.py:232 step:70K smpl:562K ep:2K epch:250.27 loss:0.043 grdn:6.276 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:01:39 ts/train.py:232 step:70K smpl:563K ep:2K epch:250.98 loss:0.043 grdn:6.101 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:01:53 ts/train.py:232 step:71K smpl:565K ep:2K epch:251.69 loss:0.043 grdn:6.456 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:02:07 ts/train.py:232 step:71K smpl:566K ep:2K epch:252.41 loss:0.042 grdn:6.254 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:02:22 ts/train.py:232 step:71K smpl:568K ep:2K epch:253.12 loss:0.044 grdn:6.339 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:02:36 ts/train.py:232 step:71K smpl:570K ep:2K epch:253.83 loss:0.044 grdn:6.720 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:02:50 ts/train.py:232 step:71K smpl:571K ep:2K epch:254.55 loss:0.042 grdn:6.028 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:03:04 ts/train.py:232 step:72K smpl:573K ep:2K epch:255.26 loss:0.044 grdn:6.236 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:03:18 ts/train.py:232 step:72K smpl:574K ep:2K epch:255.97 loss:0.042 grdn:6.354 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:03:32 ts/train.py:232 step:72K smpl:576K ep:2K epch:256.68 loss:0.043 grdn:6.008 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:03:47 ts/train.py:232 step:72K smpl:578K ep:2K epch:257.40 loss:0.043 grdn:6.608 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:04:01 ts/train.py:232 step:72K smpl:579K ep:2K epch:258.11 loss:0.042 grdn:5.767 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:04:15 ts/train.py:232 step:73K smpl:581K ep:2K epch:258.82 loss:0.042 grdn:6.058 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-15 02:04:29 ts/train.py:232 step:73K smpl:582K ep:2K epch:259.54 loss:0.042 grdn:5.928 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:04:43 ts/train.py:232 step:73K smpl:584K ep:2K epch:260.25 loss:0.042 grdn:6.008 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:04:58 ts/train.py:232 step:73K smpl:586K ep:2K epch:260.96 loss:0.042 grdn:6.359 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:05:12 ts/train.py:232 step:73K smpl:587K ep:2K epch:261.68 loss:0.042 grdn:6.003 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:05:26 ts/train.py:232 step:74K smpl:589K ep:2K epch:262.39 loss:0.042 grdn:5.966 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:05:40 ts/train.py:232 step:74K smpl:590K ep:2K epch:263.10 loss:0.042 grdn:6.431 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:05:54 ts/train.py:232 step:74K smpl:592K ep:2K epch:263.81 loss:0.043 grdn:6.376 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:06:08 ts/train.py:232 step:74K smpl:594K ep:2K epch:264.53 loss:0.043 grdn:6.170 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:06:23 ts/train.py:232 step:74K smpl:595K ep:2K epch:265.24 loss:0.042 grdn:6.038 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:06:37 ts/train.py:232 step:75K smpl:597K ep:2K epch:265.95 loss:0.041 grdn:5.885 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:06:51 ts/train.py:232 step:75K smpl:598K ep:2K epch:266.67 loss:0.042 grdn:5.893 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:07:05 ts/train.py:232 step:75K smpl:600K ep:2K epch:267.38 loss:0.042 grdn:6.183 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:07:20 ts/train.py:232 step:75K smpl:602K ep:2K epch:268.09 loss:0.042 grdn:6.227 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:07:34 ts/train.py:232 step:75K smpl:603K ep:2K epch:268.81 loss:0.042 grdn:6.243 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:07:48 ts/train.py:232 step:76K smpl:605K ep:2K epch:269.52 loss:0.042 grdn:6.316 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:08:02 ts/train.py:232 step:76K smpl:606K ep:2K epch:270.23 loss:0.041 grdn:6.225 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:08:17 ts/train.py:232 step:76K smpl:608K ep:2K epch:270.94 loss:0.042 grdn:6.199 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:08:30 ts/train.py:232 step:76K smpl:610K ep:2K epch:271.66 loss:0.041 grdn:6.032 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:08:45 ts/train.py:232 step:76K smpl:611K ep:2K epch:272.37 loss:0.041 grdn:5.946 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:08:59 ts/train.py:232 step:77K smpl:613K ep:2K epch:273.08 loss:0.041 grdn:5.773 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:09:13 ts/train.py:232 step:77K smpl:614K ep:2K epch:273.80 loss:0.042 grdn:6.100 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:09:27 ts/train.py:232 step:77K smpl:616K ep:2K epch:274.51 loss:0.041 grdn:6.014 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:09:42 ts/train.py:232 step:77K smpl:618K ep:2K epch:275.22 loss:0.041 grdn:5.869 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-15 02:09:56 ts/train.py:232 step:77K smpl:619K ep:2K epch:275.94 loss:0.040 grdn:6.026 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:10:10 ts/train.py:232 step:78K smpl:621K ep:2K epch:276.65 loss:0.041 grdn:6.137 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:10:24 ts/train.py:232 step:78K smpl:622K ep:2K epch:277.36 loss:0.041 grdn:6.259 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:10:38 ts/train.py:232 step:78K smpl:624K ep:3K epch:278.07 loss:0.040 grdn:6.067 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:10:52 ts/train.py:232 step:78K smpl:626K ep:3K epch:278.79 loss:0.040 grdn:5.739 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:11:07 ts/train.py:232 step:78K smpl:627K ep:3K epch:279.50 loss:0.040 grdn:5.994 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:11:21 ts/train.py:232 step:79K smpl:629K ep:3K epch:280.21 loss:0.042 grdn:6.285 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:11:35 ts/train.py:232 step:79K smpl:630K ep:3K epch:280.93 loss:0.041 grdn:6.044 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:11:49 ts/train.py:232 step:79K smpl:632K ep:3K epch:281.64 loss:0.040 grdn:6.118 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:12:04 ts/train.py:232 step:79K smpl:634K ep:3K epch:282.35 loss:0.040 grdn:5.683 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:12:18 ts/train.py:232 step:79K smpl:635K ep:3K epch:283.07 loss:0.039 grdn:5.616 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:12:32 ts/train.py:232 step:80K smpl:637K ep:3K epch:283.78 loss:0.040 grdn:6.087 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:12:46 ts/train.py:232 step:80K smpl:638K ep:3K epch:284.49 loss:0.040 grdn:6.091 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:13:00 ts/train.py:232 step:80K smpl:640K ep:3K epch:285.20 loss:0.040 grdn:6.038 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-15 02:13:00 ts/train.py:241 Checkpoint policy after step 80000 +INFO 2025-06-15 02:13:16 ts/train.py:232 step:80K smpl:642K ep:3K epch:285.92 loss:0.040 grdn:5.929 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:13:30 ts/train.py:232 step:80K smpl:643K ep:3K epch:286.63 loss:0.041 grdn:6.205 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:13:44 ts/train.py:232 step:81K smpl:645K ep:3K epch:287.34 loss:0.039 grdn:5.740 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:13:58 ts/train.py:232 step:81K smpl:646K ep:3K epch:288.06 loss:0.040 grdn:6.000 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:14:12 ts/train.py:232 step:81K smpl:648K ep:3K epch:288.77 loss:0.041 grdn:5.993 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:14:26 ts/train.py:232 step:81K smpl:650K ep:3K epch:289.48 loss:0.040 grdn:6.078 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:14:41 ts/train.py:232 step:81K smpl:651K ep:3K epch:290.20 loss:0.041 grdn:6.579 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:14:55 ts/train.py:232 step:82K smpl:653K ep:3K epch:290.91 loss:0.040 grdn:5.781 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:15:09 ts/train.py:232 step:82K smpl:654K ep:3K epch:291.62 loss:0.040 grdn:6.159 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:15:23 ts/train.py:232 step:82K smpl:656K ep:3K epch:292.34 loss:0.040 grdn:5.931 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:15:37 ts/train.py:232 step:82K smpl:658K ep:3K epch:293.05 loss:0.040 grdn:6.215 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:15:51 ts/train.py:232 step:82K smpl:659K ep:3K epch:293.76 loss:0.039 grdn:5.577 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:16:06 ts/train.py:232 step:83K smpl:661K ep:3K epch:294.47 loss:0.040 grdn:5.723 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:16:20 ts/train.py:232 step:83K smpl:662K ep:3K epch:295.19 loss:0.039 grdn:6.670 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:16:34 ts/train.py:232 step:83K smpl:664K ep:3K epch:295.90 loss:0.040 grdn:5.918 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:16:48 ts/train.py:232 step:83K smpl:666K ep:3K epch:296.61 loss:0.039 grdn:6.026 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:17:02 ts/train.py:232 step:83K smpl:667K ep:3K epch:297.33 loss:0.039 grdn:5.951 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:17:17 ts/train.py:232 step:84K smpl:669K ep:3K epch:298.04 loss:0.039 grdn:5.951 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:17:31 ts/train.py:232 step:84K smpl:670K ep:3K epch:298.75 loss:0.039 grdn:5.813 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:17:45 ts/train.py:232 step:84K smpl:672K ep:3K epch:299.47 loss:0.039 grdn:6.088 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:17:59 ts/train.py:232 step:84K smpl:674K ep:3K epch:300.18 loss:0.040 grdn:5.943 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:18:13 ts/train.py:232 step:84K smpl:675K ep:3K epch:300.89 loss:0.037 grdn:5.501 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:18:28 ts/train.py:232 step:85K smpl:677K ep:3K epch:301.60 loss:0.039 grdn:5.522 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:18:42 ts/train.py:232 step:85K smpl:678K ep:3K epch:302.32 loss:0.038 grdn:5.490 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:18:56 ts/train.py:232 step:85K smpl:680K ep:3K epch:303.03 loss:0.039 grdn:5.950 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:19:10 ts/train.py:232 step:85K smpl:682K ep:3K epch:303.74 loss:0.039 grdn:5.688 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:19:24 ts/train.py:232 step:85K smpl:683K ep:3K epch:304.46 loss:0.038 grdn:5.593 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:19:39 ts/train.py:232 step:86K smpl:685K ep:3K epch:305.17 loss:0.039 grdn:5.771 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:19:53 ts/train.py:232 step:86K smpl:686K ep:3K epch:305.88 loss:0.038 grdn:5.819 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:20:07 ts/train.py:232 step:86K smpl:688K ep:3K epch:306.60 loss:0.040 grdn:6.220 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:20:21 ts/train.py:232 step:86K smpl:690K ep:3K epch:307.31 loss:0.038 grdn:5.573 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:20:35 ts/train.py:232 step:86K smpl:691K ep:3K epch:308.02 loss:0.038 grdn:5.470 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:20:49 ts/train.py:232 step:87K smpl:693K ep:3K epch:308.73 loss:0.038 grdn:5.576 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:21:04 ts/train.py:232 step:87K smpl:694K ep:3K epch:309.45 loss:0.038 grdn:5.870 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:21:18 ts/train.py:232 step:87K smpl:696K ep:3K epch:310.16 loss:0.039 grdn:5.699 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:21:32 ts/train.py:232 step:87K smpl:698K ep:3K epch:310.87 loss:0.038 grdn:5.654 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:21:46 ts/train.py:232 step:87K smpl:699K ep:3K epch:311.59 loss:0.038 grdn:5.526 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:22:00 ts/train.py:232 step:88K smpl:701K ep:3K epch:312.30 loss:0.038 grdn:5.874 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:22:15 ts/train.py:232 step:88K smpl:702K ep:3K epch:313.01 loss:0.038 grdn:5.877 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:22:29 ts/train.py:232 step:88K smpl:704K ep:3K epch:313.73 loss:0.038 grdn:5.649 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:22:43 ts/train.py:232 step:88K smpl:706K ep:3K epch:314.44 loss:0.038 grdn:5.432 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:22:57 ts/train.py:232 step:88K smpl:707K ep:3K epch:315.15 loss:0.038 grdn:5.860 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:23:11 ts/train.py:232 step:89K smpl:709K ep:3K epch:315.86 loss:0.038 grdn:5.839 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:23:25 ts/train.py:232 step:89K smpl:710K ep:3K epch:316.58 loss:0.037 grdn:5.675 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:23:40 ts/train.py:232 step:89K smpl:712K ep:3K epch:317.29 loss:0.038 grdn:5.895 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:23:54 ts/train.py:232 step:89K smpl:714K ep:3K epch:318.00 loss:0.037 grdn:5.757 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:24:08 ts/train.py:232 step:89K smpl:715K ep:3K epch:318.72 loss:0.037 grdn:5.328 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:24:22 ts/train.py:232 step:90K smpl:717K ep:3K epch:319.43 loss:0.037 grdn:5.424 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:24:36 ts/train.py:232 step:90K smpl:718K ep:3K epch:320.14 loss:0.037 grdn:5.791 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:24:50 ts/train.py:232 step:90K smpl:720K ep:3K epch:320.86 loss:0.037 grdn:5.890 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:25:05 ts/train.py:232 step:90K smpl:722K ep:3K epch:321.57 loss:0.037 grdn:5.301 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-15 02:25:19 ts/train.py:232 step:90K smpl:723K ep:3K epch:322.28 loss:0.037 grdn:5.402 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:25:33 ts/train.py:232 step:91K smpl:725K ep:3K epch:322.99 loss:0.037 grdn:5.723 lr:1.0e-05 updt_s:0.069 data_s:0.001 +INFO 2025-06-15 02:25:47 ts/train.py:232 step:91K smpl:726K ep:3K epch:323.71 loss:0.037 grdn:5.590 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:26:01 ts/train.py:232 step:91K smpl:728K ep:3K epch:324.42 loss:0.036 grdn:5.477 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:26:16 ts/train.py:232 step:91K smpl:730K ep:3K epch:325.13 loss:0.037 grdn:5.779 lr:1.0e-05 updt_s:0.070 data_s:0.001 +INFO 2025-06-15 02:26:30 ts/train.py:232 step:91K smpl:731K ep:3K epch:325.85 loss:0.037 grdn:5.516 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:26:44 ts/train.py:232 step:92K smpl:733K ep:3K epch:326.56 loss:0.037 grdn:5.410 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:26:58 ts/train.py:232 step:92K smpl:734K ep:3K epch:327.27 loss:0.036 grdn:5.618 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:27:13 ts/train.py:232 step:92K smpl:736K ep:3K epch:327.99 loss:0.037 grdn:5.627 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:27:27 ts/train.py:232 step:92K smpl:738K ep:3K epch:328.70 loss:0.037 grdn:5.244 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:27:41 ts/train.py:232 step:92K smpl:739K ep:3K epch:329.41 loss:0.036 grdn:5.326 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:27:55 ts/train.py:232 step:93K smpl:741K ep:3K epch:330.12 loss:0.036 grdn:5.493 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:28:09 ts/train.py:232 step:93K smpl:742K ep:3K epch:330.84 loss:0.037 grdn:5.769 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:28:23 ts/train.py:232 step:93K smpl:744K ep:3K epch:331.55 loss:0.036 grdn:5.544 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:28:38 ts/train.py:232 step:93K smpl:746K ep:3K epch:332.26 loss:0.036 grdn:5.526 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:28:52 ts/train.py:232 step:93K smpl:747K ep:3K epch:332.98 loss:0.037 grdn:5.474 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:29:06 ts/train.py:232 step:94K smpl:749K ep:3K epch:333.69 loss:0.037 grdn:5.501 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:29:20 ts/train.py:232 step:94K smpl:750K ep:3K epch:334.40 loss:0.037 grdn:5.910 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:29:35 ts/train.py:232 step:94K smpl:752K ep:3K epch:335.12 loss:0.036 grdn:5.649 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:29:49 ts/train.py:232 step:94K smpl:754K ep:3K epch:335.83 loss:0.036 grdn:5.402 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:30:03 ts/train.py:232 step:94K smpl:755K ep:3K epch:336.54 loss:0.036 grdn:5.492 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:30:17 ts/train.py:232 step:95K smpl:757K ep:3K epch:337.25 loss:0.036 grdn:5.247 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:30:31 ts/train.py:232 step:95K smpl:758K ep:3K epch:337.97 loss:0.036 grdn:5.389 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:30:45 ts/train.py:232 step:95K smpl:760K ep:3K epch:338.68 loss:0.036 grdn:5.655 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:31:00 ts/train.py:232 step:95K smpl:762K ep:3K epch:339.39 loss:0.036 grdn:5.814 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:31:14 ts/train.py:232 step:95K smpl:763K ep:3K epch:340.11 loss:0.036 grdn:5.092 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:31:28 ts/train.py:232 step:96K smpl:765K ep:3K epch:340.82 loss:0.035 grdn:5.137 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:31:42 ts/train.py:232 step:96K smpl:766K ep:3K epch:341.53 loss:0.036 grdn:5.573 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:31:56 ts/train.py:232 step:96K smpl:768K ep:3K epch:342.25 loss:0.036 grdn:5.464 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:32:10 ts/train.py:232 step:96K smpl:770K ep:3K epch:342.96 loss:0.036 grdn:5.253 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:32:25 ts/train.py:232 step:96K smpl:771K ep:3K epch:343.67 loss:0.036 grdn:5.516 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:32:39 ts/train.py:232 step:97K smpl:773K ep:3K epch:344.39 loss:0.036 grdn:5.734 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:32:53 ts/train.py:232 step:97K smpl:774K ep:3K epch:345.10 loss:0.035 grdn:5.430 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:33:07 ts/train.py:232 step:97K smpl:776K ep:3K epch:345.81 loss:0.035 grdn:5.375 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:33:21 ts/train.py:232 step:97K smpl:778K ep:3K epch:346.52 loss:0.036 grdn:5.426 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:33:36 ts/train.py:232 step:97K smpl:779K ep:3K epch:347.24 loss:0.035 grdn:5.084 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:33:50 ts/train.py:232 step:98K smpl:781K ep:3K epch:347.95 loss:0.035 grdn:5.293 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:34:04 ts/train.py:232 step:98K smpl:782K ep:3K epch:348.66 loss:0.036 grdn:5.580 lr:1.0e-05 updt_s:0.070 data_s:0.002 +INFO 2025-06-15 02:34:18 ts/train.py:232 step:98K smpl:784K ep:3K epch:349.38 loss:0.036 grdn:5.605 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:34:33 ts/train.py:232 step:98K smpl:786K ep:3K epch:350.09 loss:0.036 grdn:5.631 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:34:47 ts/train.py:232 step:98K smpl:787K ep:3K epch:350.80 loss:0.036 grdn:5.750 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:35:01 ts/train.py:232 step:99K smpl:789K ep:3K epch:351.52 loss:0.036 grdn:5.558 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:35:15 ts/train.py:232 step:99K smpl:790K ep:3K epch:352.23 loss:0.035 grdn:5.340 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:35:29 ts/train.py:232 step:99K smpl:792K ep:3K epch:352.94 loss:0.034 grdn:5.233 lr:1.0e-05 updt_s:0.069 data_s:0.000 +INFO 2025-06-15 02:35:43 ts/train.py:232 step:99K smpl:794K ep:3K epch:353.65 loss:0.035 grdn:5.347 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:35:58 ts/train.py:232 step:99K smpl:795K ep:3K epch:354.37 loss:0.035 grdn:5.340 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:36:12 ts/train.py:232 step:100K smpl:797K ep:3K epch:355.08 loss:0.035 grdn:5.558 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:36:26 ts/train.py:232 step:100K smpl:798K ep:3K epch:355.79 loss:0.035 grdn:5.351 lr:1.0e-05 updt_s:0.070 data_s:0.000 +INFO 2025-06-15 02:36:40 ts/train.py:232 step:100K smpl:800K ep:3K epch:356.51 loss:0.035 grdn:5.366 lr:1.0e-05 updt_s:0.069 data_s:0.002 +INFO 2025-06-15 02:36:40 ts/train.py:241 Checkpoint policy after step 100000 +INFO 2025-06-15 02:36:41 ts/train.py:283 End of training diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/requirements.txt b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f3a5ff904f10fb0d11caed35d12b4260e408e2a6 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/requirements.txt @@ -0,0 +1,167 @@ +starlette==0.46.2 +httpx-sse==0.4.0 +regex==2024.11.6 +markdown-it-py==3.0.0 +mdurl==0.1.2 +groovy==0.1.2 +tqdm==4.67.1 +joblib==1.5.1 +httpcore==1.0.9 +typer==0.16.0 +pydantic_core==2.33.2 +typing_extensions==4.14.0 +mcp==1.9.0 +aiofiles==24.1.0 +ruff==0.11.13 +packaging==25.0 +pydantic-settings==2.9.1 +exceptiongroup==1.3.0 +pandas==2.3.0 +Jinja2==3.1.6 +nltk==3.9.1 +websockets==15.0.1 +tzdata==2025.2 +semantic-version==2.10.0 +fastapi==0.115.12 +orjson==3.10.18 +textblob==0.19.0 +h11==0.16.0 +python-dateutil==2.9.0.post0 +pydub==0.25.1 +sse-starlette==2.3.6 +tomlkit==0.13.3 +Pygments==2.19.1 +anyio==4.9.0 +shellingham==1.5.4 +typing-inspection==0.4.1 +pydantic==2.11.5 +rich==14.0.0 +numpy==2.2.6 +uvicorn==0.34.3 +huggingface-hub==0.32.4 +annotated-types==0.7.0 +python-dotenv==1.1.0 +safehttpx==0.1.6 +ffmpy==0.6.0 +httpx==0.28.1 +gradio_client==1.10.2 +python-multipart==0.0.20 +gradio==5.33.0 +filelock==3.18.0 +hf-xet==1.1.3 +sniffio==1.3.1 +itsdangerous==2.2.0 +toml==0.10.2 +multidict==6.4.4 +mpmath==1.3.0 +protobuf==6.31.1 +cloudpickle==3.1.1 +torchvision==0.22.1 +prompt_toolkit==3.0.51 +blinker==1.9.0 +PySocks==1.7.1 +beautifulsoup4==4.13.4 +nvidia-cuda-runtime-cu12==12.6.77 +sentry-sdk==2.30.0 +nvidia-cuda-cupti-cu12==12.6.80 +h5py==3.14.0 +aiohttp==3.12.13 +gitdb==4.0.12 +setproctitle==1.3.6 +GitPython==3.1.44 +six==1.17.0 +einops==0.8.1 +Farama-Notifications==0.0.4 +diffusers==0.33.1 +cffi==1.17.1 +pyarrow==20.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +zipp==3.23.0 +pycparser==2.22 +nvidia-nccl-cu12==2.26.2 +zarr==2.18.3 +antlr4-python3-runtime==4.9.3 +numcodecs==0.13.1 +MarkupSafe==3.0.2 +pip==25.1 +mergedeep==1.3.4 +certifi==2025.6.15 +soupsieve==2.7 +numba==0.61.2 +safetensors==0.5.3 +gymnasium==0.29.1 +multiprocess==0.70.16 +idna==3.10 +orderly-set==5.4.1 +networkx==3.4.2 +sympy==1.14.0 +xxhash==3.5.0 +pillow==11.2.1 +draccus==0.10.0 +platformdirs==4.3.8 +pyserial==3.5 +requests==2.32.4 +fasteners==0.19 +pyzmq==27.0.0 +nvidia-cudnn-cu12==9.5.1.17 +torch==2.7.1 +nvidia-cusparse-cu12==12.5.4.2 +psutil==7.0.0 +scipy==1.15.3 +async-timeout==5.0.1 +urllib3==2.4.0 +nvidia-curand-cu12==10.3.7.77 +av==14.4.0 +attrs==25.3.0 +smmap==5.0.2 +wandb==0.20.1 +hf_transfer==0.1.9 +propcache==0.3.2 +setuptools==78.1.1 +imageio-ffmpeg==0.6.0 +pynput==1.8.1 +pytz==2025.2 +pfzy==0.3.4 +torchcodec==0.4.0 +imageio==2.37.0 +nvidia-cublas-cu12==12.6.4.1 +triton==3.3.1 +pymunk==6.11.1 +wcwidth==0.2.13 +jsonlines==4.0.0 +python-xlib==0.33 +mypy_extensions==1.1.0 +cmake==4.0.3 +evdev==1.9.2 +omegaconf==2.3.0 +asciitree==0.3.3 +nvidia-cufft-cu12==11.3.0.4 +charset-normalizer==3.4.2 +deepdiff==8.5.0 +click==8.2.1 +termcolor==3.1.0 +rerun-sdk==0.23.3 +aiosignal==1.3.2 +llvmlite==0.44.0 +PyYAML==6.0.2 +importlib_metadata==8.7.0 +lerobot==0.1.0 +frozenlist==1.7.0 +Werkzeug==3.1.3 +nvidia-nvjitlink-cu12==12.6.85 +typing-inspect==0.9.0 +nvidia-cusolver-cu12==11.7.1.2 +dill==0.3.8 +pyyaml-include==1.4.1 +gdown==5.2.0 +datasets==3.6.0 +fsspec==2025.3.0 +yarl==1.20.1 +inquirerpy==0.3.4 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.1 +aiohappyeyeballs==2.6.1 +opencv-python-headless==4.11.0.86 +wheel==0.45.1 +nvidia-cufile-cu12==1.11.1.6 +nvidia-cusparselt-cu12==0.6.3 diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/wandb-metadata.json b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..034b29c79ec2e43bfdc0adfea2557980f83df8c2 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/wandb-metadata.json @@ -0,0 +1,46 @@ +{ + "os": "Linux-6.8.0-60-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.13", + "startedAt": "2025-06-15T07:34:26.987081Z", + "args": [ + "--config_path=/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/outputs/train/act_dice_art_run1/checkpoints/040000/pretrained_model/train_config.json", + "--resume=true" + ], + "program": "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "b8637c09ec93ca30d7901f5fdcf47aa4020cfe89" + }, + "root": "outputs/train/act_dice_art_run1", + "host": "hongyi-J", + "executable": "/home/hshi-j-4090/miniconda3/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 24, + "cpu_count_logical": 32, + "gpu": "NVIDIA GeForce RTX 4090", + "gpu_count": 1, + "disk": { + "/": { + "total": "3739429154816", + "used": "154254565376" + } + }, + "memory": { + "total": "67168522240" + }, + "cpu": { + "count": 24, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 4090", + "memoryTotal": "25757220864", + "cudaCores": 16384, + "architecture": "Ada", + "uuid": "GPU-5b318751-a95f-5805-6c0e-a4b86dcd442d" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/wandb-summary.json b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..571b7b37d03789c2ed5eb966a8ecd9852c4d3bbf --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/episodes":3208.5561497326203,"train/kld_loss":6.416812539100647e-05,"train/grad_norm":5.366218202114105,"train/l1_loss":0.03321615234017372,"train/loss":0.03475095812231302,"train/samples":800000,"_timestamp":1.7499802007378974e+09,"train/lr":1.0000000000000021e-05,"train/steps":100000,"_wandb":{"runtime":10394},"_runtime":10394.467139764,"train/update_s":0.06908714642999712,"_step":100000,"train/dataloading_s":0.0018981353950312042,"train/epochs":356.50623885918003} \ No newline at end of file diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-core.log b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..28072995f90ce14205de6a7b7f241caa93681cf1 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-06-15T01:25:25.939331443-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpo5o2pd57/port-4339.txt","pid":4339,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-06-15T01:25:25.944519426-07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":4339} +{"time":"2025-06-15T01:25:25.944844747-07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46177,"Zone":""}} +{"time":"2025-06-15T01:25:25.979295723-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60568"} +{"time":"2025-06-15T01:25:25.991245917-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"fhdgxxjz","id":"127.0.0.1:60568"} +{"time":"2025-06-15T01:25:26.634091387-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fhdgxxjz","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:36:42.454078305-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:36:42.454203578-07:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:36:42.454265175-07:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-06-15T02:36:42.454396504-07:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:36:42.824005867-07:00","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:46177->127.0.0.1:60568: use of closed network connection","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:37:32.062385761-07:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:37:32.062408574-07:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:60568"} +{"time":"2025-06-15T02:37:32.06242249-07:00","level":"INFO","msg":"server is closed"} diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-internal.log b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cf0c80cfe098b6969067e156afefc2c3a29c843d --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-internal.log @@ -0,0 +1,136 @@ +{"time":"2025-06-15T01:25:25.992015864-07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-core.log"} +{"time":"2025-06-15T01:25:26.634032319-07:00","level":"INFO","msg":"stream: created new stream","id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634083631-07:00","level":"INFO","msg":"stream: started","id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634215949-07:00","level":"INFO","msg":"writer: Do: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634288668-07:00","level":"INFO","msg":"handler: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:26.634389474-07:00","level":"INFO","msg":"sender: started","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T01:25:27.320758871-07:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-06-15T01:25:46.568983039-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.568997155-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.568999143-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569000545-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569001878-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569003382-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569004827-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569006116-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569007422-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569008882-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:25:46.569048108-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201} +{"time":"2025-06-15T01:26:02.367793397-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372335004-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372337971-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372339512-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372340951-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372342409-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372343769-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372345043-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.3723463-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372348347-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:02.372349688-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201} +{"time":"2025-06-15T01:26:17.685821042-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685849081-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685857431-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685858732-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685859866-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685861139-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685862255-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685863394-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685864601-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.68586591-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:17.685867074-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201} +{"time":"2025-06-15T01:26:32.130793723-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130813145-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130815054-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130816448-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130817763-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130819273-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130820563-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130821885-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130823187-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.13082474-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:32.130826044-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201} +{"time":"2025-06-15T01:26:46.825477529-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825530193-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825558155-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825582435-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825603491-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825629646-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825656804-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825682964-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825707833-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825737502-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:26:46.825760265-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201} +{"time":"2025-06-15T01:27:02.16185199-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161878645-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161884483-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161888247-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161891557-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161894938-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161898154-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161901259-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161904335-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161907587-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:02.161910751-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201} +{"time":"2025-06-15T01:27:16.797906839-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.7979248-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797928888-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797955435-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797965742-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.797986483-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798010312-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798035412-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798056372-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798085178-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:16.798107623-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201} +{"time":"2025-06-15T01:27:31.191814532-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.19183601-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191838508-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.19184059-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.19184258-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191850218-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191851983-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191854215-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191855837-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191857573-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:31.191859143-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201} +{"time":"2025-06-15T01:27:45.982376932-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982418361-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982429595-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982466168-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982474892-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982495778-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982520859-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982542092-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982564589-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982593443-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:27:45.982610475-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201} +{"time":"2025-06-15T01:28:01.229859045-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229889059-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229894087-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229897505-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229900368-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229903215-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229905919-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229908651-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229911346-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.22991437-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:01.229917398-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201} +{"time":"2025-06-15T01:28:15.6350119-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635042909-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635078209-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635093855-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635112045-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635150996-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.6351606-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635184442-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.63520897-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635243681-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T01:28:15.635260056-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201} +{"time":"2025-06-15T02:36:42.45420484-07:00","level":"INFO","msg":"stream: closing","id":"fhdgxxjz"} +{"time":"2025-06-15T02:36:42.454249855-07:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-06-15T02:36:42.454306083-07:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-06-15T02:37:31.874312646-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-06-15T02:37:32.062233809-07:00","level":"INFO","msg":"handler: closed","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T02:37:32.062277604-07:00","level":"INFO","msg":"writer: Close: closed","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T02:37:32.062318929-07:00","level":"INFO","msg":"sender: closed","stream_id":"fhdgxxjz"} +{"time":"2025-06-15T02:37:32.06237074-07:00","level":"INFO","msg":"stream: closed","id":"fhdgxxjz"} diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug.log b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a5429fcf7dccd45ca90aa13016821371ae582889 --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug.log @@ -0,0 +1,23 @@ +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1 +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Configure stats pid to 4339 +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/.config/wandb/settings +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/wandb/settings +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug.log +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-internal.log +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():831] calling init triggers +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():836] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'BoredMachinist/dice-art-run1', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {'observation.state': {'type': , 'shape': [6]}, 'observation.images.arm': {'type': , 'shape': [3, 480, 640]}, 'observation.images.world': {'type': , 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': , 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': 0, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_dice_art_run1', 'job_name': 'act_dice_art_run1', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'fhdgxxjz', 'mode': None}, '_wandb': {}} +2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():872] starting backend +2025-06-15 01:25:25,979 INFO MainThread:4339 [wandb_init.py:init():875] sending inform_init request +2025-06-15 01:25:25,986 INFO MainThread:4339 [wandb_init.py:init():883] backend started and connected +2025-06-15 01:25:25,989 INFO MainThread:4339 [wandb_init.py:init():956] updated telemetry +2025-06-15 01:25:26,005 INFO MainThread:4339 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout +2025-06-15 01:25:27,275 INFO MainThread:4339 [wandb_init.py:init():1027] run resumed +2025-06-15 01:25:27,279 INFO MainThread:4339 [wandb_init.py:init():1032] starting run threads in backend +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_console_start():2453] atexit reg +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2301] redirect: wrap_raw +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2370] Wrapping output streams. +2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2393] Redirects installed. +2025-06-15 01:25:27,406 INFO MainThread:4339 [wandb_init.py:init():1078] run started, returning control to user process +2025-06-15 02:36:42,453 INFO MsgRouterThr:4339 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 3 handles. diff --git a/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/run-fhdgxxjz.wandb b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/run-fhdgxxjz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c7017745af0664c18e5f4b364a2a1ded9ac3aa4a --- /dev/null +++ b/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/run-fhdgxxjz.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50a91f38d04d93fd78f3a6141d1d7a8e3eaddb462306f755347ed88bec5a3def +size 586857