diff --git "a/log.txt" "b/log.txt" new file mode 100644--- /dev/null +++ "b/log.txt" @@ -0,0 +1,369 @@ +Running 1 job +{ + "type": "ui_trainer", + "training_folder": "/home/featurize/ai-toolkit/output", + "sqlite_db_path": "/home/featurize/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4000, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/home/featurize/ai-toolkit/datasets/mimi_AI_art_Infinite_future_landscape_images_captioned", + "control_path": null, + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": false, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768, + 1024 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 30000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "qfloat8", + "arch": "qwen_image", + "low_vram": true, + "model_kwargs": {} + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 1024, + "samples": [ + { + "prompt": "In the style of Infinite future landscape , This is a digitally created, hyper-realistic CGI image depicting a fantastical aerial view of a sprawling cityscape. The image showcases a massive, ancient stone arch bridge stretching across the foreground, with intricate, gothic-style spires and towers along its edges. Below the bridge, a vast, turquoise ocean with visible coral reefs and islands can be seen. The city itself is densely packed with tall, dark buildings and spires, creating a labyrinthine effect. The background features a vibrant blue sky filled with fluffy white clouds, with the sun shining brightly in the top right corner, casting dramatic shadows. The overall color palette is dominated by rich blues and greens, with contrasting dark stone textures." + }, + { + "prompt": "In the style of Infinite future landscape , Sunlight filters through clouds onto a cherry blossom-filled meadow, where a blue-haired youth in an indigo coat stands by a vermilion bridge, holding a book as petals drift past his smile. The scene blends dynamic nature with human stillness, using pink, red and blue hues to create harmony, while symbolic elements like blossoms and books add depth." + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /home/featurize/ai-toolkit/aitk_db.db +Job ID: "4390f86f-d6d5-4785-a860-d9aef1081795" + +############################################# +# Running job: qwen_image_infinite_future_lora_v1 +############################################# + + +Running 1 process +Loading Qwen Image model +Loading transformer + Fetching 9 files: 0%| | 0/9 [00:00 + File "/home/featurize/ai-toolkit/run.py", line 120, in + main()main() + + File "/home/featurize/ai-toolkit/run.py", line 108, in main + File "/home/featurize/ai-toolkit/run.py", line 108, in main + raise eraise e + + File "/home/featurize/ai-toolkit/run.py", line 96, in main + File "/home/featurize/ai-toolkit/run.py", line 96, in main + job.run()job.run() + + File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run()process.run() + + File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2248, in run + File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2248, in run + self.end_step_hook()self.end_step_hook() + + File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 188, in end_step_hook + File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 188, in end_step_hook + self.maybe_stop()self.maybe_stop() + + File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 79, in maybe_stop + File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 79, in maybe_stop + raise Exception("Job stopped")raise Exception("Job stopped") + +ExceptionException: : Job stoppedJob stopped + + qwen_image_infinite_future_lora_v1: 9%|####5 | 2758/30000 [5:31:56<54:38:48, 7.22s/it, lr: 1.0e-04 loss: 1.961e-01] qwen_image_infinite_future_lora_v1: 9%|####5 | 2758/30000 [5:31:56<54:38:48, 7.22s/it, lr: 1.0e-04 loss: 1.961e-01] +