End of training

README.md

@@ -12,7 +12,7 @@ inference: true
 ---
     
 # LoRA text2image fine-tuning - deepgoyal19/lora5_tb
-These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the lambdalabs/pokemon-blip-captions dataset. You can find some example images in the following. 
+These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the None dataset. You can find some example images in the following. 
 
 ![img_0](./image_0.png)
 ![img_1](./image_1.png)

pytorch_lora_weights.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a667d4733a587ab1fd82d05132f013e57f474869c0d379c2d9e13858753f44ce
+oid sha256:8bd7b799e3d9a0ab12285bb41a8b54814dfea33b59b49d3a670ecab6d06c8420
 size 3287771

wandb/wandb/debug.log

@@ -1,30 +1,30 @@
-2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
-2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Configure stats pid to 3904922
-2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/.config/wandb/settings
-2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/lmflowdiff/Model/wandb/wandb/settings
-2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CACHE_DIR
-2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CONFIG_DIR
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/deepanshu/lmflowdiff/Model/wandb'}
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
-2023-06-14 16:09:40,199 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/deepanshu/LMFlow-diffusion-main/examples/finetune.py
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py'}
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:_log_setup():507] Logging user logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug.log
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:_log_setup():508] Logging internal logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug-internal.log
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():547] calling init triggers
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Configure stats pid to 3906677
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/.config/wandb/settings
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/lmflowdiff/Model/wandb/wandb/settings
+2023-06-14 16:30:50,525 WARNING MainThread:3906677 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CACHE_DIR
+2023-06-14 16:30:50,525 WARNING MainThread:3906677 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CONFIG_DIR
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/deepanshu/lmflowdiff/Model/wandb'}
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-06-14 16:30:50,525 WARNING MainThread:3906677 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/deepanshu/LMFlow-diffusion-main/examples/finetune.py
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py'}
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:_log_setup():507] Logging user logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/logs/debug.log
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:_log_setup():508] Logging internal logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/logs/debug-internal.log
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():547] calling init triggers
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
 config: {}
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():596] starting backend
-2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():600] setting up manager
-2023-06-14 16:09:40,201 INFO    MainThread:3904922 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2023-06-14 16:09:40,206 INFO    MainThread:3904922 [wandb_init.py:init():606] backend started and connected
-2023-06-14 16:09:40,210 INFO    MainThread:3904922 [wandb_init.py:init():703] updated telemetry
-2023-06-14 16:09:40,398 INFO    MainThread:3904922 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
-2023-06-14 16:09:41,650 INFO    MainThread:3904922 [wandb_run.py:_on_init():2176] communicating current version
-2023-06-14 16:09:42,662 INFO    MainThread:3904922 [wandb_run.py:_on_init():2185] got version response 
-2023-06-14 16:09:42,662 INFO    MainThread:3904922 [wandb_init.py:init():787] starting run threads in backend
-2023-06-14 16:09:51,480 INFO    MainThread:3904922 [wandb_run.py:_console_start():2155] atexit reg
-2023-06-14 16:09:51,481 INFO    MainThread:3904922 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
-2023-06-14 16:09:51,482 INFO    MainThread:3904922 [wandb_run.py:_redirect():2075] Wrapping output streams.
-2023-06-14 16:09:51,482 INFO    MainThread:3904922 [wandb_run.py:_redirect():2100] Redirects installed.
-2023-06-14 16:09:51,483 INFO    MainThread:3904922 [wandb_init.py:init():828] run started, returning control to user process
-2023-06-14 16:09:51,483 INFO    MainThread:3904922 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False}
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():596] starting backend
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():600] setting up manager
+2023-06-14 16:30:50,527 INFO    MainThread:3906677 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-06-14 16:30:50,528 INFO    MainThread:3906677 [wandb_init.py:init():606] backend started and connected
+2023-06-14 16:30:50,534 INFO    MainThread:3906677 [wandb_init.py:init():703] updated telemetry
+2023-06-14 16:30:50,643 INFO    MainThread:3906677 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-06-14 16:30:52,776 INFO    MainThread:3906677 [wandb_run.py:_on_init():2176] communicating current version
+2023-06-14 16:30:54,228 INFO    MainThread:3906677 [wandb_run.py:_on_init():2185] got version response 
+2023-06-14 16:30:54,228 INFO    MainThread:3906677 [wandb_init.py:init():787] starting run threads in backend
+2023-06-14 16:31:01,872 INFO    MainThread:3906677 [wandb_run.py:_console_start():2155] atexit reg
+2023-06-14 16:31:01,873 INFO    MainThread:3906677 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-06-14 16:31:01,873 INFO    MainThread:3906677 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-06-14 16:31:01,873 INFO    MainThread:3906677 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-06-14 16:31:01,874 INFO    MainThread:3906677 [wandb_init.py:init():828] run started, returning control to user process
+2023-06-14 16:31:01,875 INFO    MainThread:3906677 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 2, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False}

wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log

@@ -61,3 +61,31 @@ pytorch_lora_weights.bin: 100%|████████████████
 events.out.tfevents.1686740628.A40-20230526-003.3863181.0: 100%|███████████████████████████████████████████████████████████████████████████████| 1.25M/1.25M [00:01<00:00, 898kB/s]
 events.out.tfevents.1686740628.A40-20230526-003.3863181.0:   0%|                                                                                       | 0.00/1.25M [00:00<?, ?B/s]
 
+events.out.tfevents.1686719590.A40-20230526-003.3796603.0: 100%|██████████████████████████████████████████████████████████████████████████████| 3.72M/3.72M [00:10<00:00, 10.8MB/s]
+
+`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden.           | 11/24 [03:44<04:47, 22.09s/it]
+{'prediction_type'} was not found in config. Values will be initialized to default values.
+{'scaling_factor'} was not found in config. Values will be initialized to default values.
+  0%|                                                                                                                                                       | 0/20 [00:00<?, ?it/s]
+Traceback (most recent call last):                                                                                                                          | 0/20 [00:00<?, ?it/s]
+  File "/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py", line 74, in <module>
+    tuned_model = finetuner.finetune(model=model, dataset=dataset)
+  File "/home/deepanshu/LMFlow-diffusion-main/src/lmflow_aigen/pipeline/diffusion_finetuner.py", line 444, in finetune
+    model.log_validation(self.finetuner_args,self.accelerator,global_step)
+  File "/home/deepanshu/LMFlow-diffusion-main/src/lmflow_aigen/models/diffusion_model.py", line 295, in log_validation
+    self.images.append(pipeline(args.validation_prompts[i], num_inference_steps=20, generator=generator).images[0])
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py", line 684, in __call__
+    noise_pred = self.unet(
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/diffusers/models/unet_2d_condition.py", line 718, in forward
+    sample = self.conv_in(sample)
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/torch/nn/modules/conv.py", line 463, in forward
+    return self._conv_forward(input, self.weight, self.bias)
+  File "/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/torch/nn/modules/conv.py", line 459, in _conv_forward
+    return F.conv2d(input, weight, bias, self.stride,
+RuntimeError: Input type (c10::Half) and bias type (float) should be the same

wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json

@@ -1 +1 @@
-{"train_loss": 0.43844109773635864, "_timestamp": 1686759028.6411998, "_runtime": 48.43455791473389, "_step": 60, "validation": {"_type": "images/separated", "width": 512, "height": 512, "format": "png", "count": 3, "filenames": ["media/images/validation_60_e9c4a470ad900801f7de.png", "media/images/validation_60_2ff5d73691ebb72c877d.png", "media/images/validation_60_b3033a9fe6335a935570.png"], "captions": ["0: pikachu", "1: bulbasaur", "2: charmandar"]}}
+{"train_loss": 0.43844109773635864, "_timestamp": 1686759028.6411998, "_runtime": 48.43455791473389, "_step": 60, "validation": {"_type": "images/separated", "width": 512, "height": 512, "format": "png", "count": 3, "filenames": ["media/images/validation_60_e9c4a470ad900801f7de.png", "media/images/validation_60_2ff5d73691ebb72c877d.png", "media/images/validation_60_b3033a9fe6335a935570.png"], "captions": ["0: pikachu", "1: bulbasaur", "2: charmandar"]}, "_wandb": {"runtime": 285}}

wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug-internal.log

@@ -447,3 +447,147 @@
 2023-06-14 16:14:08,110 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
 2023-06-14 16:14:12,702 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
 2023-06-14 16:14:13,703 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:19,280 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:19,893 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:14:21,494 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:14:21,495 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:14:25,255 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:25,895 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:14:27,899 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:14:28,038 DEBUG   SenderThread:3905241 [sender.py:send():369] send: exit
+2023-06-14 16:14:28,038 INFO    SenderThread:3905241 [sender.py:send_exit():574] handling exit code: 1
+2023-06-14 16:14:28,039 INFO    SenderThread:3905241 [sender.py:send_exit():576] handling runtime: 285
+2023-06-14 16:14:28,039 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:14:28,039 INFO    SenderThread:3905241 [sender.py:send_exit():582] send defer
+2023-06-14 16:14:28,040 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:28,040 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 0
+2023-06-14 16:14:28,040 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:28,040 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 0
+2023-06-14 16:14:28,040 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 1
+2023-06-14 16:14:28,040 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:28,040 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 1
+2023-06-14 16:14:28,040 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:28,040 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 1
+2023-06-14 16:14:28,040 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 2
+2023-06-14 16:14:28,040 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:28,040 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 2
+2023-06-14 16:14:28,040 INFO    HandlerThread:3905241 [system_monitor.py:finish():190] Stopping system monitor
+2023-06-14 16:14:28,041 INFO    HandlerThread:3905241 [interfaces.py:finish():202] Joined cpu monitor
+2023-06-14 16:14:28,042 INFO    HandlerThread:3905241 [interfaces.py:finish():202] Joined disk monitor
+2023-06-14 16:14:28,042 DEBUG   SystemMonitor:3905241 [system_monitor.py:_start():166] Finished system metrics aggregation loop
+2023-06-14 16:14:28,042 DEBUG   SystemMonitor:3905241 [system_monitor.py:_start():170] Publishing last batch of metrics
+2023-06-14 16:14:28,899 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:14:29,056 INFO    HandlerThread:3905241 [interfaces.py:finish():202] Joined gpu monitor
+2023-06-14 16:14:29,057 INFO    HandlerThread:3905241 [interfaces.py:finish():202] Joined memory monitor
+2023-06-14 16:14:29,057 INFO    HandlerThread:3905241 [interfaces.py:finish():202] Joined network monitor
+2023-06-14 16:14:29,057 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: poll_exit
+2023-06-14 16:14:29,058 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:29,058 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 2
+2023-06-14 16:14:29,058 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 3
+2023-06-14 16:14:29,058 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:14:29,059 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: poll_exit
+2023-06-14 16:14:29,058 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:29,059 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 3
+2023-06-14 16:14:29,059 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:29,059 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 3
+2023-06-14 16:14:29,059 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 4
+2023-06-14 16:14:29,059 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:29,059 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 4
+2023-06-14 16:14:29,059 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:29,059 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 4
+2023-06-14 16:14:29,059 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 5
+2023-06-14 16:14:29,059 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:29,060 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 5
+2023-06-14 16:14:29,060 DEBUG   SenderThread:3905241 [sender.py:send():369] send: summary
+2023-06-14 16:14:29,060 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:14:29,060 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:29,060 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 5
+2023-06-14 16:14:29,060 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 6
+2023-06-14 16:14:29,060 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:29,060 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 6
+2023-06-14 16:14:29,061 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:29,061 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 6
+2023-06-14 16:14:29,061 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 7
+2023-06-14 16:14:29,061 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:29,061 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:29,061 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 7
+2023-06-14 16:14:29,061 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:29,061 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 7
+2023-06-14 16:14:29,900 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:14:29,900 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:14:30,039 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: poll_exit
+2023-06-14 16:14:31,364 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 8
+2023-06-14 16:14:31,364 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: poll_exit
+2023-06-14 16:14:31,364 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:31,365 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 8
+2023-06-14 16:14:31,365 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:31,365 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 8
+2023-06-14 16:14:31,365 INFO    SenderThread:3905241 [job_builder.py:build():232] Attempting to build job artifact
+2023-06-14 16:14:31,365 INFO    SenderThread:3905241 [job_builder.py:build():266] no source found
+2023-06-14 16:14:31,365 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 9
+2023-06-14 16:14:31,365 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:31,365 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 9
+2023-06-14 16:14:31,365 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:31,366 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 9
+2023-06-14 16:14:31,366 INFO    SenderThread:3905241 [dir_watcher.py:finish():359] shutting down directory watcher
+2023-06-14 16:14:31,900 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:14:31,901 INFO    SenderThread:3905241 [dir_watcher.py:finish():389] scan: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files
+2023-06-14 16:14:31,901 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/requirements.txt requirements.txt
+2023-06-14 16:14:31,902 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml conda-environment.yaml
+2023-06-14 16:14:31,902 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml config.yaml
+2023-06-14 16:14:31,902 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json wandb-summary.json
+2023-06-14 16:14:31,904 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-metadata.json wandb-metadata.json
+2023-06-14 16:14:31,906 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log output.log
+2023-06-14 16:14:31,911 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png media/images/validation_60_2ff5d73691ebb72c877d.png
+2023-06-14 16:14:31,911 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png media/images/validation_60_b3033a9fe6335a935570.png
+2023-06-14 16:14:31,911 INFO    SenderThread:3905241 [dir_watcher.py:finish():403] scan save: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png media/images/validation_60_e9c4a470ad900801f7de.png
+2023-06-14 16:14:31,911 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 10
+2023-06-14 16:14:31,914 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:31,914 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 10
+2023-06-14 16:14:31,918 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:31,919 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 10
+2023-06-14 16:14:31,919 INFO    SenderThread:3905241 [file_pusher.py:finish():159] shutting down file pusher
+2023-06-14 16:14:32,749 INFO    wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml
+2023-06-14 16:14:32,775 INFO    wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:14:33,435 INFO    wandb-upload_2:3905241 [upload_job.py:push():133] Uploaded file /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/requirements.txt
+2023-06-14 16:14:33,604 INFO    wandb-upload_4:3905241 [upload_job.py:push():133] Uploaded file /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:14:33,630 INFO    wandb-upload_3:3905241 [upload_job.py:push():133] Uploaded file /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml
+2023-06-14 16:14:33,831 INFO    Thread-11 :3905241 [sender.py:transition_state():602] send defer: 11
+2023-06-14 16:14:33,831 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:33,831 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 11
+2023-06-14 16:14:33,832 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:33,832 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 11
+2023-06-14 16:14:33,832 INFO    SenderThread:3905241 [file_pusher.py:join():164] waiting for file pusher
+2023-06-14 16:14:33,832 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 12
+2023-06-14 16:14:33,832 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:33,832 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 12
+2023-06-14 16:14:33,833 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:33,833 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 12
+2023-06-14 16:14:35,046 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive
+2023-06-14 16:14:35,150 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 13
+2023-06-14 16:14:35,151 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:35,151 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 13
+2023-06-14 16:14:35,151 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:35,151 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 13
+2023-06-14 16:14:35,152 INFO    SenderThread:3905241 [sender.py:transition_state():602] send defer: 14
+2023-06-14 16:14:35,152 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: defer
+2023-06-14 16:14:35,152 INFO    HandlerThread:3905241 [handler.py:handle_request_defer():170] handle defer: 14
+2023-06-14 16:14:35,152 DEBUG   SenderThread:3905241 [sender.py:send():369] send: final
+2023-06-14 16:14:35,152 DEBUG   SenderThread:3905241 [sender.py:send():369] send: footer
+2023-06-14 16:14:35,152 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: defer
+2023-06-14 16:14:35,152 INFO    SenderThread:3905241 [sender.py:send_request_defer():598] handle sender defer: 14
+2023-06-14 16:14:35,153 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: poll_exit
+2023-06-14 16:14:35,153 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: poll_exit
+2023-06-14 16:14:35,154 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: server_info
+2023-06-14 16:14:35,154 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: server_info
+2023-06-14 16:14:35,157 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: get_summary
+2023-06-14 16:14:35,157 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: sampled_history
+2023-06-14 16:14:35,750 INFO    MainThread:3905241 [wandb_run.py:_footer_history_summary_info():3467] rendering history
+2023-06-14 16:14:35,750 INFO    MainThread:3905241 [wandb_run.py:_footer_history_summary_info():3499] rendering summary
+2023-06-14 16:14:35,751 INFO    MainThread:3905241 [wandb_run.py:_footer_sync_info():3426] logging synced files
+2023-06-14 16:14:35,751 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: shutdown
+2023-06-14 16:14:35,752 INFO    HandlerThread:3905241 [handler.py:finish():854] shutting down handler
+2023-06-14 16:14:36,154 INFO    WriterThread:3905241 [datastore.py:close():298] close: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/run-0vx95vx0.wandb
+2023-06-14 16:14:36,751 INFO    SenderThread:3905241 [sender.py:finish():1526] shutting down sender
+2023-06-14 16:14:36,752 INFO    SenderThread:3905241 [file_pusher.py:finish():159] shutting down file pusher
+2023-06-14 16:14:36,752 INFO    SenderThread:3905241 [file_pusher.py:join():164] waiting for file pusher

wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug.log

@@ -28,3 +28,4 @@ config: {}
 2023-06-14 16:09:51,482 INFO    MainThread:3904922 [wandb_run.py:_redirect():2100] Redirects installed.
 2023-06-14 16:09:51,483 INFO    MainThread:3904922 [wandb_init.py:init():828] run started, returning control to user process
 2023-06-14 16:09:51,483 INFO    MainThread:3904922 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False}
+2023-06-14 16:14:37,598 WARNING MsgRouterThr:3904922 [router.py:message_loop():77] message_loop has been closed

wandb/wandb/run-20230614_163050-nlm9mp31/files/conda-environment.yaml

@@ -0,0 +1,191 @@
+name: deepanshu
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
+  - ca-certificates=2023.5.7=hbcca054_0
+  - debugpy=1.5.1=py39h295c915_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - ipykernel=6.15.0=pyh210e3f2_0
+  - ipython=8.13.2=pyh41d4057_0
+  - jedi=0.18.2=pyhd8ed1ab_0
+  - jupyter_client=7.0.6=pyhd8ed1ab_0
+  - jupyter_core=5.3.0=py39hf3d152e_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.6=pyhd8ed1ab_0
+  - openssl=1.1.1t=h7f8727e_0
+  - packaging=23.1=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.0.1=py39h06a4308_0
+  - platformdirs=3.5.1=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.38=pyha770c72_0
+  - prompt_toolkit=3.0.38=hd8ed1ab_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.15.1=pyhd8ed1ab_0
+  - python=3.9.16=h7a1cb2a_2
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.9=2_cp39
+  - pyzmq=19.0.2=py39hb69f2a1_2
+  - readline=8.2=h5eee18b_0
+  - setuptools=66.0.0=py39h06a4308_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h1ccaba5_0
+  - tornado=6.1=py39hb9d737c_3
+  - traitlets=5.9.0=pyhd8ed1ab_0
+  - typing_extensions=4.6.2=pyha770c72_0
+  - wcwidth=0.2.6=pyhd8ed1ab_0
+  - wheel=0.38.4=py39h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - zeromq=4.3.4=h9c3ff4c_1
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - absl-py==1.4.0
+      - accelerate==0.19.0
+      - aiofiles==23.1.0
+      - aiohttp==3.8.4
+      - aiosignal==1.3.1
+      - altair==5.0.1
+      - anyio==3.7.0
+      - appdirs==1.4.4
+      - async-timeout==4.0.2
+      - attrs==23.1.0
+      - bitsandbytes==0.39.0
+      - cachetools==5.3.1
+      - certifi==2022.12.7
+      - charset-normalizer==2.1.1
+      - click==8.1.3
+      - clip==1.0
+      - cmake==3.25.0
+      - contourpy==1.0.7
+      - cycler==0.11.0
+      - datasets==2.12.0
+      - deepspeed==0.9.2
+      - diffusers==0.16.1
+      - dill==0.3.6
+      - docker-pycreds==0.4.0
+      - exceptiongroup==1.1.1
+      - fastapi==0.95.2
+      - ffmpy==0.3.0
+      - filelock==3.9.0
+      - fonttools==4.39.4
+      - frozenlist==1.3.3
+      - fsspec==2023.5.0
+      - ftfy==6.1.1
+      - gitdb==4.0.10
+      - gitpython==3.1.31
+      - google-auth==2.20.0
+      - google-auth-oauthlib==1.0.0
+      - gradio==3.32.0
+      - gradio-client==0.2.5
+      - grpcio==1.54.2
+      - h11==0.14.0
+      - hjson==3.1.0
+      - httpcore==0.17.2
+      - httpx==0.24.1
+      - huggingface-hub==0.14.1
+      - idna==3.4
+      - importlib-metadata==6.6.0
+      - importlib-resources==5.12.0
+      - jinja2==3.1.2
+      - jsonschema==4.17.3
+      - kiwisolver==1.4.4
+      - linkify-it-py==2.0.2
+      - lit==15.0.7
+      - markdown==3.4.3
+      - markdown-it-py==2.2.0
+      - markupsafe==2.1.2
+      - matplotlib==3.7.1
+      - mdit-py-plugins==0.3.3
+      - mdurl==0.1.2
+      - mpmath==1.2.1
+      - multidict==6.0.4
+      - multiprocess==0.70.14
+      - mxnet==1.9.1
+      - mypy-extensions==1.0.0
+      - natsort==8.3.1
+      - networkx==3.0
+      - ninja==1.11.1
+      - numpy==1.24.1
+      - oauthlib==3.2.2
+      - open-clip-torch==2.20.0
+      - orjson==3.8.14
+      - pandas==2.0.1
+      - pathtools==0.1.2
+      - pillow==9.3.0
+      - plotly==5.15.0
+      - protobuf==3.20.3
+      - psutil==5.9.5
+      - py-cpuinfo==9.0.0
+      - pyarrow==12.0.0
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pydantic==1.10.8
+      - pydub==0.25.1
+      - pyparsing==3.0.9
+      - pyre-extensions==0.0.29
+      - pyrsistent==0.19.3
+      - python-graphviz==0.8.4
+      - python-multipart==0.0.6
+      - pytz==2023.3
+      - pyyaml==6.0
+      - regex==2023.5.5
+      - requests==2.28.1
+      - requests-oauthlib==1.3.1
+      - responses==0.18.0
+      - rsa==4.9
+      - safetensors==0.3.1
+      - scipy==1.10.1
+      - semantic-version==2.10.0
+      - sentencepiece==0.1.99
+      - sentry-sdk==1.25.1
+      - setproctitle==1.3.2
+      - smmap==5.0.0
+      - sniffio==1.3.0
+      - starlette==0.27.0
+      - sympy==1.11.1
+      - tenacity==8.2.2
+      - tensorboard==2.13.0
+      - tensorboard-data-server==0.7.1
+      - timm==0.9.2
+      - tokenizers==0.13.3
+      - toolz==0.12.0
+      - torch==2.0.1+cu118
+      - torchaudio==2.0.2+cu118
+      - torchvision==0.15.2+cu118
+      - tqdm==4.65.0
+      - transformers==4.29.2
+      - triton==2.0.0
+      - typing-extensions==4.4.0
+      - typing-inspect==0.9.0
+      - tzdata==2023.3
+      - uc-micro-py==1.0.2
+      - urllib3==1.26.13
+      - uvicorn==0.22.0
+      - wandb==0.15.4
+      - websockets==11.0.3
+      - werkzeug==2.3.6
+      - xformers==0.0.20
+      - xxhash==3.2.0
+      - yarl==1.9.2
+      - zipp==3.15.0
+prefix: /home/deepanshu/anaconda3/envs/deepanshu

wandb/wandb/run-20230614_163050-nlm9mp31/files/config.yaml

@@ -0,0 +1,182 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.9.16
+    cli_version: 0.15.4
+    framework: huggingface
+    huggingface_version: 4.29.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1686760250.529075
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      3:
+      - 23
+      4: 3.9.16
+      5: 0.15.4
+      6: 4.29.2
+      8:
+      - 5
+num_validation_images:
+  desc: null
+  value: 1
+validation_epochs:
+  desc: null
+  value: 60
+max_train_samples:
+  desc: null
+  value: null
+output_dir:
+  desc: null
+  value: /home/deepanshu/lmflowdiff/Model
+seed:
+  desc: null
+  value: 123
+train_batch_size:
+  desc: null
+  value: 1
+num_train_epochs:
+  desc: null
+  value: 2
+max_train_steps:
+  desc: null
+  value: 60
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+gradient_checkpointing:
+  desc: null
+  value: true
+learning_rate:
+  desc: null
+  value: 1.0e-06
+scale_lr:
+  desc: null
+  value: false
+lr_scheduler:
+  desc: null
+  value: constant
+lr_warmup_steps:
+  desc: null
+  value: 0
+snr_gamma:
+  desc: null
+  value: null
+use_8bit_adam:
+  desc: null
+  value: false
+allow_tf32:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_weight_decay:
+  desc: null
+  value: 0.01
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1
+push_to_hub:
+  desc: null
+  value: true
+hub_token:
+  desc: null
+  value: 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
+hub_model_id:
+  desc: null
+  value: lora5_tb
+logging_dir:
+  desc: null
+  value: log
+mixed_precision:
+  desc: null
+  value: fp16
+resume_from_checkpoint:
+  desc: null
+  value: latest
+fp16_backend:
+  desc: null
+  value: auto
+fp16_opt_level:
+  desc: null
+  value: O1
+disable_tqdm:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 1000
+save_total_limit:
+  desc: null
+  value: null
+eval_dataset_path:
+  desc: null
+  value: null
+tracker_project_name:
+  desc: null
+  value: text2image-fine-tune
+checkpoints_total_limit:
+  desc: null
+  value: null
+enable_xformers_memory_efficient_attention:
+  desc: null
+  value: true
+noise_offset:
+  desc: null
+  value: 1
+input_perturbation:
+  desc: null
+  value: 0
+checkpointing_steps:
+  desc: null
+  value: 500
+report_to:
+  desc: null
+  value: wandb
+accelerate_device:
+  desc: null
+  value: cuda:1
+accelerate_local_process_index:
+  desc: null
+  value: 0
+accelerate_num_processes:
+  desc: null
+  value: 1
+accelerate_optimizer_step_was_skipped:
+  desc: null
+  value: false
+accelerate_process_index:
+  desc: null
+  value: 0
+accelerate_sync_gradients:
+  desc: null
+  value: true
+accelerate_use_distributed:
+  desc: null
+  value: false

wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log

@@ -0,0 +1,32 @@
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - ***** Running training *****
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Num examples = 50
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Num Epochs = 2
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Instantaneous batch size per device = 1
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Total train batch size (w. parallel, distributed & accumulation) = 1
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Gradient Accumulation steps = 1
+06/14/2023 16:31:01 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Total optimization steps = 60
+Steps:   0%|                                                                                                                                                | 0/60 [00:00<?, ?it/s]
+Steps:   0%|                                                                                                                                                | 0/60 [00:00<?, ?it/s]/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/xformers/ops/fmha/flash.py:339: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
+  and inp.query.storage().data_ptr() == inp.key.storage().data_ptr()
+
+
+
+
+
+
+
+
+Steps: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:23<00:00,  2.66it/s, lr=1e-6, step_loss=0.532]06/14/2023 16:31:25 - INFO - lmflow_aigen.models.diffusion_model - Running validation...
+ Generating 1 images with prompt: ['pikachu', 'bulbasaur', 'charmandar'].
+{'requires_safety_checker'} was not found in config. Values will be initialized to default values.
+`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden.
+{'prediction_type'} was not found in config. Values will be initialized to default values.
+{'scaling_factor'} was not found in config. Values will be initialized to default values.
+
+Potential NSFW content was detected in one or more images. A black image will be returned instead. Try again with a different prompt and/or seed.  | 19/20 [00:02<00:00, 10.08it/s]
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:01<00:00, 10.27it/s]
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:01<00:00, 10.93it/s]
+
+
+Model weights saved in /home/deepanshu/lmflowdiff/Model/pytorch_lora_weights.bin███████████████████████████████████████████████████████████▉       | 19/20 [00:01<00:00, 11.13it/s]
+Upload 1 LFS files:   0%|                                                                                                                                    | 0/1 [00:00<?, ?it/s]

wandb/wandb/run-20230614_163050-nlm9mp31/files/requirements.txt

@@ -0,0 +1,163 @@
+absl-py==1.4.0
+accelerate==0.19.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyio==3.7.0
+appdirs==1.4.4
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+backcall==0.2.0
+backports.functools-lru-cache==1.6.4
+bitsandbytes==0.39.0
+cachetools==5.3.1
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.3
+clip==1.0
+cmake==3.25.0
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+debugpy==1.5.1
+decorator==5.1.1
+deepspeed==0.9.2
+diffusers==0.16.1
+dill==0.3.6
+docker-pycreds==0.4.0
+entrypoints==0.4
+exceptiongroup==1.1.1
+executing==1.2.0
+fastapi==0.95.2
+ffmpy==0.3.0
+filelock==3.9.0
+fonttools==4.39.4
+frozenlist==1.3.3
+fsspec==2023.5.0
+ftfy==6.1.1
+gitdb==4.0.10
+gitpython==3.1.31
+google-auth-oauthlib==1.0.0
+google-auth==2.20.0
+gradio-client==0.2.5
+gradio==3.32.0
+graphviz==0.8.4
+grpcio==1.54.2
+h11==0.14.0
+hjson==3.1.0
+httpcore==0.17.2
+httpx==0.24.1
+huggingface-hub==0.14.1
+idna==3.4
+importlib-metadata==6.6.0
+importlib-resources==5.12.0
+ipykernel==6.15.0
+ipython==8.13.2
+jedi==0.18.2
+jinja2==3.1.2
+jsonschema==4.17.3
+jupyter-client==7.0.6
+jupyter-core==5.3.0
+kiwisolver==1.4.4
+linkify-it-py==2.0.2
+lit==15.0.7
+lmflow-aigen==0.0.1
+markdown-it-py==2.2.0
+markdown==3.4.3
+markupsafe==2.1.2
+matplotlib-inline==0.1.6
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mpmath==1.2.1
+multidict==6.0.4
+multiprocess==0.70.14
+mxnet==1.9.1
+mypy-extensions==1.0.0
+natsort==8.3.1
+nest-asyncio==1.5.6
+networkx==3.0
+ninja==1.11.1
+numpy==1.24.1
+oauthlib==3.2.2
+open-clip-torch==2.20.0
+orjson==3.8.14
+packaging==23.1
+pandas==2.0.1
+parso==0.8.3
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.3.0
+pip==23.0.1
+platformdirs==3.5.1
+plotly==5.15.0
+prompt-toolkit==3.0.38
+protobuf==3.20.3
+psutil==5.9.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==12.0.0
+pyasn1-modules==0.3.0
+pyasn1==0.5.0
+pydantic==1.10.8
+pydub==0.25.1
+pygments==2.15.1
+pyparsing==3.0.9
+pyre-extensions==0.0.29
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pyyaml==6.0
+pyzmq==19.0.2
+regex==2023.5.5
+requests-oauthlib==1.3.1
+requests==2.28.1
+responses==0.18.0
+rsa==4.9
+safetensors==0.3.1
+scipy==1.10.1
+semantic-version==2.10.0
+sentencepiece==0.1.99
+sentry-sdk==1.25.1
+setproctitle==1.3.2
+setuptools==66.0.0
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+stack-data==0.6.2
+starlette==0.27.0
+sympy==1.11.1
+tenacity==8.2.2
+tensorboard-data-server==0.7.1
+tensorboard==2.13.0
+timm==0.9.2
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.1+cu118
+torchaudio==2.0.2+cu118
+torchvision==0.15.2+cu118
+tornado==6.1
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.29.2
+triton==2.0.0
+typing-extensions==4.4.0
+typing-inspect==0.9.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+urllib3==1.26.13
+uvicorn==0.22.0
+wandb==0.15.4
+wcwidth==0.2.6
+websockets==11.0.3
+werkzeug==2.3.6
+wheel==0.38.4
+xformers==0.0.20
+xxhash==3.2.0
+yarl==1.9.2
+zipp==3.15.0

wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-metadata.json

@@ -0,0 +1,712 @@
+{
+    "os": "Linux-5.4.0-90-generic-x86_64-with-glibc2.31",
+    "python": "3.9.16",
+    "heartbeatAt": "2023-06-14T16:30:54.388684",
+    "startedAt": "2023-06-14T16:30:50.522065",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py",
+    "git": {
+        "remote": "https://github.com/hendrydong/LMFlow-diffusion.git",
+        "commit": "7eb976e6588110787f15fb1ef1b6ce7c4d84ae87"
+    },
+    "email": "[email protected]",
+    "root": "/home/deepanshu/lmflowdiff",
+    "host": "A40-20230526-003",
+    "username": "deepanshu",
+    "executable": "/home/deepanshu/anaconda3/envs/deepanshu/bin/python",
+    "cpu_count": 64,
+    "cpu_count_logical": 128,
+    "cpu_freq": {
+        "current": 3.4042656249999994,
+        "min": 1500.0,
+        "max": 2800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.403,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.402,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.399,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.399,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.398,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.399,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.401,
+            "min": 1500.0,
+            "max": 2800.0
+        },
+        {
+            "current": 3.4,
+            "min": 1500.0,
+            "max": 2800.0
+        }
+    ],
+    "disk": {
+        "total": 218.5130844116211,
+        "used": 49.349273681640625
+    },
+    "gpu": "NVIDIA A40",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        },
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 51527024640
+        }
+    ],
+    "memory": {
+        "total": 755.7487335205078
+    }
+}

wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train_loss": 0.5324989557266235, "_timestamp": 1686760299.1669528, "_runtime": 48.63787794113159, "_step": 60, "validation": {"_type": "images/separated", "width": 512, "height": 512, "format": "png", "count": 3, "filenames": ["media/images/validation_60_e9c4a470ad900801f7de.png", "media/images/validation_60_7673b7ce51f7ba8dbeaa.png", "media/images/validation_60_657ebe0b72dbaae9b6f3.png"], "captions": ["0: pikachu", "1: bulbasaur", "2: charmandar"]}}

wandb/wandb/run-20230614_163050-nlm9mp31/logs/debug-internal.log

@@ -0,0 +1,354 @@
+2023-06-14 16:30:50,531 INFO    StreamThr :3907002 [internal.py:wandb_internal():86] W&B internal server running at pid: 3907002, started at: 2023-06-14 16:30:50.530674
+2023-06-14 16:30:50,533 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status
+2023-06-14 16:30:50,535 INFO    WriterThread:3907002 [datastore.py:open_for_write():85] open: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/run-nlm9mp31.wandb
+2023-06-14 16:30:50,536 DEBUG   SenderThread:3907002 [sender.py:send():369] send: header
+2023-06-14 16:30:50,645 DEBUG   SenderThread:3907002 [sender.py:send():369] send: run
+2023-06-14 16:30:52,766 INFO    SenderThread:3907002 [dir_watcher.py:__init__():211] watching files in: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files
+2023-06-14 16:30:52,767 INFO    SenderThread:3907002 [sender.py:_start_run_threads():1100] run started: nlm9mp31 with start time 1686760250.529075
+2023-06-14 16:30:52,768 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:30:52,768 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:30:52,777 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: check_version
+2023-06-14 16:30:52,780 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: check_version
+2023-06-14 16:30:53,773 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:30:54,234 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: run_start
+2023-06-14 16:30:54,244 DEBUG   HandlerThread:3907002 [system_info.py:__init__():31] System info init
+2023-06-14 16:30:54,244 DEBUG   HandlerThread:3907002 [system_info.py:__init__():46] System info init done
+2023-06-14 16:30:54,244 INFO    HandlerThread:3907002 [system_monitor.py:start():181] Starting system monitor
+2023-06-14 16:30:54,262 INFO    SystemMonitor:3907002 [system_monitor.py:_start():145] Starting system asset monitoring threads
+2023-06-14 16:30:54,262 INFO    HandlerThread:3907002 [system_monitor.py:probe():201] Collecting system info
+2023-06-14 16:30:54,278 INFO    SystemMonitor:3907002 [interfaces.py:start():190] Started cpu monitoring
+2023-06-14 16:30:54,305 INFO    SystemMonitor:3907002 [interfaces.py:start():190] Started disk monitoring
+2023-06-14 16:30:54,316 INFO    SystemMonitor:3907002 [interfaces.py:start():190] Started gpu monitoring
+2023-06-14 16:30:54,320 INFO    SystemMonitor:3907002 [interfaces.py:start():190] Started memory monitoring
+2023-06-14 16:30:54,321 INFO    SystemMonitor:3907002 [interfaces.py:start():190] Started network monitoring
+2023-06-14 16:30:54,388 DEBUG   HandlerThread:3907002 [system_info.py:probe():195] Probing system
+2023-06-14 16:30:54,402 DEBUG   HandlerThread:3907002 [system_info.py:_probe_git():180] Probing git
+2023-06-14 16:30:54,483 DEBUG   HandlerThread:3907002 [system_info.py:_probe_git():188] Probing git done
+2023-06-14 16:30:54,483 DEBUG   HandlerThread:3907002 [system_info.py:probe():240] Probing system done
+2023-06-14 16:30:54,483 DEBUG   HandlerThread:3907002 [system_monitor.py:probe():210] {'os': 'Linux-5.4.0-90-generic-x86_64-with-glibc2.31', 'python': '3.9.16', 'heartbeatAt': '2023-06-14T16:30:54.388684', 'startedAt': '2023-06-14T16:30:50.522065', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py', 'git': {'remote': 'https://github.com/hendrydong/LMFlow-diffusion.git', 'commit': '7eb976e6588110787f15fb1ef1b6ce7c4d84ae87'}, 'email': '[email protected]', 'root': '/home/deepanshu/lmflowdiff', 'host': 'A40-20230526-003', 'username': 'deepanshu', 'executable': '/home/deepanshu/anaconda3/envs/deepanshu/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3.4042656249999994, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.403, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.399, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.399, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.398, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.399, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}], 'disk': {'total': 218.5130844116211, 'used': 49.349273681640625}, 'gpu': 'NVIDIA A40', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}], 'memory': {'total': 755.7487335205078}}
+2023-06-14 16:30:54,483 INFO    HandlerThread:3907002 [system_monitor.py:probe():211] Finished collecting system info
+2023-06-14 16:30:54,483 INFO    HandlerThread:3907002 [system_monitor.py:probe():214] Publishing system info
+2023-06-14 16:30:54,483 DEBUG   HandlerThread:3907002 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
+2023-06-14 16:30:54,484 DEBUG   HandlerThread:3907002 [system_info.py:_save_pip():67] Saving pip packages done
+2023-06-14 16:30:54,484 DEBUG   HandlerThread:3907002 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment
+2023-06-14 16:30:54,773 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/requirements.txt
+2023-06-14 16:30:54,775 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/conda-environment.yaml
+2023-06-14 16:31:01,773 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/conda-environment.yaml
+2023-06-14 16:31:01,862 DEBUG   HandlerThread:3907002 [system_info.py:_save_conda():86] Saving conda packages done
+2023-06-14 16:31:01,863 INFO    HandlerThread:3907002 [system_monitor.py:probe():216] Finished publishing system info
+2023-06-14 16:31:01,867 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:01,867 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: keepalive
+2023-06-14 16:31:01,867 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:01,868 DEBUG   SenderThread:3907002 [sender.py:send():369] send: files
+2023-06-14 16:31:01,868 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now
+2023-06-14 16:31:01,873 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:31:01,873 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:31:02,774 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-metadata.json
+2023-06-14 16:31:03,057 DEBUG   SenderThread:3907002 [sender.py:send():369] send: telemetry
+2023-06-14 16:31:03,057 DEBUG   SenderThread:3907002 [sender.py:send():369] send: config
+2023-06-14 16:31:03,418 INFO    wandb-upload_0:3907002 [upload_job.py:push():133] Uploaded file /tmp/tmpr4sjdcacwandb/fpgf4rpk-wandb-metadata.json
+2023-06-14 16:31:03,775 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:05,776 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:06,490 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:06,694 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:07,021 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:07,022 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:07,023 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:07,023 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:07,319 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:07,320 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:07,321 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:07,321 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:07,638 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:07,639 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:07,639 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:07,640 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:07,777 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:07,778 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:07,959 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:07,960 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:07,960 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:07,960 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:08,340 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:08,341 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:08,341 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:08,342 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:08,641 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:08,642 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:08,642 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:08,643 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:08,778 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:08,944 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:08,944 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:08,944 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:08,944 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:09,206 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:09,209 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:09,209 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:09,210 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:09,551 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:09,552 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:09,552 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:09,552 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:09,778 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:09,779 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:09,788 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:09,788 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:09,789 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:09,789 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:10,059 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:10,059 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:10,060 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:10,060 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:10,336 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:10,336 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:10,337 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:10,337 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:10,596 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:10,597 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:10,597 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:10,597 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:10,779 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:10,864 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:10,865 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:10,865 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:10,866 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:11,156 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:11,157 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:11,158 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:11,158 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:11,389 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:11,390 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:11,390 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:11,391 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:11,695 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:11,695 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:11,696 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:11,696 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:11,696 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:11,779 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:11,779 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:11,957 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:11,958 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:11,958 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:11,958 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:12,214 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:12,214 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:12,214 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:12,215 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:12,459 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:12,460 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:12,460 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:12,461 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:12,779 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:12,815 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:12,816 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:12,816 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:12,817 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:13,140 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:13,140 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:13,141 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:13,141 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:13,431 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:13,432 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:13,432 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:13,432 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:13,742 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:13,743 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:13,743 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:13,744 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:13,786 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:13,786 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:14,010 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:14,011 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:14,011 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:14,011 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:14,336 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:14,337 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:14,337 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:14,338 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:14,594 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:14,595 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:14,595 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:14,596 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:14,780 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:14,933 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:14,934 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:14,935 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:14,935 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:15,202 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:15,203 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:15,203 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:15,203 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:15,565 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:15,567 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:15,567 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:15,567 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:15,781 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:15,781 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:15,887 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:15,890 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:15,891 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:15,891 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:16,181 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:16,182 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:16,182 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:16,182 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:16,423 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:16,424 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:16,424 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:16,425 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:16,658 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:16,659 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:16,659 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:16,659 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:16,781 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:16,873 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:31:16,873 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:31:16,967 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:17,199 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:17,266 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:17,266 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:17,266 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:17,268 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:17,269 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:17,269 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:17,269 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:17,460 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:17,461 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:17,461 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:17,461 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:17,714 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:17,721 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:17,722 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:17,722 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:17,782 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:17,782 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:18,032 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:18,034 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:18,035 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:18,035 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:18,293 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:18,294 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:18,294 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:18,295 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:18,615 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:18,616 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:18,617 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:18,617 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:18,782 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:19,072 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:19,074 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:19,074 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:19,075 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:19,649 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:19,650 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:19,650 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:19,651 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:19,783 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:19,783 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:19,942 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:19,943 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:19,944 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:19,944 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:20,378 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:20,379 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:20,379 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:20,380 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:20,783 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:20,857 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:20,859 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:20,859 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:20,860 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:21,473 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:21,474 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:21,475 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:21,475 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:21,789 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:21,791 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:22,168 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:22,170 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:22,170 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:22,172 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:22,536 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:22,538 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:22,789 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:23,005 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:23,239 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:23,383 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:23,383 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:23,384 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:23,384 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:23,384 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:23,384 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:23,384 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:23,384 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:23,384 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:23,471 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:23,472 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:23,472 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:23,472 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:23,688 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:23,689 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:23,689 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:23,690 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:23,790 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:23,790 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:23,790 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/config.yaml
+2023-06-14 16:31:23,911 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:23,912 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:23,912 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:23,912 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:24,155 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:24,156 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:24,156 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:24,156 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:24,406 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:24,407 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:24,407 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:24,407 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:24,754 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:24,755 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:24,756 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:24,756 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:24,790 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:25,131 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:25,132 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:25,133 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:25,133 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:25,646 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:25,650 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:25,651 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:25,652 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:25,794 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:25,796 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:27,794 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:29,158 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:29,795 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:31,796 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:31,880 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:31:31,881 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:31:33,797 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:34,333 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:35,797 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:37,798 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:39,140 DEBUG   SenderThread:3907002 [sender.py:send():369] send: files
+2023-06-14 16:31:39,140 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file media/images/validation_60_e9c4a470ad900801f7de.png with policy now
+2023-06-14 16:31:39,148 DEBUG   SenderThread:3907002 [sender.py:send():369] send: files
+2023-06-14 16:31:39,148 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file media/images/validation_60_7673b7ce51f7ba8dbeaa.png with policy now
+2023-06-14 16:31:39,149 DEBUG   SenderThread:3907002 [sender.py:send():369] send: files
+2023-06-14 16:31:39,149 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file media/images/validation_60_657ebe0b72dbaae9b6f3.png with policy now
+2023-06-14 16:31:39,167 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:31:39,168 DEBUG   SenderThread:3907002 [sender.py:send():369] send: history
+2023-06-14 16:31:39,168 DEBUG   SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:31:39,168 INFO    SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:31:39,799 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json
+2023-06-14 16:31:39,799 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:39,800 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images/validation_60_7673b7ce51f7ba8dbeaa.png
+2023-06-14 16:31:39,800 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images/validation_60_657ebe0b72dbaae9b6f3.png
+2023-06-14 16:31:39,800 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images/validation_60_e9c4a470ad900801f7de.png
+2023-06-14 16:31:39,800 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images
+2023-06-14 16:31:39,800 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media
+2023-06-14 16:31:40,295 DEBUG   HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:31:41,801 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log
+2023-06-14 16:31:43,801 INFO    Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log

wandb/wandb/run-20230614_163050-nlm9mp31/logs/debug.log

@@ -0,0 +1,30 @@
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Configure stats pid to 3906677
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/.config/wandb/settings
+2023-06-14 16:30:50,524 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/lmflowdiff/Model/wandb/wandb/settings
+2023-06-14 16:30:50,525 WARNING MainThread:3906677 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CACHE_DIR
+2023-06-14 16:30:50,525 WARNING MainThread:3906677 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CONFIG_DIR
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/deepanshu/lmflowdiff/Model/wandb'}
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-06-14 16:30:50,525 WARNING MainThread:3906677 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/deepanshu/LMFlow-diffusion-main/examples/finetune.py
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py'}
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:_log_setup():507] Logging user logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/logs/debug.log
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:_log_setup():508] Logging internal logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/logs/debug-internal.log
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():547] calling init triggers
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():596] starting backend
+2023-06-14 16:30:50,525 INFO    MainThread:3906677 [wandb_init.py:init():600] setting up manager
+2023-06-14 16:30:50,527 INFO    MainThread:3906677 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-06-14 16:30:50,528 INFO    MainThread:3906677 [wandb_init.py:init():606] backend started and connected
+2023-06-14 16:30:50,534 INFO    MainThread:3906677 [wandb_init.py:init():703] updated telemetry
+2023-06-14 16:30:50,643 INFO    MainThread:3906677 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-06-14 16:30:52,776 INFO    MainThread:3906677 [wandb_run.py:_on_init():2176] communicating current version
+2023-06-14 16:30:54,228 INFO    MainThread:3906677 [wandb_run.py:_on_init():2185] got version response 
+2023-06-14 16:30:54,228 INFO    MainThread:3906677 [wandb_init.py:init():787] starting run threads in backend
+2023-06-14 16:31:01,872 INFO    MainThread:3906677 [wandb_run.py:_console_start():2155] atexit reg
+2023-06-14 16:31:01,873 INFO    MainThread:3906677 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-06-14 16:31:01,873 INFO    MainThread:3906677 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-06-14 16:31:01,873 INFO    MainThread:3906677 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-06-14 16:31:01,874 INFO    MainThread:3906677 [wandb_init.py:init():828] run started, returning control to user process
+2023-06-14 16:31:01,875 INFO    MainThread:3906677 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 2, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False}