diff --git a/.gitignore b/.gitignore
index 6d8b31a5bd5d4008b097bc4b7e45b83909e55d0e..6c311be704b9b912702a7ad4f6c67f7367e03fa4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
.DS_Store
*/__pycache__
-root
\ No newline at end of file
+*/trained_ddpm
+root
diff --git a/.ipynb_checkpoints/experiment_creator-checkpoint.ipynb b/.ipynb_checkpoints/experiment_creator-checkpoint.ipynb
index 5d49730e936eb6a3087b058c676c73b29603859d..2e35c4ee2b0852f0c14234c07a8669da508e5f5b 100644
--- a/.ipynb_checkpoints/experiment_creator-checkpoint.ipynb
+++ b/.ipynb_checkpoints/experiment_creator-checkpoint.ipynb
@@ -37,20 +37,20 @@
"\n",
"#basic settings:\n",
"learning_rate = 0.0001\n",
- "batchsize = 128\n",
- "datapath = \"work/lect0100/lhq256\"\n",
+ "batchsize = 8\n",
+ "datapath = \"/work/lect0100/lhq_256\"\n",
"checkpoint_path = None #when training from checkpoint\n",
- "experimentname = \"/work/lect0100/results/\" + \"test1\" #always change experiment name! \n",
+ "experimentname = \"/Users/gonzalo/Desktop/testing/\" + \"test1\" #always change experiment name! \n",
"epochs = 20\n",
- "diffusion_steps = 500\n",
+ "diffusion_steps = 25\n",
"image_size = 64\n",
"channels = 3\n",
"store_iter = 5\n",
"optimizername = \"torch.optim.AdamW\"\n",
- "name_appendix = 'DM_testing_0'# id for WANDB\n",
+ "name_appendix = 'DM_bottleneck'# id for WANDB\n",
"\n",
"#advanced settings: change directly in dictionary \n",
- "meta_setting = dict(modelname = \"UNet_Unconditional_Diffusion\",\n",
+ "meta_setting = dict(modelname = \"UNet_Unconditional_Diffusion_Bottleneck_Variant\",\n",
" dataset = \"UnconditionalDataset\",\n",
" framework = \"DDPM\",\n",
" trainloop_function = \"ddpm_trainer\",\n",
@@ -76,7 +76,7 @@
" time_channels=diffusion_steps, # number of time channels #TODO same as diffusion steps? \n",
" num_stages=4, # number of stages in contracting/expansive path\n",
" stage_list=None, # specify number of features produced by stages\n",
- " num_blocks=2, # number of ConvResBlock in each contracting/expansive path\n",
+ " num_blocks=1, # number of ConvResBlock in each contracting/expansive path\n",
" num_groupnorm_groups=32, # number of groups used in Group Normalization inside a ConvResBlock\n",
" dropout=0.1, # drop-out to be applied inside a ConvResBlock\n",
" attention_list=None, # specify MHA pattern across stages\n",
diff --git a/README.md b/README.md
index 20795a961fb5682ca572784f722decbe42dcf144..3534d8bd5ee8850588e0c621718a27bba85f4a14 100644
--- a/README.md
+++ b/README.md
@@ -3,3 +3,12 @@
## Description
This is the repository for the *diffusion project* for the **(PR) Laboratory Deep Learning 23ss**
+This repository houses our comprehensive pipeline, designed to conveniently train, sample from, and evaluate our unconditional diffusion model.
+The pipeline is initiated via the experiment_creator.ipynb notebook, which is separately run our local machine. This notebook allows for the configuration of every aspect of the diffusion model, including all hyperparameters. These configurations extend to the underlying neural backbone UNet, as well as the training parameters, such as training from checkpoint, Weights & Biases run name for resumption, optimizer selection, adjustment of the learning rate for manual learning rate scheduling, and more. Moreover, it includes parameters for evaluating a and sampling images via a trained diffusion models.
+
+Upon execution, the notebook generates individual JSON files, encapsulating all the hyperparameter information. When running the model on the HPC, we can choose between the operations 'train', 'sample', and 'evaluate'. These operations automatically extract the necessary hyperparameters from the JSON files and perform their respective tasks. This process is managed by the main.py file. The remaining files contain all the necessary functions optimized for HPC to perform the aforementioned tasks.
+
+Every uniquely trained diffusion model has its own experiment folder, given by its WANDB run name. It holds four different directories: settings, trained_ddpm, samples, and evaluations. The settings folder holds the JSON files specifying the diffusion model's configurations as well as the arguments for the training, sampling, and evaluation functions. The trained_ddpm folder contains .pth files storing the weights and biases of this experiment's diffusion model, which have been saved at different epoch milestones while training. Upon resuming training, the pipeline automatically finds the highest epoch model in trained_ddpm and continues training from there. When sampling images from these trained diffusion models, the samples are stored in different directories named epoch_{i}. This is done so we know what epoch i version of the diffusion model was used to generate these samples.
+
+
+
diff --git a/dataloader/load.py b/dataloader/load.py
index a57ea04a6e0c438f9ba654874a3952c81b63f781..93d033b4f1d6ff30ae0bc8abde1aebcaaa894c49 100644
--- a/dataloader/load.py
+++ b/dataloader/load.py
@@ -4,11 +4,11 @@ from torchvision import transforms
import os
from PIL import Image
import pandas as pd
+import numpy as np
class UnconditionalDataset(Dataset):
def __init__(self,fpath,img_size,train,frac =0.8,skip_first_n = 0,ext = ".png",transform=True ):
"""
- Customized to datasets where all images are within a folder and the filenames are sorted by likeliehood.(Landscape Dataset)
Args:
fpath (string): Path to the folder where images are stored
img_size (int): size of output image img_size=height=width
@@ -42,17 +42,30 @@ class UnconditionalDataset(Dataset):
self.df = df_test
if transform:
- self.transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5)),
- transforms.Resize(img_size,antialias=True),transforms.RandomHorizontalFlip(p=0.5)])
+ intermediate_size = 150
+ theta = np.pi/4 -np.arccos(intermediate_size/(np.sqrt(2)*img_size)) #Check dataloading.ipynb in analysis-depot for more details
+
+ transform_rotate = transforms.Compose([transforms.ToTensor(),transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5)),
+ transforms.Resize(intermediate_size,antialias=True),
+ transforms.RandomRotation(theta/np.pi*180,interpolation=transforms.InterpolationMode.BILINEAR),
+ transforms.CenterCrop(img_size),transforms.RandomHorizontalFlip(p=0.5)])
+
+ transform_randomcrop = transforms.Compose([transforms.ToTensor(),transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5)),
+ transforms.Resize(intermediate_size),transforms.RandomCrop(img_size),transforms.RandomHorizontalFlip(p=0.5)])
+
+ self.transform = transforms.RandomChoice([transform_rotate,transform_randomcrop])
+ else :
+ self.transform = transforms.Compose([transforms.ToTensor(),
+ transforms.Resize(img_size)])
def __len__(self):
return len(self.df)
def __getitem__(self,idx):
- path = self.df.iloc[idx].Filepath
+ path = self.df.iloc[idx].Filepaths
img = Image.open(path)
return self.transform(img),0
-
+
def tensor2PIL(self,img):
back2pil = transforms.Compose([transforms.Normalize(mean=(-1,-1,-1),std=(2,2,2)),transforms.ToPILImage()])
return back2pil(img)
\ No newline at end of file
diff --git a/evaluation/evaluate.py b/evaluation/evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd489723d8e7009d6832e43c9aeee20e16461628
--- /dev/null
+++ b/evaluation/evaluate.py
@@ -0,0 +1,19 @@
+from evaluation.sample import ddpm_sampler
+
+def ddpm_evaluator(model,
+ device,
+ dataloader,
+ checkpoint,
+ experiment_path
+ ):
+ '''
+ Takes a trained diffusion model from 'checkpoint' and evaluates its performance on the test
+ dataset 'dataloader'. ....
+
+ model: Properly initialized DDPM model
+ checkpoint: Name of the saved pth. file containing the trained weights and biases
+ experiment_path: Path to the experiment folder where the evaluation results will be stored
+ testloader: Loads the test dataset
+ TODO ...
+ '''
+ return None
diff --git a/evaluation/sample.py b/evaluation/sample.py
new file mode 100644
index 0000000000000000000000000000000000000000..b7504d3140778b58b87700a115a60c9c3db301b3
--- /dev/null
+++ b/evaluation/sample.py
@@ -0,0 +1,72 @@
+import os
+import torch
+from torchvision import transforms
+import re
+
+def ddpm_sampler(model, checkpoint, experiment_path, device, intermediate=False, batch_size=15):
+ '''
+ Samples a tensor of 'batch_size' images from a trained diffusion model with 'checkpoint'. The generated
+ images are stored in the directory 'experiment_path/samples/epoch_{e}/sample_{j}. Where e is the epoch
+ w.r.t. the model which we are sampling form and j is an index separating images from each call of the
+ sampling function for the given mode.
+
+ model: Diffusion model
+ checkpoint: Name of the saved pth. file containing the trained weights and biases
+ experiment_path: Path to the experiment directory where the samples will saved under the diectory samples
+ batch_size: The number of images to sample
+ intermediate: Bool value. If False the sampling function will draw a batch of images, else it will just
+ sample a single image, but store all the intermediate noised latents along the reverse chain
+ '''
+
+ # load model
+ try:
+ checkpoint_path = f'{experiment_path}trained_ddpm/{checkpoint}'
+ checkpoint = torch.load(checkpoint_path)
+ # load weights and biases of the U-Net
+ net_state_dict = checkpoint['model']
+ model.net.load_state_dict(net_state_dict)
+ model = model.to(device)
+ except Exception as e:
+ print("Error loading checkpoint. Exception:", e)
+
+ # create samples directory for the complete experiment (if first time sampling images)
+ output_dir = f'{experiment_path}samples/'
+ #output_dir = os.path.join(experiment_path,'/samples/')
+ os.makedirs(output_dir, exist_ok=True)
+
+ # create sample directory for the current version of the trained model
+ model_name = os.path.basename(checkpoint_path)
+ epoch = re.findall(r'\d+', model_name)
+ if epoch:
+ e = int(epoch[0])
+ else:
+ raise ValueError(f"No digit found in the filename: {filename}")
+ model_dir = os.path.join(output_dir,f'epoch_{e}')
+ os.makedirs(model_dir, exist_ok=True)
+
+ # create the sample directory for this sampling run for the current version of the model
+ sample_dir_list = [d for d in os.listdir(model_dir) if os.path.isdir(os.path.join(model_dir, d))]
+ indx_list = [int(d.split('_')[1]) for d in sample_dir_list if d.startswith('sample_')]
+ j = max(indx_list, default=-1) + 1
+ sample_dir = os.path.join(model_dir, f'sample_{j}')
+ os.makedirs(sample_dir, exist_ok=True)
+
+ # transform
+ back2pil = transforms.Compose([transforms.Normalize(mean=(-1,-1,-1),std=(2,2,2)),transforms.ToPILImage()])
+
+ # generate batch_size images
+ if intermediate:
+ generated = model.sample_intermediates_latents()
+ name = 'sample_intermediate'
+ else:
+ generated = model.sample(batch_size=batch_size)
+ name = 'sample'
+
+ # save generated images
+ for i in range(generated.size(0)):
+ image = back2pil(generated[i])
+ image_path = os.path.join(sample_dir, f'{name}_{j}_{i}.png')
+ try:
+ image.save(image_path)
+ except Exception as e:
+ print("Error saving image. Exception:", e)
diff --git a/experiment_creator.ipynb b/experiment_creator.ipynb
index 0a552b3c428b07aa07c990f6bc90ffc2947279f1..17963ce4ccd8bc3c2e41dc95cea6aca9c9c783ea 100644
--- a/experiment_creator.ipynb
+++ b/experiment_creator.ipynb
@@ -3,26 +3,32 @@
{
"cell_type": "code",
"execution_count": 7,
- "metadata": {},
+ "metadata": {
+ "scrolled": true
+ },
"outputs": [],
"source": [
"from trainer.train import *\n",
"from dataloader.load import *\n",
"from models.Framework import *\n",
- "from models.unet_unconditional_diffusion import *\n",
+ "from models.all_unets import *\n",
"import torch \n",
- "from torch import nn \n"
+ "from torch import nn "
]
},
{
+ "attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# Prepare experiment\n",
- "1. Adapt settings below (for data path, only use absolute paths!!)\n",
- "2. run both cells of the notebook, this creates a folder containing the json setting files \n",
- "2. put the folder on the HPC\n",
- "3. the following command starts the training `python main.py train \"<absolute path of folder in hpc>\"` add it to the batch file "
+ "1. Choose Hyperparameter Settings\n",
+ "2. Run notebook on local maschine to generate experiment folder with the JSON files containing the settings\n",
+ "3. scp experiment folder to the HPC\n",
+ "4. Run Pipeline by adding following to batch file:\n",
+ "- Train Model:       `python main.py train \"<absolute path of experiment folder in hpc>\"`\n",
+ "- Sample Images:     `python main.py sample \"<absolute path of experiment folder in hpc>\"`\n",
+ "- Evaluate Model:     `python main.py evaluate \"<absolute path of experiment folder in hpc>\"`"
]
},
{
@@ -33,31 +39,59 @@
"source": [
"import torch \n",
"\n",
- "#path to store, path to load data , path to checkpoint \n",
+ "####\n",
+ "# Settings\n",
+ "####\n",
"\n",
- "#basic settings:\n",
- "learning_rate = 0.0001\n",
- "batchsize = 128\n",
- "datapath = \"work/lect0100/lhq_256\"\n",
- "checkpoint_path = None #when training from checkpoint\n",
- "experimentname = \"/work/lect0100/results/\" + \"test1\" #always change experiment name! \n",
- "epochs = 20\n",
- "diffusion_steps = 500\n",
+ "# Dataset path\n",
+ "datapath = \"/work/lect0100/lhq_256\"\n",
+ "\n",
+ "# Experiment setup\n",
+ "run_name = 'batch_timesteps' # WANDB and experiment folder Name!\n",
+ "checkpoint = None #'model_epoch_8.pth' # Name of checkpoint pth file or None \n",
+ "experiment_path = '/work/lect0100/experiments_gonzalo/'+ run_name +'/'\n",
+ "\n",
+ "# Path to save generated experiment folder on local machine\n",
+ "local_path =\"/Users/gonzalo/Desktop/\" + run_name + '/settings'\n",
+ "\n",
+ "# Diffusion Model Settings\n",
+ "diffusion_steps = 200\n",
"image_size = 64\n",
"channels = 3\n",
- "store_iter = 5\n",
+ "\n",
+ "# Training\n",
+ "batchsize = 32\n",
+ "epochs = 30\n",
+ "store_iter = 1\n",
+ "eval_iter = 500\n",
+ "learning_rate = 0.0001\n",
"optimizername = \"torch.optim.AdamW\"\n",
- "name_appendix = 'DM_testing_0'# id for WANDB\n",
+ "optimizer_params = None\n",
+ "verbose = True\n",
+ "# checkpoint = None #(If no checkpoint training, ie. random weights)\n",
+ "\n",
+ "# Sampling \n",
+ "sample_size = 10\n",
+ "intermediate = False # True if you want to sample one image and all ist intermediate latents\n",
"\n",
- "#advanced settings: change directly in dictionary \n",
- "meta_setting = dict(modelname = \"UNet_Unconditional_Diffusion\",\n",
+ "\n",
+ "# Evaluating\n",
+ "...\n",
+ "\n",
+ "\n",
+ "\n",
+ "###\n",
+ "# Advanced Settings Dictionaries\n",
+ "###\n",
+ "\n",
+ "meta_setting = dict(modelname = \"UNet_Res\",\n",
" dataset = \"UnconditionalDataset\",\n",
" framework = \"DDPM\",\n",
" trainloop_function = \"ddpm_trainer\",\n",
- " batchsize = batchsize,\n",
+ " sampling_function = 'ddpm_sampler',\n",
+ " evaluation_function = 'ddpm_evaluator',\n",
+ " batchsize = batchsize\n",
" )\n",
- "\n",
- "\n",
"dataset_setting = dict(fpath = datapath,\n",
" img_size = image_size,\n",
" frac =0.8,\n",
@@ -66,7 +100,12 @@
" transform=True\n",
" )\n",
"\n",
- "\n",
+ "model_setting = dict( n_channels=64,\n",
+ " fctr = [1,2,4,4,8],\n",
+ " time_dim=256,\n",
+ " )\n",
+ "\"\"\"\n",
+ "outdated\n",
"model_setting = dict( channels_in=channels, \n",
" channels_out =channels , \n",
" activation='relu', # activation function. Options: {'relu', 'leakyrelu', 'selu', 'gelu', 'silu'/'swish'}\n",
@@ -76,14 +115,13 @@
" time_channels=diffusion_steps, # number of time channels #TODO same as diffusion steps? \n",
" num_stages=4, # number of stages in contracting/expansive path\n",
" stage_list=None, # specify number of features produced by stages\n",
- " num_blocks=2, # number of ConvResBlock in each contracting/expansive path\n",
+ " num_blocks=1, # number of ConvResBlock in each contracting/expansive path\n",
" num_groupnorm_groups=32, # number of groups used in Group Normalization inside a ConvResBlock\n",
" dropout=0.1, # drop-out to be applied inside a ConvResBlock\n",
" attention_list=None, # specify MHA pattern across stages\n",
" num_attention_heads=1,\n",
" )\n",
- "\n",
- "\n",
+ "\"\"\"\n",
"framework_setting = dict(\n",
" diffusion_steps = diffusion_steps, # dont change!!\n",
" out_shape = (channels,image_size,image_size), # dont change!!\n",
@@ -93,26 +131,34 @@
" alpha_bar_lower_bound = 0.9,\n",
" var_schedule = 'same', \n",
" kl_loss = 'simplified', \n",
- " recon_loss = 'none',\n",
- " \n",
+ " recon_loss = 'nll',\n",
" )\n",
- "\n",
- "\n",
"training_setting = dict(\n",
- " epochs = epochs,\n",
- " store_iter = store_iter,\n",
- " eval_iter = 3,\n",
- " optimizer_class=optimizername, \n",
- " optimizer_params=dict(lr=learning_rate), # don't change!\n",
- " scheduler_class= None, \n",
- " scheduler_params=None,\n",
- " last_epoch=-1,\n",
- " learning_rate = learning_rate,\n",
- " lr_schedule = False,\n",
- " verbose = True,\n",
- " name_appendix=name_appendix,\n",
- " checkpoint_path= checkpoint_path,\n",
- " )"
+ " epochs = epochs,\n",
+ " store_iter = store_iter,\n",
+ " eval_iter = eval_iter,\n",
+ " optimizer_class=optimizername, \n",
+ " optimizer_params = optimizer_params,\n",
+ " #optimizer_params=dict(lr=learning_rate), # don't change! \n",
+ " learning_rate = learning_rate,\n",
+ " run_name=run_name,\n",
+ " checkpoint= checkpoint,\n",
+ " experiment_path = experiment_path,\n",
+ " verbose = verbose,\n",
+ " T_max = 5*10000, # cosine lr param\n",
+ " eta_min= 1e-5, # cosine lr param\n",
+ " )\n",
+ "sampling_setting = dict( \n",
+ " checkpoint = checkpoint, \n",
+ " experiment_path = experiment_path, \n",
+ " batch_size = sample_size,\n",
+ " intermediate = intermediate\n",
+ " )\n",
+ "# TODO\n",
+ "evaluation_setting = dict(\n",
+ " checkpoint = checkpoint,\n",
+ " experiment_path = experiment_path,\n",
+ " ) "
]
},
{
@@ -124,21 +170,30 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "path already exists, pick a new name!\n",
- "break\n"
+ "create folder\n",
+ "folder created \n",
+ "stored json files in folder\n",
+ "{'modelname': 'UNet_Unconditional_Diffusion_Bottleneck_Variant', 'dataset': 'UnconditionalDataset', 'framework': 'DDPM', 'trainloop_function': 'ddpm_trainer', 'sampling_function': 'ddpm_sampler', 'evaluation_function': 'ddpm_evaluator', 'batchsize': 32}\n",
+ "{'fpath': '/work/lect0100/lhq_256', 'img_size': 64, 'frac': 0.8, 'skip_first_n': 0, 'ext': '.png', 'transform': True}\n",
+ "{'channels_in': 3, 'channels_out': 3, 'activation': 'relu', 'weight_init': 'he', 'projection_features': 64, 'time_dim': 32, 'time_channels': 200, 'num_stages': 4, 'stage_list': None, 'num_blocks': 1, 'num_groupnorm_groups': 32, 'dropout': 0.1, 'attention_list': None, 'num_attention_heads': 1}\n",
+ "{'diffusion_steps': 200, 'out_shape': (3, 64, 64), 'noise_schedule': 'linear', 'beta_1': 0.0001, 'beta_T': 0.02, 'alpha_bar_lower_bound': 0.9, 'var_schedule': 'same', 'kl_loss': 'simplified', 'recon_loss': 'nll'}\n",
+ "{'epochs': 30, 'store_iter': 1, 'eval_iter': 500, 'optimizer_class': 'torch.optim.AdamW', 'optimizer_params': None, 'learning_rate': 0.0001, 'run_name': 'batch_timesteps', 'checkpoint': None, 'experiment_path': '/work/lect0100/experiments_gonzalo/batch_timesteps/', 'verbose': True}\n",
+ "{'checkpoint': None, 'experiment_path': '/work/lect0100/experiments_gonzalo/batch_timesteps/', 'batch_size': 10, 'intermediate': False}\n",
+ "{'checkpoint': None, 'experiment_path': '/work/lect0100/experiments_gonzalo/batch_timesteps/'}\n"
]
}
],
"source": [
"import os\n",
"import json\n",
- "f = experimentname\n",
+ "f = local_path\n",
"if os.path.exists(f):\n",
" print(\"path already exists, pick a new name!\")\n",
" print(\"break\")\n",
"else:\n",
" print(\"create folder\")\n",
- " os.mkdir(f)\n",
+ " #os.mkdir(f)\n",
+ " os.makedirs(f, exist_ok=True)\n",
" print(\"folder created \")\n",
" with open(f+\"/meta_setting.json\",\"w+\") as fp:\n",
" json.dump(meta_setting,fp)\n",
@@ -154,22 +209,44 @@
"\n",
" with open(f+\"/training_setting.json\",\"w+\") as fp:\n",
" json.dump(training_setting,fp)\n",
- "\n",
+ " \n",
+ " with open(f+\"/sampling_setting.json\",\"w+\") as fp:\n",
+ " json.dump(sampling_setting,fp)\n",
+ " \n",
+ " with open(f+\"/evaluation_setting.json\",\"w+\") as fp:\n",
+ " json.dump(evaluation_setting,fp)\n",
+ " \n",
" print(\"stored json files in folder\")\n",
" print(meta_setting)\n",
" print(dataset_setting)\n",
" print(model_setting)\n",
" print(framework_setting)\n",
" print(training_setting)\n",
+ " print(sampling_setting)\n",
+ " print(evaluation_setting)\n",
" "
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3.9 (pytorch)",
+ "display_name": "env",
"language": "python",
- "name": "pytorch"
+ "name": "env"
},
"language_info": {
"codemirror_mode": {
@@ -181,7 +258,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.16"
+ "version": "3.10.6"
}
},
"nbformat": 4,
diff --git a/main.py b/main.py
index b0c5ea6ed9ebfeff450ea82a3f887c2f957976a6..dedee957f94dd6cd4a20341633a8e30ee494726c 100644
--- a/main.py
+++ b/main.py
@@ -1,4 +1,13 @@
+import json
+import sys
+from dataloader.load import *
+from models.Framework import *
+from trainer.train import ddpm_trainer
+from evaluation.sample import ddpm_sampler
+from evaluation.evaluate import ddpm_evaluator
+from models.all_unets import *
+import torch
def train_func(f):
@@ -31,8 +40,11 @@ def train_func(f):
test_dataloader = torch.utils.data.DataLoader(test_dataset,batch_size=batchsize)
- model = globals()[meta_setting["modelname"]](**model_setting)
- framework = globals()[meta_setting["framework"]](net = model,device=device, **framework_setting)
+ net = globals()[meta_setting["modelname"]](**model_setting).to(device)
+ #net = torch.compile(net)
+ net = net.to(device)
+
+ framework = globals()[meta_setting["framework"]](net = net,device=device, **framework_setting)
print(f"META SETTINGS:\n\n {meta_setting}\n\n")
print(f"DATASET SETTINGS:\n\n {dataset_setting}\n\n")
@@ -43,33 +55,99 @@ def train_func(f):
print("\n\nSTART TRAINING\n\n")
globals()[meta_setting["trainloop_function"]](model=framework,device=device, trainloader = training_dataloader, testloader = test_dataloader,safepath = f,**training_setting,)
print("\n\nFINISHED TRAINING\n\n")
-def generate_func():
- print('generated images')
+
+
+
+def sample_func(f):
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
+ print(f"device: {device}\n\n")
+ print(f"folderpath: {f}\n\n")
+
+ with open(f+"/meta_setting.json","r") as fp:
+ meta_setting = json.load(fp)
+
+ with open(f+"/model_setting.json","r") as fp:
+ model_setting = json.load(fp)
+
+ with open(f+"/framework_setting.json","r") as fp:
+ framework_setting = json.load(fp)
+
+ with open(f+"/sampling_setting.json","r") as fp:
+ sampling_setting = json.load(fp)
+
+ # init Unet
+ batchsize = meta_setting["batchsize"]
+ net = globals()[meta_setting["modelname"]](**model_setting).to(device)
+ #net = torch.compile(net)
+ net = net.to(device)
+ # init unconditional diffusion model
+ framework = globals()[meta_setting["framework"]](net = net,device=device, **framework_setting)
+
+ print(f"META SETTINGS:\n\n {meta_setting}\n\n")
+ print(f"MODEL SETTINGS:\n\n {model_setting}\n\n")
+ print(f"FRAMEWORK SETTINGS:\n\n {framework_setting}\n\n")
+ print(f"SAMPLING SETTINGS:\n\n {sampling_setting}\n\n")
+
+ print("\n\nSTART SAMPLING\n\n")
+ globals()[meta_setting["sampling_function"]](model=framework,device=device, **sampling_setting,)
+ print("\n\nFINISHED SAMPLING\n\n")
+
+
+
+def evaluate_func(f):
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
+ print(f"device: {device}\n\n")
+ print(f"folderpath: {f}\n\n")
+
+ with open(f+"/meta_setting.json","r") as fp:
+ meta_setting = json.load(fp)
+
+ with open(f+"/model_setting.json","r") as fp:
+ model_setting = json.load(fp)
+
+ with open(f+"/framework_setting.json","r") as fp:
+ framework_setting = json.load(fp)
+
+ with open(f+"/evaluation_setting.json","r") as fp:
+ evaluation_setting = json.load(fp)
+
+ with open(f+"/dataset_setting.json","r") as fp:
+ dataset_setting = json.load(fp)
+
+ # load dataset
+ batchsize = meta_setting["batchsize"]
+ test_dataset = globals()[meta_setting["dataset"]](train = False,**dataset_setting)
+ test_dataloader = torch.utils.data.DataLoader(test_dataset,batch_size=batchsize)
+
+ # init Unet
+ net = globals()[meta_setting["modelname"]](**model_setting).to(device)
+ #net = torch.compile(net)
+ net = net.to(device)
-def pipeline_func(f):
- # TODO
- train_func(f)
- generate_func()
+ # init unconditional diffusion model
+ framework = globals()[meta_setting["framework"]](net = net,device=device, **framework_setting)
+
+ print(f"META SETTINGS:\n\n {meta_setting}\n\n")
+ print(f"DATASET SETTINGS:\n\n {dataset_setting}\n\n")
+ print(f"MODEL SETTINGS:\n\n {model_setting}\n\n")
+ print(f"FRAMEWORK SETTINGS:\n\n {framework_setting}\n\n")
+ print(f"EVALUATION SETTINGS:\n\n {evaluation_setting}\n\n")
+
+ print("\n\nSTART EVALUATION\n\n")
+ globals()[meta_setting["evaluation_function"]](model=framework, device=device, testloader = test_dataloader,safepath = f,**evaluation_setting,)
+ print("\n\nFINISHED EVALUATION\n\n")
+
+
+
-def hello(name):
- print(f'Hello {name}!')
if __name__ == '__main__':
- import json
- import sys
- from trainer.train import *
- from dataloader.load import *
- from models.Framework import *
- from models.unet_unconditional_diffusion import *
- from models.unet import UNet
- import torch
- from torch import nn
print(sys.argv)
- functions = {'train': train_func,"hello":hello}
+ functions = {'train': train_func,'sample': sample_func,'evaluate': evaluate_func}
functions[sys.argv[1]](sys.argv[2])
-
+
diff --git a/models/Framework.py b/models/Framework.py
index c289697d6ffb2b6380555546274585ce6a635087..84783d3cf4d12350f2d1de54947dcd7ffb49c766 100644
--- a/models/Framework.py
+++ b/models/Framework.py
@@ -1,10 +1,11 @@
import torch
from torch import nn
import torch.nn.functional as F
+
class DDPM(nn.Module):
def __init__(self,
- net=None, # net:nn.Module
+ net=None,
diffusion_steps = 50,
out_shape = (3,32,32),
noise_schedule = 'linear',
@@ -14,17 +15,18 @@ class DDPM(nn.Module):
var_schedule = 'same',
kl_loss = 'simplified',
recon_loss = 'none',
- device=None):
-# net: U-Net
-# diffusion_steps: length of the Markov chain
-# out_shape: shape of the models's in- and output images
-# noise_schedule: methods of initialization for the noise dist. variances, 'linear', 'cosine' or bounded_cosine
-# beta_1, beta_T: variances for the first and last noise dist. (only for the 'linear' noise schedule)
-# alpha_bar_lower_bound: upper bound for the varaince of the complete noise dist. (only for the 'cosine_bounded' noise schedule)
-# var_schedule: options to initialize or learn the denoising dist. variances, 'same', 'true'
-# kl_loss: choice between the mathematically correct 'weighted' or in practice most commonly used 'simplified' KL loss
-# recon_loss: 'none' to ignore the reconstruction loss or 'nll' to compute the negative log likelihood
-
+ device=None):
+ '''
+ net: U-Net
+ diffusion_steps: Length of the Markov chain
+ out_shape: Shape of the models's in- and output images
+ noise_schedule: Methods of initialization for the noise dist. variances, 'linear', 'cosine' or bounded_cosine
+ beta_1, beta_T: Variances for the first and last noise dist. (only for the 'linear' noise schedule)
+ alpha_bar_lower_bound: Upper bound for the varaince of the complete noise dist. (only for the 'cosine_bounded' noise schedule)
+ var_schedule: Options to initialize or learn the denoising dist. variances, 'same', 'true'
+ kl_loss: Choice between the mathematically correct 'weighted' or in practice most commonly used 'simplified' KL loss
+ recon_loss: Is 'none' to ignore the reconstruction loss or 'nll' to compute the negative log likelihood
+ '''
super(DDPM,self).__init__()
self.device = device
@@ -53,6 +55,8 @@ class DDPM(nn.Module):
self.net = net
self.diffusion_steps = diffusion_steps
+ self.noise_schedule = noise_schedule
+ self.var_schedule = var_schedule
self.beta = beta
self.alpha = alpha
self.alpha_bar = alpha_bar
@@ -67,18 +71,42 @@ class DDPM(nn.Module):
self.noise_scaler = (1-alpha)/( self.sqrt_1_minus_alpha_bar)
self.mean_scaler = 1/torch.sqrt(self.alpha)
self.mse_weight = (self.beta**2)/(2*self.var*self.alpha*(1-self.alpha_bar))
-
-
+
@staticmethod
def linear_schedule(diffusion_steps, beta_1, beta_T, device):
+ ''''
+ Function that returns the noise distribution hyperparameters for the linear schedule.
+
+ Parameters:
+ diffusion_steps (int): Length of the Markov chain.
+ beta_1 (float): Variance of the first noise distribution.
+ beta_T (float): Variance of the last noise distribution.
+
+ Returns:
+ beta (tensor): Linearly scaled from beta[0] = beta_1 to beta[-1] = beta_T, length is diffusion_steps.
+ alpha (tensor): Length is diffusion_steps.
+ alpha_bar (tensor): Length is diffusion_steps.
+ '''
beta = torch.linspace(beta_1, beta_T, diffusion_steps,device=device)
alpha = 1 - beta
alpha_bar = torch.cumprod(alpha, dim=0)
return beta, alpha, alpha_bar
- # Cosine schedule function form "Improved Denoising Diffusion Probabilistic Models" by Nichol and Dhariwal
@staticmethod
def cosine_schedule(diffusion_steps, device):
+ '''
+ Function that returns the noise distribution hyperparameters for the cosine schedule.
+ From "Improved Denoising Diffusion Probabilistic Models" by Nichol and Dhariwal.
+
+ Parameters:
+ diffusion_steps (int): Length of the Markov chain.
+
+ Returns:
+ beta (tensor): Length is diffusion_steps.
+ alpha (tensor): Length is diffusion_steps.
+ alpha_bar (tensor): Follows a sigmoid-like curve with a linear drop-off in the middle.
+ Length is diffusion_steps.
+ '''
cosine_0 = DDPM.cosine(0, diffusion_steps= diffusion_steps)
alpha_bar = [DDPM.cosine(t,diffusion_steps = diffusion_steps)/cosine_0
for t in range(1, diffusion_steps+1)]
@@ -89,15 +117,25 @@ class DDPM(nn.Module):
alpha_bar = torch.tensor(alpha_bar)
return beta, alpha, alpha_bar
- # Our experimental version of a bounded cosine schedule. Still has a linear drop-off in alpha_bar
- # while imposing a lower bound on it, such that we avoid reaching high variances that add too much noise.
- # Shape for alpha_bar is not sigmoidal anymore, betas are not smooth
- # Benefits only seem to appear for higher resolution images (see analysis notebook)"
@staticmethod
def bounded_cosine_schedule(diffusion_steps, alpha_bar_lower_bound, device):
+ '''
+ Function that returns the noise distribution hyperparameters for our experimental version of a
+ bounded cosine schedule. Benefits are still unproven. It still has a linear drop-off in alpha_bar,
+ but it's not sigmoidal and the betas are no longer smooth.
+
+ Parameters:
+ diffusion_steps (int): Length of the Markov chain
+
+ Returns:
+ beta (tensor): Length is diffusion_steps
+ alpha (tensor): Length is diffusion_steps
+ alpha_bar (tensor): Bounded between (alpha_bar_lower_bound, 1) with a linear drop-off in the middle.
+ Length is diffusion_steps
+ '''
# get cosine alpha_bar (that range from 1 to 0)
_, _, alpha_bar = DDPM.cosine_schedule(diffusion_steps, device)
- # apply min max normalization on alpha_bar (range from 0.999 to lower_bound)
+ # apply min max normalization on alpha_bar (range from lower_bound to 0.999)
min_val = torch.min(alpha_bar)
max_val = torch.max(alpha_bar)
alpha_bar = (alpha_bar - min_val) / (max_val - min_val)
@@ -112,106 +150,273 @@ class DDPM(nn.Module):
alpha_bar = torch.cumprod(alpha, dim=0)
return beta, alpha, alpha_bar
- # Offset s should be chosen such that sqrt.(beta[0])~1/127.5 (for small T=50 not possible)
@staticmethod
- def cosine(t, diffusion_steps, s = 0.008): # offset suggested by paper
+ def cosine(t, diffusion_steps, s = 0.008):
+ '''
+ Helper function that computes the cosine function from "Improved Denoising Diffusion Probabilistic Models"
+ by Nichol and Dhariwal, used for the cosine noise schedules.
+
+ Parameters:
+ t (int): Current timestep
+ diffusion_steps (int): Length of the Markov chain
+ s (float): Offset value suggested by the paper. Should be chosen such that sqrt(beta[0]) ~ 1/127.5
+ (for small T=50, this is not possible)
+
+ Returns:
+ (numpy.float64): Value of the cosine function at timestep t
+ '''
return (np.cos((((t/diffusion_steps)+s)*np.pi)/((1+s)*2)))**2
####
- # Important to Note, timesteps are adjusted to the ranges t in [1, diffusion_steps] akin to the paper
+ # Important to note: Timesteps are adjusted to the range t in [1, diffusion_steps] akin to the paper
# equations, where x_0 denotes the input image and x_t the noised latent after adding noise t times.
- # Both trajectories work on batches assuming shape=(batch_size, channels, height, width)
+ # Both trajectories work on batches assuming shape=(batch_size, channels, height, width).
####
# Forward Trajectory Functions:
- # Applies noise t times to each input image in the batch x_0
+ @torch.no_grad()
def forward_trajectory(self, x_0, t = None):
+ '''
+ Applies noise t times to each input image in the batch x_0.
+
+ Parameters:
+ x_0 (tensor): Batch of input images, with color channels assumed to be normalized between [-1,1]
+ t (tensor): Batch of timesteps, by default goes through full forward trajectory
+
+ Returns:
+ x_T (tensor): Batch of noised images at timestep t
+ forward_noise (tensor): Batch of noise parameters from the noise distribution reparametrization used to draw x_T
+ '''
if t is None:
- t = self.diffusion_steps
- elif t == 0:
- return x_0, torch.zeros(x_0.shape, device = self.device)
+ t = torch.full((x_0.shape[0],), self.diffusion_steps, device = self.device)
+ elif torch.any(t == 0):
+ raise ValueError("The tensor 't' contains a timestep zero.")
forward_noise = torch.randn(x_0.shape, device = self.device)
x_T = self.noised_latent(forward_noise, x_0, t)
return x_T , forward_noise
- # Computes the noised latents at timestep t for the given batch of isotropic noise tensors
+ @torch.no_grad()
def noised_latent(self, forward_noise, x_0, t):
+ '''
+ Given a batch of noise parameters, this function recomputes the batch of noised images at their respective timesteps t.
+ This allows us to avoid storing all the intermediate latents x_t along the forward trajectory.
+
+ Parameters:
+ forward_noise (tensor): Batch of noise parameters from the noise distribution reparametrization used to draw x_t
+ x_0 (tensor): Batch of input images, with color channels assumed to be normalized between [-1,1]
+ t (tensor): Batch of timesteps
+
+ Returns:
+ x_t (tensor): Batch of noised images at timestep t
+ '''
mean, std = self.forward_dist_param(x_0, t)
x_t = mean + std*forward_noise
return x_t
- # Returns the paramters of the combined forward dist. that add noise to the images in batch x_0 t times
+ @torch.no_grad()
def forward_dist_param(self, x_0, t):
- mean = self.sqrt_alpha_bar[t-1]*x_0
- std = self.sqrt_1_minus_alpha_bar[t-1]
- return mean, std
+ '''
+ Computes the parameters of the complete noise distribution.
- # Returns the paramters of the individual forward dist. that add noise to the images in the batch x_t at time t
- def single_forward_dist_param(self, x_t_1, t):
- mean = torch.sqrt(1-self.beta[t-1])*x_t_1
- std = torch.sqrt(self.beta[t-1])
+ Parameters:
+ x_0 (tensor): Batch of input images, with color channels assumed to be normalized between [-1,1]
+ t (tensor): Batch of timesteps
+
+ Returns:
+ mean (tensor): Batch of means for the complete noise distribution for each image in the batch x_0
+ std (tensor): Batch of std scalars for the complete noise distribution for each image in the batch x_0
+ '''
+ mean = self.sqrt_alpha_bar[t-1][:,None,None,None]*x_0
+ std = self.sqrt_1_minus_alpha_bar[t-1][:,None,None,None]
return mean, std
+ @torch.no_grad()
+ def single_forward_dist_param(self, x_t_1, t):
+ '''
+ Computes the parameters of the individual noise distribution.
+
+ Parameters:
+ x_t_1 (tensor): Batch of noised images at timestep t-1
+ t (tensor): Batch of timesteps
+
+ Returns:
+ mean (tensor): Batch of means for the individual noise distribution for each image in the batch x_t_1
+ std (tensor): Batch of std scalars for the individual noise distribution for each image in the batch x_t_1
+ '''
+ mean = torch.sqrt(1-self.beta[t-1])[:,None,None,None]*x_t_1
+ std = torch.sqrt(self.beta[t-1])[:,None,None,None]
+ return mean, std
+
+
# Reverse Trajectory Functions:
- # Sampling function that generates a batch of images
- def sample(self, batch_size = 10):
- if batch_size == 1:
- x_t_1 = torch.randn(self.out_shape, device=self.device)
- x_t_1 = torch.randn((batch_size,)+self.out_shape, device=self.device)
+ def reverse_trajectory(self, x_t, t):
+ '''
+ Draws a denoised images x_{t-1} by reparametrizing the denoising distribution at times t for the current noised
+ latents x_t.
+
+ Parameters:
+ x_t (tensor): Batch of input images, with color channels assumed to be normalized between [-1,1]
+ t (tensor): Batch of timestep
+
+ Returns:
+ x_t_1 (tensor): Batch of denoised images at timestep t-1
+ '''
+ noise = torch.randn(x_t.shape, device=self.device)
+ mean, std , _ = self.forward(x_t, t)
+ x_t_1 = mean + std*noise
+ return x_t_1
+
+ def forward(self, x_t, t):
+ '''
+ Passes the current noised images x_t and timesteps t through the U-Net in order to compute the
+ predicted noise, which is later used to determine the current denoising distribution parameters
+ (mean and std) in the reverse trajectory.
+ Since the DDPM class is inheriting from the nn.Module class, this function is required to share
+ the name 'forward'. This naming scheme does not refer to the forward trajectory, but the forward
+ pass of the model itself, which concerns to the reverse trajectory.
+
+ Parameters:
+ x_t (tensor): Batch of input images, with color channels assumed to be normalized between [-1,1]
+ t (tensor): Batch of timesteps
+
+ Returns:
+ mean (tensor): Batch of means for the complete noise dist. for each image in the batch x_t
+ std (tensor): Batch of std scalars for the complete noise dist. for each image in the batch x_t
+ pred_noise (tensor): Predicted noise for each image in the batch x_t
+ '''
+ pred_noise = self.net(x_t,t)
+ mean = self.mean_scaler[t-1][:,None,None,None]*(x_t - self.noise_scaler[t-1][:,None,None,None]*pred_noise)
+ std = self.std[t-1][:,None,None,None]
+ return mean, std, pred_noise
+
+
+ # Forward and Reverse Trajectory:
+
+ @torch.no_grad()
+ def complete_trajectory(self, x_0):
+ '''
+ Takes a batch of images and applies both trajectories sequentially, i.e. first adds noise to all
+ images along the forward chain and later removes the noise with the reverse chain.
+ This function will be used in the evaluation pipeline as a means to evaluate its performance on
+ how well it is able to reconstruct/recover the training images after applying the forward trajectory.
+
+ Parameters:
+ x_0 (tensor): Batch of input images, with color channels assumed to be normalized between [-1,1]
+
+ Returns:
+ x_0_recon (tensor): Batch of images given by the model reconstruction of x_0
+ '''
+ # apply forward trajectory
+ x_0_recon, _ = self.forward_trajectory(x_0)
+ # apply reverse trajectory
+ for t in reversed(range(1, self.diffusion_steps + 1)):
+ # draw noise used in the denoising dist. reparametrization
+ if t > 1:
+ noise = torch.randn(x_0_recon.shape, device=self.device)
+ else:
+ noise = torch.zeros(x_0_recon.shape, device=self.device)
+ # get denoising dist. param
+ mean, std, _ = self.forward(x_0_recon, torch.full((x_0_recon.shape[0],), t ,device = self.device))
+ # compute the drawn denoised latent at time t
+ x_0_recon = mean + std * noise
+ return x_0_recon
+
+
+ # Sampling Functions:
+
+ @torch.no_grad()
+ def sample(self, batch_size = 10, x_T=None):
+ '''
+ Samples batch_size images by passing a batch of randomly drawn noise parameters through the complete
+ reverse trajectory. The last denoising step is deterministic as suggested by the paper
+ "Denoising Diffusion Probabilistic Models" by Ho et al.
+
+ Parameters:
+ batch_size (int): Number of images to be sampled/generated from the diffusion model
+ x_T (tensor): Input of the reverse trajectory. Batch of noised images usually drawn
+ from an isotropic Gaussian, but can be set manually if desired.
+
+ Returns:
+ x_t_1 (tensor): Batch of sampled/generated images
+ '''
+ # start with a batch of isotropic noise images (or given arguemnt)
+ if x_T:
+ x_t_1 = x_T
+ else:
+ x_t_1 = torch.randn((batch_size,)+tuple(self.out_shape), device=self.device)
+ # apply reverse trajectory
for t in reversed(range(1, self.diffusion_steps+1)):
+ # draw noise used in the denoising dist. reparametrization
if t>1:
noise = torch.randn(x_t_1.shape, device=self.device)
else:
- noise = torch.zeros(x_t_1.shape, device=self.device) # suggested by paper "Denoising Diffusion Probabilistic Models" by Ho et al.
- mean, std, _ = self.forward(x_t_1, t)
+ noise = torch.zeros(x_t_1.shape, device=self.device)
+ # get denoising dist. param
+ mean, std, _ = self.forward(x_t_1, torch.full((x_t_1.shape[0],), t ,device = self.device))
+ # compute the drawn densoined latent at time t
x_t_1 = mean + std*noise
return x_t_1
-
- # Sampling function that generates a SINGLE image and provides all denoised latents along the chain.
- # used for analysis and visualizations
- def sample_full_chain(self):
- x = torch.empty((self.diffusion_steps+1,) + self.out_shape, device=self.device)
- x_t_1 = torch.randn(self.out_shape, device=self.device)
- # store the starting noised latent
- x[-1] = x_t_1
+
+ @torch.no_grad()
+ def sample_intermediates_latents(self):
+ '''
+ Samples a single image and provides all intermediate denoised images that were drawn along the reverse
+ trajectory. The last denoising step is deterministic as suggested by the paper "Denoising Diffusion
+ Probabilistic Models" by Ho et al.
+
+ Returns:
+ x (tensor): Contains the self.diffusion_steps+1 denoised image tensors
+ '''
+ # start with an image of pure noise (batch_size 1) and store it as part of the output
+ x_t_1 = torch.randn((1,) + tuple(self.out_shape), device=self.device)
+ x = torch.empty((self.diffusion_steps+1,) + tuple(self.out_shape), device=self.device)
+ x[-1] = x_t_1.squeeze(0)
+ # apply reverse trajectory
for t in reversed(range(1, self.diffusion_steps+1)):
+ # draw noise used in the denoising dist. reparametrization
if t>1:
noise = torch.randn(x_t_1.shape, device=self.device)
else:
- noise = torch.zeros(x_t_1.shape, device=self.device) # suggested by paper "Denoising Diffusion Probabilistic Models" by Ho et al.
- mean, std, _ = self.forward(x_t_1, t)
+ noise = torch.zeros(x_t_1.shape, device=self.device)
+ # get denoising dist. param
+ mean, std, _ = self.forward(x_t_1, torch.full((x_t_1.shape[0],), t ,device = self.device))
+ # compute the drawn densoined latent at time t
x_t_1 = mean + std*noise
- # store the drawn latent at time t
- x[t-1] = x_t_1
+ # store noised image
+ x[t-1] = x_t_1.squeeze(0)
+ #x_sq = x.squeeze(1)
+ #return x_sq
return x
+
+
+ # Loss functions
- # Reverse trajectory function.
- def reverse_trajectory(self, x_t, t):
- noise = torch.randn(x_t.shape, device=self.device)
- mean, std , _ = self.forward(x_t, t)
- x_t_1 = mean + std*noise
- return x_t_1
-
- # Computes the paramters of reverse dist. at timestep t specified by the models neural network
- # called forward since DDPM is part of the nn.Module, but it performs the reverse trajectory
- def forward(self, x_t, t):
- pred_noise = self.net(x_t, t)
- mean = self.mean_scaler[t-1]*(x_t - self.noise_scaler[t-1]*pred_noise)
- std = self.std[t-1]
- return mean, std, pred_noise
-
+ def loss_simplified(self, forward_noise, pred_noise, t=None):
+ '''
+ Returns the Mean Squared Error (MSE) between the forward_noise used to compute the noised images x_t
+ along the forward trajectory and the predicted noise computed by the U-Net with the noised images
+ x_t and timestep t.
+ '''
+ return F.mse_loss(forward_noise, pred_noise)
- # Loss functions (different ones for efficiency reasons, only check the type of model once and use appropiate loss)
- def loss_simplified(self, forward_noise, pred_noise):
- return F.mse_loss(forward_noise, pred_noise)
+ def loss_weighted(self, forward_noise, pred_noise, t):
+ '''
+ Returns the mathematically correct weighted version of the simplified loss.
+ '''
+ return self.mse_weight[t-1][:,None,None,None]*F.mse_loss(forward_noise, pred_noise)
- def loss_weighted(self, forward_noise, pred_noise, t = None):
- return self.weight[t-1]*F.mse_loss(forward_noise, pred_noise)
# If t=0 and self.recon_loss == 'nll'
- def loss_recon(self, x_0, mean_1, std_1):
- return -torch.distributions.Normal(mean_1, std_1).log_prob(x_0).mean() # possible conflicts if run on batches
\ No newline at end of file
+ def loss_recon(self, x_0, mean_1, std_1):
+ '''
+ Returns the reconstruction loss given by the mean negative log-likelihood of x_0 under the last
+ denoising Gaussian distribution with mean mean_1 and standard deviation std_1.
+ '''
+ return -torch.distributions.Normal(mean_1, std_1).log_prob(x_0).mean()
+
+
+
diff --git a/models/all_unets.py b/models/all_unets.py
new file mode 100644
index 0000000000000000000000000000000000000000..9fe4f6bf77f5c11b20af250614a67d7f1e170e46
--- /dev/null
+++ b/models/all_unets.py
@@ -0,0 +1,240 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+import einops
+import numpy as np
+
+# U-Net model
+class UNet_Res(nn.Module):
+
+ def __init__(self, attention,channels_in=3, n_channels=64,fctr = [1,2,4,4,8],time_dim=256,**args):
+ """
+ attention : (Bool) wether to use attention layers or not
+ channels_in : (Int)
+ n_channels : (Int) Channel size after first convolution
+ fctr : (list) list of factors for further channel size wrt n_channels
+ time_dim : (Int) dimenison size for time embeding vector
+ """
+ super().__init__()
+ channels_out = channels_in
+ fctr = np.asarray(fctr)*n_channels
+ # learned time embeddings
+ self.time_embedder = TimeEmbedding(time_dim = time_dim)
+ self.time_embedder0 = torch.nn.Sequential(nn.Linear(time_dim,fctr[0]),nn.SELU(),nn.Linear(fctr[0],fctr[0]))
+ self.time_embedder1 = torch.nn.Sequential(nn.Linear(time_dim,fctr[1]),nn.SELU(),nn.Linear(fctr[1],fctr[1]))
+ self.time_embedder2 = torch.nn.Sequential(nn.Linear(time_dim,fctr[2]),nn.SELU(),nn.Linear(fctr[2],fctr[2]))
+ self.time_embedder3 = torch.nn.Sequential(nn.Linear(time_dim,fctr[3]),nn.SELU(),nn.Linear(fctr[3],fctr[3]))
+ self.time_embedder4 = torch.nn.Sequential(nn.Linear(time_dim,fctr[4]),nn.SELU(),nn.Linear(fctr[4],fctr[4]))
+
+ # first conv block
+ self.first_conv = nn.Conv2d(channels_in,fctr[0],kernel_size=3, padding='same', bias=True)
+
+ #down blocks
+ self.down1 = DownsampleBlock_Res(fctr[0],fctr[1],time_dim)
+ self.down2 = DownsampleBlock_Res(fctr[1],fctr[2],time_dim)
+ self.down3 = DownsampleBlock_Res(fctr[2],fctr[3],time_dim,attention=attention)
+ self.down4 = DownsampleBlock_Res(fctr[3],fctr[4],time_dim,attention=attention)
+
+ #middle layer
+ self.mid1 = MidBlock_Res(fctr[4],time_dim,attention=attention)
+
+
+ #up blocks
+ self.up1 = UpsampleBlock_Res(fctr[1],fctr[0],time_dim)
+ self.up2 = UpsampleBlock_Res(fctr[2],fctr[1],time_dim)
+ self.up3 = UpsampleBlock_Res(fctr[3],fctr[2],time_dim,attention=attention)
+ self.up4 = UpsampleBlock_Res(fctr[4],fctr[3],time_dim)
+
+ # final 1x1 conv
+ self.end_conv = nn.Conv2d(fctr[0], channels_out, kernel_size=1,bias=True)
+
+ # Attention Layers
+ self.mha21 = MHABlock(fctr[2])
+ self.mha22 = MHABlock(fctr[2])
+ self.mha31 = MHABlock(fctr[3])
+ self.mha32 = MHABlock(fctr[3])
+ self.mha41 = MHABlock(fctr[4])
+ self.mha42 = MHABlock(fctr[4])
+
+ def forward(self, input, t):
+ t_emb = self.time_embedder(t).to(input.device)
+
+ t_emb0 = self.time_embedder0(t_emb)
+ t_emb1 = self.time_embedder1(t_emb)
+ t_emb2 = self.time_embedder2(t_emb)
+ t_emb3 = self.time_embedder3(t_emb)
+ t_emb4 = self.time_embedder4(t_emb)
+
+ # first two conv layers
+ x = self.first_conv(input) + t_emb0[:,:,None,None]
+ #timemb
+ skip1 =x
+ skip1,x = self.down1(x,t_emb1)
+ skip2,x = self.down2(x,t_emb2)
+ skip3,x = self.down3(x,t_emb3)
+ skip4,x = self.down4(x,t_emb4)
+
+ x = self.mid1(x,t_emb4)
+
+ x = self.up4(x,skip4,t_emb3)
+ x = self.up3(x,skip3,t_emb2)
+ x = self.up2(x,skip2,t_emb1)
+ x = self.up1(x,skip1,t_emb0)
+ x = self.end_conv(x)
+
+ return x
+
+
+
+#TimeEmbedding
+class TimeEmbedding(nn.Module):
+
+ def __init__(self, time_dim=64):
+ super().__init__()
+
+ self.time_dim = time_dim
+ n = 10000
+ self.factor = torch.pow(n*torch.ones(size=(time_dim//2,)),(-2/time_dim*torch.arange(time_dim//2)))
+
+ def forward(self, t):
+ """
+ input is t (B,)
+ factor dim (time_dim,)
+ output is (B,time_dim)
+ """
+ self.factor = self.factor.to(t.device)
+ theta = torch.outer(t,self.factor)
+
+
+ # shape of embedding [time_channels, dim]
+ emb = torch.zeros(t.size(0), self.time_dim,device=t.device)
+ emb[:, 0::2] = torch.sin(theta)
+ emb[:, 1::2] = torch.cos(theta)
+
+ return emb
+
+# Self Attention
+class MHABlock(nn.Module):
+
+ def __init__(self,
+ channels_in,
+ num_attention_heads=1 # number of attention heads in MHA
+ ):
+ super().__init__()
+
+ self.channels_in = channels_in
+ self.num_attention_heads = num_attention_heads
+ self.self_attention = nn.MultiheadAttention(channels_in, num_heads=self.num_attention_heads)
+
+ def forward(self, x):
+ skip = x
+ batch_size,_,height,width = x.size()
+
+ x = x.permute(2, 3, 0, 1).reshape(height * width, batch_size, -1)
+ attn_output, _ = self.self_attention(x, x, x)
+ attn_output = attn_output.reshape(batch_size, -1, height, width)
+
+ return attn_output+skip
+
+# Residual Convolution Block
+class ConvBlock_Res(nn.Module):
+
+ def __init__(self,
+ channels_in, # number of input channels fed into the block
+ channels_out, # number of output channels produced by the block
+ time_dim,
+ attention,
+ num_groups=32, # number of groups used in Group Normalization; channels_in must be divisible by num_groups
+ ):
+ super().__init__()
+
+ self.attention = attention
+ if self.attention:
+ self.attlayer = MHABlock(channels_in=channels_out)
+
+ # Convolution layer 1
+ self.conv1 = nn.Conv2d(channels_in, channels_out, kernel_size=3, padding='same', bias=True)
+ self.gn1 = nn.GroupNorm(num_groups, channels_out)
+ self.act1 = nn.SiLU()
+
+ # Convolution layer 2
+ self.conv2 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding='same', bias=True)
+ self.gn2 = nn.GroupNorm(num_groups, channels_out)
+ self.act2 = nn.SiLU()
+
+ # Convolution layer 3
+ self.conv3 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding='same', bias=True)
+ self.gn3 = nn.GroupNorm(num_groups, channels_out)
+ self.act3 = nn.SiLU()
+
+ #Convolution skip
+ self.res_skip = nn.Conv2d(channels_in,channels_out,kernel_size=1)
+
+ nn.init.xavier_normal_(self.conv1.weight)
+ nn.init.xavier_normal_(self.conv2.weight)
+ nn.init.xavier_normal_(self.conv3.weight)
+
+ def forward(self, x, t):
+ res = self.res_skip(x)
+ # second convolution layer
+ x = self.act1(self.gn1(self.conv1(x)))
+
+
+ h =x + t[:,:,None,None]
+
+
+ # third convolution layer
+ h = self.act2(self.gn2(self.conv2(h)))
+
+ h = self.act3(self.gn3(self.conv3(h)))
+
+ if self.attention:
+ h = self.attlayer(h)
+
+ return h +res
+
+# Down Sample
+class DownsampleBlock_Res(nn.Module):
+
+ def __init__(self, channels_in, channels_out,time_dim,attention=False):
+ super().__init__()
+
+
+ self.pool = nn.MaxPool2d((2,2), stride=2)
+ self.convblock = ConvBlock_Res(channels_in, channels_out,time_dim,attention=attention)
+
+ def forward(self, x, t):
+
+ x = self.convblock(x, t)
+ h = self.pool(x)
+ return x,h
+
+# Upsample Block
+class UpsampleBlock_Res(nn.Module):
+
+ def __init__(self, channels_in, channels_out,time_dim,attention=False):
+ super().__init__()
+
+ self.upconv = nn.ConvTranspose2d(channels_in, channels_in, kernel_size=2, stride=2)
+ self.convblock = ConvBlock_Res(channels_in, channels_out,time_dim,attention=attention)
+
+ def forward(self, x, skip_x, t):
+ x = self.upconv(x)
+
+ # skip-connection - merge features from contracting path to its symmetric counterpart in expansive path
+ out = x + skip_x
+
+ out = self.convblock(out, t)
+ return out
+
+# Middle Block
+class MidBlock_Res(nn.Module):
+ def __init__(self,channels,time_dim,attention=False):
+ super().__init__()
+ self.convblock1 = ConvBlock_Res(channels,channels,time_dim,attention=attention)
+ self.convblock2 = ConvBlock_Res(channels,channels,time_dim,attention=False)
+ def forward(self,x,t):
+ x = self.convblock1(x,t)
+ return self.convblock2(x,t)
+
diff --git a/models/unet_unconditional_diffusion.py b/models/unet_unconditional_diffusion.py
index ae0dcffce535ce3d3e9d36c9bcd8f09a4909acab..3f56aaaf1956a6073bf7fe65296c99ab80f5da16 100644
--- a/models/unet_unconditional_diffusion.py
+++ b/models/unet_unconditional_diffusion.py
@@ -403,13 +403,14 @@ class UNet_Unconditional_Diffusion(nn.Module):
def forward(self, x, t):
-
+ t = torch.tensor(t)
+
# to store feature maps from contracting path
skip = []
# time embedding for time step t (int)
- t = self.time_embedding(t)
-
+ t = self.time_embedding(t).to('cuda')
+
# first conv layer to project input image (3, *, *) into (projection_features=64, *, *)
x = self.first_act(self.first_conv(x))
@@ -442,4 +443,4 @@ class UNet_Unconditional_Diffusion(nn.Module):
# final conv layer
x = self.end_gn(x)
x = self.end_act(self.end_conv(x))
- return x
\ No newline at end of file
+ return x
diff --git a/requirements.txt b/requirements.txt
index 7ae30625a7b505636ee6f2ff3c91d4b04d1943de..18a2715a942079fda50b4e5a746d299ce4c0a4d9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,11 @@
-fire
-intel-numpy==1.21.4
+pandas
+numpy
matplotlib
+Pillow
+h5py
scipy
---find links https://download.pytorch.org/whl/cu116
-torch==1.13.1
-torchvision==0.14.1
-torchaudio==0.13.1
+wandb
+torch
+torchvision
+torchaudio
+einops
\ No newline at end of file
diff --git a/trainer/train.py b/trainer/train.py
index 62dcc7629f71326a7acbdef98c8d22daae804a61..80431b0c02342e1dfba41a4b47cb9afc13fa87ce 100644
--- a/trainer/train.py
+++ b/trainer/train.py
@@ -1,6 +1,6 @@
import numpy as np
-
+import copy
import torch
from torch import nn
from torchvision import datasets,transforms
@@ -10,10 +10,10 @@ import numpy as np
import torch.nn.functional as F
import os
import wandb
+from copy import deepcopy
device = 'cuda' if torch.cuda.is_available() else 'cpu'
-
# Simple Training function for the unconditional diffusion model
def simple_trainer(model,device,epochs,trainloader,testloader,bs,lr,T,criterion = nn.MSELoss()):
criterion.to(device)
@@ -54,6 +54,37 @@ def simple_trainer(model,device,epochs,trainloader,testloader,bs,lr,T,criterion
print(f"Testloss in step {epoch} :{np.mean(running_testloss)}")
+# EMA class
+# Important! This EMA class code is not ours and was taken from the Pytorch Image Models library called timm and performs exponential moving
+# average on the trained weights for a given models neural net which was suggested by the paper "Improved Denoising Diffusion Probabilistic Models"
+# by Nichol and Dhariwal to stabilize and improve the training and generalization process.
+# https://github.com/huggingface/pytorch-image-models/blob/main/timm/utils/model_ema.py
+class ModelEmaV2(nn.Module):
+ def __init__(self, model, decay=0.9999, device=None):
+ super(ModelEmaV2, self).__init__()
+ # make a copy of the model for accumulating moving average of weights
+ self.module = deepcopy(model)
+ self.module.eval()
+ self.decay = decay
+ self.device = device # perform ema on different device from model if set
+ if self.device is not None:
+ self.module.to(device=device)
+
+ def _update(self, model, update_fn):
+ with torch.no_grad():
+ for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()):
+ if self.device is not None:
+ model_v = model_v.to(device=self.device)
+ ema_v.copy_(update_fn(ema_v, model_v))
+
+ def update(self, model):
+ self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m)
+
+ def set(self, model):
+ self._update(model, update_fn=lambda e, m: m)
+
+
+
# Training function for the unconditional diffusion model
def ddpm_trainer(model,
@@ -62,136 +93,200 @@ def ddpm_trainer(model,
store_iter = 10,
eval_iter = 10,
epochs = 50,
- last_epoch=-1,
optimizer_class=torch.optim.AdamW,
optimizer_params=None,
learning_rate = 0.001,
- lr_schedule = False,
verbose = False,
- name_appendix=None,
- checkpoint_path= None,
+ run_name=None,
+ checkpoint= None,
+ experiment_path = None,
+ T_max = 5*10000, # None,
+ eta_min= 1e-5,
+ ema_training = True,
+ decay = 0.9999,
**args
):
'''
model: Properly initialized DDPM model.
store_iter: Stores the trained DDPM every store_iter epochs.
+ experiment_path: Path to the models experiment folder, where the trained model will be stored every store_iter epochs
eval_iter: Evaluates the trained DDPM on testing data every eval_iter epochs.
- epochs: Number of epochs we train the model further.
- last_epoch: Last epoch the model was trained on (default is -1 if no checkpoint exists).
+ epochs: Number of epochs we train the model further.
optimizer_class: PyTorch optimizer.
optimizer_param: Parameters for the PyTorch optimizer.
- learning_rate: For optimizer initialization or if a checkpoint exists for our manual learning rate.
- lr_schedule: If True, manually sets the learning rate of the optimizer to the given one.
+ learning_rate: For optimizer initialization when training from zero, i.e. no checkpoint
verbose: If True, prints the running losses for every epoch.
- name_appendix: Adds an appendix to the run name for WandB. REALLY IMPORTANT IF YOU WANT TO TRAIN
- FROM CHECKPOINT AND HAVE THE LOGS FOLLOW THE SAME RUN AS BEFORE!
+ run_name: Run name for WandB. IF YOU TRAIN FROM CHECKPOINT MAKE SURE TO USE THE SAME
+ 'run_name' FOR THE DATA TO BE LOGGED ON THE SAME WANDB RUN!
+ trainloader: Loads the train dataset
+ testloader: Loads the test dataset
+ checkpoint: Name of the saved pth. file containing the trained weights and biases
+ T_max: CosineAnnealingLR scheduler argument (nr of steps in training for a full cycle)
+ eta_min: CosineAnnealingLR scheduler argument (scheduler oscillates between highest lr 'leraning_rate' and minimum lr 'eta_min')
+ decay: EMA decay rate that is used to weight the effect of the ema model when computing the weighted avg between trained and
+ ema weights for the networks weight update
'''
- # set optimizer
+ # set optimizer parameters and learning rate
if optimizer_params is None:
optimizer_params = dict(lr=learning_rate)
- else:
- optimizer_params['lr'] = learning_rate
- optimizer = optimizer_class(model.parameters(), **optimizer_params)
-
- # If checkpoint exist, load checkpoint
- if checkpoint_path:
- model_files = os.listdir(checkpoint_path)
- # get the first file that ends with pth in the checkpoint path
- for el in model_files:
- if el.endswith('.pth'):
- checkpoint = torch.load(os.path.join(checkpoint_path, el))
- # update last_epoch
- last_epoch = checkpoint['epoch']
- # load weights and biases of the U-Net
- model_state_dict = checkpoint['model']
- model.load_state_dict(model_state_dict)
- model = model.to(device)
- # load optimizer state
- optimizer_state_dict = checkpoint['optimizer']
- optimizer.load_state_dict(optimizer_state_dict)
- # If we want to decrease the learning rate manually
- if lr_schedule:
- for param_group in optimizer.param_groups:
- param_group['lr'] = learning_rate
-
- # Define run name [from DL Notebook from Lab]
- model_name = model.__class__.__name__
- optimizer_name = optimizer.__class__.__name__
- run_name = f'{model_name}-{optimizer_name}-lr{learning_rate}'
- # name_appendix to resume the run on WandB
- if name_appendix:
- run_name += '_' + name_appendix
-
+ optimizer = optimizer_class(model.net.parameters(), **optimizer_params)
+ # set lr cosine schedule (comonly used in diffusion models)
+ scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=T_max, eta_min=eta_min)
+
+ # if checkpoint path is given, load the model from checkpoint
+ last_epoch = -1
+ if checkpoint:
+ try:
+ checkpoint_path = f'{experiment_path}trained_ddpm/{checkpoint}'
+ # Load the checkpoint
+ checkpoint = torch.load(checkpoint_path)
+ # update last_epoch
+ last_epoch = checkpoint['epoch']
+ # load weights and biases of the U-Net
+ model_state_dict = checkpoint['model']
+ model.net.load_state_dict(model_state_dict)
+ model = model.to(device)
+ # load optimizer state
+ optimizer_state_dict = checkpoint['optimizer']
+ optimizer.load_state_dict(optimizer_state_dict)
+ # load learning rate schedule state
+ scheduler_state_dict = checkpoint['scheduler']
+ scheduler.load_state_dict(scheduler_state_dict)
+ scheduler.last_epoch = last_epoch
+ except Exception as e:
+ print("Error loading checkpoint. Exception: ", e)
+
+ # pick kl loss function
+ if model.kl_loss == 'weighted':
+ loss_func = model.loss_weighted
+ else:
+ loss_func = model.loss_simplified
+
+ # pick lowest timestep
+ low = 1
+ if model.recon_loss == 'nll':
+ low = 0
+
+ # EMA
+ if ema_training:
+ ema = ModelEmaV2(model, decay=decay, device = model.device)
+
# Using W&B
- with wandb.init(project='test-project', name=name_appendix, entity='gonzalomartingarcia0', id=name_appendix, resume=True) as run:
+ with wandb.init(project='test-project', name=run_name, entity='gonzalomartingarcia0', id=run_name, resume=True) as run:
# Log some info
run.config.learning_rate = learning_rate
run.config.optimizer = optimizer.__class__.__name__
- run.watch(model)
-
- if lr_schedule:
- # log the manual change of learning rate
- wandb.log({"learning_rate": learning_rate})
+ #run.watch(model.net)
# training loop
- # last model was stored at epoch last_epoch, we continue training from there, i.e. last_epoch+1
+ # last model was stored at epoch last_epoch, we continue training from there, i.e. last_epoch+1 (else we start at epoch 0)
for epoch in range(last_epoch+1, (last_epoch+1)+epochs):
running_trainloss = 0
nr_train_batches = 0
-
+
# train
- model.train()
+ model.net.train()
for idx,(x_0, _) in enumerate(trainloader):
x_0 = x_0.to(device)
- t = torch.randint(low=1, high=model.diffusion_steps, size=(1,)).item()
- x_t, forward_noise = model.forward_trajectory(x_0,t)
+ t = torch.randint(low=low, high=model.diffusion_steps, size=(x_0.shape[0],), device = device)
+ optimizer.zero_grad()
+
+ # Define masks for zero and non-zero elements of t
+ mask_zero_t = (t == 0)
+ mask_non_zero_t = (t != 0)
- optimizer.zero_grad()
- _,_,pred_noise = model.forward(x_t,t)
- loss = model.loss_simplified(forward_noise,pred_noise)
- loss.backward()
+ t[mask_zero_t] = 1
+ x_t, forward_noise = model.forward_trajectory(x_0,t)
+ mean, std, pred_noise = model.forward(x_t,t)
+
+ loss = 0
+ # Compute kl loss
+ if torch.any(mask_non_zero_t):
+ loss = loss_func(forward_noise[mask_non_zero_t], pred_noise[mask_non_zero_t], t[mask_non_zero_t])
+ running_trainloss += loss.item()
+ nr_train_batches += 1
+ run.log({'loss': loss.item(), "learning_rate": scheduler.get_last_lr()[0], 'epoch': epoch, 'batch': idx})
+
+ # If reconstrcution loss was drawn
+ if torch.any(mask_zero_t):
+ recon_loss = model.loss_recon(x_0[mask_zero_t], mean[mask_zero_t], std[mask_zero_t])
+ loss += recon_loss
+ run.log({'recon_loss': recon_loss.item(), 'epoch': epoch, 'batch': idx})
+
+ loss.backward()
optimizer.step()
-
- running_trainloss += loss.item()
- nr_train_batches += 1
- run.log({'loss': loss.item(), 'epoch': epoch, 'batch': idx})
+ if ema_training:
+ ema.update(model)
+ scheduler.step()
+
if verbose:
print(f"Loss in epoch {epoch}:{running_trainloss/nr_train_batches}")
run.log({'running_loss': running_trainloss/nr_train_batches})
-
+
# evaluation
if ((epoch+1) % eval_iter == 0) or ((epoch+1) % store_iter == 0):
running_testloss = 0
nr_test_batches = 0
- model.eval()
+ model.net.eval()
with torch.no_grad():
for idx,(x_0,_) in enumerate(testloader):
x_0 = x_0.to(device)
- t = torch.randint(low=1,high=model.diffusion_steps, size=(1,)).item()
- x_t, forward_noise = model.forward_trajectory(x_0,t)
- _,_,pred_noise = model.forward(x_t,t)
- loss = model.loss_simplified(forward_noise,pred_noise)
+ t = torch.randint(low=low, high=model.diffusion_steps, size=(x_0.shape[0],), device = device)
- running_testloss += loss.item()
- nr_test_batches += 1
- run.log({'test_loss': loss.item(), 'epoch': epoch, 'batch': idx})
+ # Define masks for zero and non-zero elements of t
+ mask_zero_t = (t == 0)
+ mask_non_zero_t = (t != 0)
+
+ t[mask_zero_t] = 1
+ x_t, forward_noise = model.forward_trajectory(x_0,t)
+ mean, std, pred_noise = model.forward(x_t,t)
+
+ loss = 0
+ # Compute kl loss
+ if torch.any(mask_non_zero_t):
+ loss = loss_func(forward_noise[mask_non_zero_t], pred_noise[mask_non_zero_t], t[mask_non_zero_t])
+ running_testloss += loss.item()
+ nr_test_batches += 1
+ run.log({'test_loss': loss.item(), "learning_rate": scheduler.get_last_lr()[0], 'epoch': epoch, 'batch': idx})
+
+ # If reconstrcution loss was drawn
+ if torch.any(mask_zero_t):
+ recon_loss = model.loss_recon(x_0[mask_zero_t], mean[mask_zero_t], std[mask_zero_t])
+ loss += recon_loss
+ run.log({'recon_test_loss': recon_loss.item(), 'epoch': epoch, 'batch': idx})
+
if verbose:
print(f"Test loss in epoch {epoch}:{running_testloss/nr_test_batches}")
run.log({'running_test_loss': running_testloss/nr_test_batches})
# store model
if ((epoch+1) % store_iter == 0):
- if not os.path.exists('trained_ddpm'):
- os.makedirs('trained_ddpm')
+ save_dir = os.path.join(experiment_path, 'trained_ddpm/')
+ os.makedirs(save_dir, exist_ok=True)
torch.save({
'epoch': epoch,
- 'model': model.state_dict(),
+ 'model': model.net.state_dict(),
'optimizer': optimizer.state_dict(),
+ 'scheduler': scheduler.state_dict(),
'running_loss': running_trainloss/nr_train_batches,
'running_test_loss': running_testloss/nr_test_batches,
- }, os.path.join('trained_ddpm', f"model_epoch_{epoch}.pth"))
+ }, os.path.join(save_dir, f"model_epoch_{epoch}.pth"))
+
+ # always store the last version of the model if we trained through all epochs
+ final = (last_epoch+1)+epochs
+ save_dir = os.path.join(experiment_path, 'trained_ddpm/')
+ os.makedirs(save_dir, exist_ok=True)
+ torch.save({
+ 'epoch': final,
+ 'model': model.net.state_dict(),
+ 'optimizer': optimizer.state_dict(),
+ 'scheduler': scheduler.state_dict(),
+ 'running_loss': running_trainloss/nr_train_batches,
+ 'running_test_loss': running_testloss/nr_test_batches,
+ }, os.path.join(save_dir, f"model_epoch_{final}.pth"))
+
diff --git a/wandb/debug-cli.gonzalo.log b/wandb/debug-cli.gonzalo.log
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log
deleted file mode 120000
index 56694c10d1b2eb3f612ae155b0bebd9926cbfc1f..0000000000000000000000000000000000000000
--- a/wandb/debug-internal.log
+++ /dev/null
@@ -1 +0,0 @@
-run-20230602_141242-akua37dh/logs/debug-internal.log
\ No newline at end of file
diff --git a/wandb/debug.log b/wandb/debug.log
deleted file mode 120000
index d802c73383b936fd6e0179eccba188f2cde18b89..0000000000000000000000000000000000000000
--- a/wandb/debug.log
+++ /dev/null
@@ -1 +0,0 @@
-run-20230602_141242-akua37dh/logs/debug.log
\ No newline at end of file
diff --git a/wandb/latest-run b/wandb/latest-run
deleted file mode 120000
index 523fa9ee5c8b5c39a85e73ee0f2c58b969ccf6a3..0000000000000000000000000000000000000000
--- a/wandb/latest-run
+++ /dev/null
@@ -1 +0,0 @@
-run-20230602_141242-akua37dh
\ No newline at end of file
diff --git a/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml b/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml
deleted file mode 100644
index cf9866c399f506b3dd570d38781f16b7946e811f..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml
+++ /dev/null
@@ -1,256 +0,0 @@
-name: torch
-channels:
- - pytorch-nightly
- - conda-forge
- - defaults
-dependencies:
- - anyio=3.6.2=pyhd8ed1ab_0
- - aom=3.5.0=h7ea286d_0
- - appnope=0.1.3=pyhd8ed1ab_0
- - argon2-cffi=21.3.0=pyhd8ed1ab_0
- - argon2-cffi-bindings=21.2.0=py39h02fc5c5_3
- - asttokens=2.2.1=pyhd8ed1ab_0
- - attrs=22.2.0=pyh71513ae_0
- - backcall=0.2.0=pyh9f0ad1d_0
- - backports=1.0=pyhd8ed1ab_3
- - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
- - beautifulsoup4=4.12.2=pyha770c72_0
- - bleach=6.0.0=pyhd8ed1ab_0
- - boto3=1.26.116=pyhd8ed1ab_0
- - botocore=1.29.116=pyhd8ed1ab_0
- - brotli=1.0.9=h1a8c8d9_8
- - brotli-bin=1.0.9=h1a8c8d9_8
- - brotlipy=0.7.0=py39h02fc5c5_1005
- - bzip2=1.0.8=h3422bc3_4
- - c-ares=1.18.1=h3422bc3_0
- - ca-certificates=2023.01.10=hca03da5_0
- - cached-property=1.5.2=hd8ed1ab_1
- - cached_property=1.5.2=pyha770c72_1
- - certifi=2023.5.7=py39hca03da5_0
- - cffi=1.15.1=py39h7e6b969_3
- - charset-normalizer=3.1.0=pyhd8ed1ab_0
- - click=8.1.3=unix_pyhd8ed1ab_2
- - colorama=0.4.6=pyhd8ed1ab_0
- - comm=0.1.3=pyhd8ed1ab_0
- - contourpy=1.0.7=py39haaf3ac1_0
- - cryptography=40.0.2=py39he2a39a8_0
- - cycler=0.11.0=pyhd8ed1ab_0
- - debugpy=1.6.7=py39h23fbdae_0
- - decorator=5.1.1=pyhd8ed1ab_0
- - defusedxml=0.7.1=pyhd8ed1ab_0
- - entrypoints=0.4=pyhd8ed1ab_0
- - executing=1.2.0=pyhd8ed1ab_0
- - expat=2.5.0=hb7217d7_1
- - ffmpeg=5.1.2=gpl_hf318d42_106
- - filelock=3.12.0=pyhd8ed1ab_0
- - flask=2.2.3=pyhd8ed1ab_0
- - flit-core=3.8.0=pyhd8ed1ab_0
- - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
- - font-ttf-inconsolata=3.000=h77eed37_0
- - font-ttf-source-code-pro=2.038=h77eed37_0
- - font-ttf-ubuntu=0.83=hab24e00_0
- - fontconfig=2.14.2=h82840c6_0
- - fonts-conda-ecosystem=1=0
- - fonts-conda-forge=1=0
- - fonttools=4.39.3=py39h02fc5c5_0
- - freetype=2.12.1=hd633e50_1
- - gettext=0.21.1=h0186832_0
- - gmp=6.2.1=h9f76cd9_0
- - gmpy2=2.1.2=py39h0b4f9c6_1
- - gnutls=3.7.8=h9f1a10d_0
- - h5py=3.8.0=nompi_py39hbcdb4fd_101
- - hdf5=1.14.0=nompi_h6b85c65_103
- - icu=72.1=he12128b_0
- - idna=3.4=pyhd8ed1ab_0
- - importlib-metadata=6.5.0=pyha770c72_0
- - importlib-resources=5.12.0=pyhd8ed1ab_0
- - importlib_metadata=6.5.0=hd8ed1ab_0
- - importlib_resources=5.12.0=pyhd8ed1ab_0
- - ipykernel=6.22.0=pyh736e0ef_0
- - ipython=8.12.0=pyhd1c38e8_0
- - ipython_genutils=0.2.0=py_1
- - ipywidgets=8.0.6=pyhd8ed1ab_0
- - itsdangerous=2.1.2=pyhd8ed1ab_0
- - jedi=0.18.2=pyhd8ed1ab_0
- - jinja2=3.1.2=pyhd8ed1ab_1
- - jmespath=1.0.1=pyhd8ed1ab_0
- - joblib=1.2.0=pyhd8ed1ab_0
- - jpeg=9e=h1a8c8d9_3
- - jsonschema=4.17.3=pyhd8ed1ab_0
- - jupyter=1.0.0=py39h2804cbe_8
- - jupyter_client=8.2.0=pyhd8ed1ab_0
- - jupyter_console=6.6.3=pyhd8ed1ab_0
- - jupyter_core=5.3.0=py39h2804cbe_0
- - jupyter_events=0.6.3=pyhd8ed1ab_0
- - jupyter_server=2.5.0=pyhd8ed1ab_0
- - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1
- - jupyterlab_pygments=0.2.2=pyhd8ed1ab_0
- - jupyterlab_widgets=3.0.7=pyhd8ed1ab_0
- - kiwisolver=1.4.4=py39haaf3ac1_1
- - krb5=1.20.1=h69eda48_0
- - lame=3.100=h1a8c8d9_1003
- - lcms2=2.15=h481adae_0
- - lerc=4.0.0=h9a09cb3_0
- - libaec=1.0.6=hb7217d7_1
- - libblas=3.9.0=16_osxarm64_openblas
- - libbrotlicommon=1.0.9=h1a8c8d9_8
- - libbrotlidec=1.0.9=h1a8c8d9_8
- - libbrotlienc=1.0.9=h1a8c8d9_8
- - libcblas=3.9.0=16_osxarm64_openblas
- - libcurl=8.0.1=heffe338_0
- - libcxx=16.0.1=h75e25f2_0
- - libdeflate=1.17=h1a8c8d9_0
- - libedit=3.1.20191231=hc8eb9b7_2
- - libev=4.33=h642e427_1
- - libexpat=2.5.0=hb7217d7_1
- - libffi=3.4.2=h3422bc3_5
- - libgfortran=5.0.0=12_2_0_hd922786_31
- - libgfortran5=12.2.0=h0eea778_31
- - libiconv=1.17=he4db4b2_0
- - libidn2=2.3.4=h1a8c8d9_0
- - liblapack=3.9.0=16_osxarm64_openblas
- - libnghttp2=1.52.0=hae82a92_0
- - libopenblas=0.3.21=openmp_hc731615_3
- - libopus=1.3.1=h27ca646_1
- - libpng=1.6.39=h76d750c_0
- - libsodium=1.0.18=h27ca646_1
- - libsqlite=3.40.0=h76d750c_0
- - libssh2=1.10.0=h7a5bd25_3
- - libtasn1=4.19.0=h1a8c8d9_0
- - libtiff=4.5.0=h5dffbdd_2
- - libunistring=0.9.10=h3422bc3_0
- - libvpx=1.11.0=hc470f4d_3
- - libwebp-base=1.3.0=h1a8c8d9_0
- - libxcb=1.13=h9b22ae9_1004
- - libxml2=2.10.4=h2aff0a6_0
- - libxslt=1.1.37=h1bd8bc4_0
- - libzlib=1.2.13=h03a7124_4
- - llvm-openmp=16.0.1=h7cfbb63_0
- - lxml=4.9.2=py39h0520ce3_0
- - markupsafe=2.1.2=py39h02fc5c5_0
- - matplotlib=3.7.1=py39hdf13c20_0
- - matplotlib-base=3.7.1=py39h35e9e80_0
- - matplotlib-inline=0.1.6=pyhd8ed1ab_0
- - mistune=2.0.5=pyhd8ed1ab_0
- - mpc=1.3.1=h91ba8db_0
- - mpfr=4.2.0=he09a6ba_0
- - mpmath=1.3.0=pyhd8ed1ab_0
- - munkres=1.1.4=pyh9f0ad1d_0
- - nbclassic=0.5.5=pyhb4ecaf3_1
- - nbclient=0.7.3=pyhd8ed1ab_0
- - nbconvert=7.3.1=pyhd8ed1ab_0
- - nbconvert-core=7.3.1=pyhd8ed1ab_0
- - nbconvert-pandoc=7.3.1=pyhd8ed1ab_0
- - nbformat=5.8.0=pyhd8ed1ab_0
- - ncurses=6.3=h07bb92c_1
- - nest-asyncio=1.5.6=pyhd8ed1ab_0
- - nettle=3.8.1=h63371fa_1
- - networkx=3.1=pyhd8ed1ab_0
- - notebook=6.5.4=pyha770c72_0
- - notebook-shim=0.2.2=pyhd8ed1ab_0
- - numpy=1.24.2=py39hff61c6a_0
- - openh264=2.3.1=hb7217d7_2
- - openjpeg=2.5.0=hbc2ba62_2
- - openssl=3.1.0=h03a7124_0
- - p11-kit=0.24.1=h29577a5_0
- - packaging=23.1=pyhd8ed1ab_0
- - pandas=2.0.0=py39hde7b980_0
- - pandas-datareader=0.10.0=pyh6c4a22f_0
- - pandoc=2.19.2=hce30654_2
- - pandocfilters=1.5.0=pyhd8ed1ab_0
- - parso=0.8.3=pyhd8ed1ab_0
- - pexpect=4.8.0=pyh1a96a4e_2
- - pickleshare=0.7.5=py_1003
- - pillow=9.4.0=py39h8bd98a6_1
- - pip=23.1=pyhd8ed1ab_0
- - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_0
- - platformdirs=3.2.0=pyhd8ed1ab_0
- - pooch=1.7.0=pyha770c72_3
- - prometheus_client=0.16.0=pyhd8ed1ab_0
- - prompt-toolkit=3.0.38=pyha770c72_0
- - prompt_toolkit=3.0.38=hd8ed1ab_0
- - psutil=5.9.5=py39h02fc5c5_0
- - pthread-stubs=0.4=h27ca646_1001
- - ptyprocess=0.7.0=pyhd3deb0d_0
- - pure_eval=0.2.2=pyhd8ed1ab_0
- - pycparser=2.21=pyhd8ed1ab_0
- - pygments=2.15.1=pyhd8ed1ab_0
- - pyopenssl=23.1.1=pyhd8ed1ab_0
- - pyparsing=3.0.9=pyhd8ed1ab_0
- - pyrsistent=0.19.3=py39h02fc5c5_0
- - pysocks=1.7.1=pyha2e5f31_6
- - python=3.9.16=hea58f1e_0_cpython
- - python-dateutil=2.8.2=pyhd8ed1ab_0
- - python-fastjsonschema=2.16.3=pyhd8ed1ab_0
- - python-json-logger=2.0.7=pyhd8ed1ab_0
- - python-tzdata=2023.3=pyhd8ed1ab_0
- - python_abi=3.9=3_cp39
- - pytorch=2.1.0.dev20230419=py3.9_0
- - pytz=2023.3=pyhd8ed1ab_0
- - pyyaml=6.0=py39h02fc5c5_5
- - pyzmq=25.0.2=py39h0553236_0
- - readline=8.2=h92ec313_1
- - requests=2.28.2=pyhd8ed1ab_1
- - rfc3339-validator=0.1.4=pyhd8ed1ab_0
- - rfc3986-validator=0.1.1=pyh9f0ad1d_0
- - s3transfer=0.6.0=pyhd8ed1ab_0
- - scikit-learn=1.2.2=py39h6180588_1
- - scipy=1.10.1=py39hba9bd2d_0
- - seaborn=0.12.2=py39hca03da5_0
- - send2trash=1.8.0=pyhd8ed1ab_0
- - setuptools=67.6.1=pyhd8ed1ab_0
- - six=1.16.0=pyh6c4a22f_0
- - sniffio=1.3.0=pyhd8ed1ab_0
- - soupsieve=2.3.2.post1=pyhd8ed1ab_0
- - stack_data=0.6.2=pyhd8ed1ab_0
- - svt-av1=1.4.1=h7ea286d_0
- - sympy=1.11.1=pypyh9d50eac_103
- - terminado=0.17.1=pyhd1c38e8_0
- - threadpoolctl=3.1.0=pyh8a188c0_0
- - tinycss2=1.2.1=pyhd8ed1ab_0
- - tk=8.6.12=he1e0b03_0
- - torchvision=0.16.0.dev20230419=py39_cpu
- - tornado=6.3=py39h02fc5c5_0
- - tqdm=4.65.0=pyhd8ed1ab_1
- - traitlets=5.9.0=pyhd8ed1ab_0
- - typing-extensions=4.5.0=hd8ed1ab_0
- - typing_extensions=4.5.0=pyha770c72_0
- - tzdata=2023c=h71feb2d_0
- - unicodedata2=15.0.0=py39h02fc5c5_0
- - urllib3=1.26.15=pyhd8ed1ab_0
- - wcwidth=0.2.6=pyhd8ed1ab_0
- - webencodings=0.5.1=py_1
- - websocket-client=1.5.1=pyhd8ed1ab_0
- - werkzeug=2.2.3=pyhd8ed1ab_0
- - wheel=0.40.0=pyhd8ed1ab_0
- - widgetsnbextension=4.0.7=pyhd8ed1ab_0
- - x264=1!164.3095=h57fd34a_2
- - x265=3.5=hbc6ce65_3
- - xorg-libxau=1.0.9=h27ca646_0
- - xorg-libxdmcp=1.1.3=h27ca646_0
- - xz=5.2.6=h57fd34a_0
- - yaml=0.2.5=h3422bc3_2
- - zeromq=4.3.4=hbdafb3b_1
- - zipp=3.15.0=pyhd8ed1ab_0
- - zstd=1.5.2=hf913c23_6
- - pip:
- - appdirs==1.4.4
- - bayesian-optimization==1.4.2
- - cloudpickle==2.2.1
- - docker-pycreds==0.4.0
- - einops==0.6.1
- - fastprogress==1.0.3
- - gitdb==4.0.10
- - gitpython==3.1.31
- - gym==0.26.2
- - gym-notices==0.0.8
- - kaggle==1.5.13
- - pathtools==0.1.2
- - protobuf==4.22.3
- - python-slugify==8.0.1
- - sentry-sdk==1.21.1
- - setproctitle==1.3.2
- - smmap==5.0.0
- - text-unidecode==1.3
- - wandb==0.15.1
-prefix: /Users/gonzalo/miniconda3/envs/torch
diff --git a/wandb/run-20230602_140554-akua37dh/files/config.yaml b/wandb/run-20230602_140554-akua37dh/files/config.yaml
deleted file mode 100644
index 4fefdd1dc62ab854ea1744c19ef94a0533a3c2fd..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/files/config.yaml
+++ /dev/null
@@ -1,36 +0,0 @@
-wandb_version: 1
-
-_wandb:
- desc: null
- value:
- python_version: 3.9.16
- cli_version: 0.15.1
- framework: torch
- is_jupyter_run: false
- is_kaggle_kernel: false
- start_time: 1685707554.373241
- t:
- 1:
- - 1
- - 41
- - 55
- 2:
- - 1
- - 41
- - 55
- 3:
- - 1
- - 2
- - 19
- - 23
- 4: 3.9.16
- 5: 0.15.1
- 8:
- - 4
- - 5
-learning_rate:
- desc: null
- value: 0.001
-optimizer:
- desc: null
- value: AdamW
diff --git a/wandb/run-20230602_140554-akua37dh/files/requirements.txt b/wandb/run-20230602_140554-akua37dh/files/requirements.txt
deleted file mode 100644
index 6e8f1b9d82b6285ec6a443854e17c34ef398f84b..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/files/requirements.txt
+++ /dev/null
@@ -1,154 +0,0 @@
-anyio==3.6.2
-appdirs==1.4.4
-appnope==0.1.3
-argon2-cffi-bindings==21.2.0
-argon2-cffi==21.3.0
-asttokens==2.2.1
-attrs==22.2.0
-backcall==0.2.0
-backports.functools-lru-cache==1.6.4
-bayesian-optimization==1.4.2
-beautifulsoup4==4.12.2
-bleach==6.0.0
-boto3==1.26.116
-botocore==1.29.116
-brotlipy==0.7.0
-cached-property==1.5.2
-certifi==2023.5.7
-cffi==1.15.1
-charset-normalizer==3.1.0
-click==8.1.3
-cloudpickle==2.2.1
-colorama==0.4.6
-comm==0.1.3
-contourpy==1.0.7
-cryptography==40.0.2
-cycler==0.11.0
-debugpy==1.6.7
-decorator==5.1.1
-defusedxml==0.7.1
-docker-pycreds==0.4.0
-einops==0.6.1
-entrypoints==0.4
-executing==1.2.0
-fastjsonschema==2.16.3
-fastprogress==1.0.3
-filelock==3.12.0
-flask==2.2.3
-flit-core==3.8.0
-fonttools==4.39.3
-gitdb==4.0.10
-gitpython==3.1.31
-gmpy2==2.1.2
-gym-notices==0.0.8
-gym==0.26.2
-h5py==3.8.0
-idna==3.4
-importlib-metadata==6.5.0
-importlib-resources==5.12.0
-ipykernel==6.22.0
-ipython-genutils==0.2.0
-ipython==8.12.0
-ipywidgets==8.0.6
-itsdangerous==2.1.2
-jedi==0.18.2
-jinja2==3.1.2
-jmespath==1.0.1
-joblib==1.2.0
-jsonschema==4.17.3
-jupyter-client==8.2.0
-jupyter-console==6.6.3
-jupyter-core==5.3.0
-jupyter-events==0.6.3
-jupyter-server-terminals==0.4.4
-jupyter-server==2.5.0
-jupyter==1.0.0
-jupyterlab-pygments==0.2.2
-jupyterlab-widgets==3.0.7
-kaggle==1.5.13
-kiwisolver==1.4.4
-lxml==4.9.2
-markupsafe==2.1.2
-matplotlib-inline==0.1.6
-matplotlib==3.7.1
-mistune==2.0.5
-mpmath==1.3.0
-munkres==1.1.4
-nbclassic==0.5.5
-nbclient==0.7.3
-nbconvert==7.3.1
-nbformat==5.8.0
-nest-asyncio==1.5.6
-networkx==3.1
-notebook-shim==0.2.2
-notebook==6.5.4
-numpy==1.24.2
-packaging==23.1
-pandas-datareader==0.10.0
-pandas==2.0.0
-pandocfilters==1.5.0
-parso==0.8.3
-pathtools==0.1.2
-pexpect==4.8.0
-pickleshare==0.7.5
-pillow==9.4.0
-pip==23.1
-pkgutil-resolve-name==1.3.10
-platformdirs==3.2.0
-pooch==1.7.0
-prometheus-client==0.16.0
-prompt-toolkit==3.0.38
-protobuf==4.22.3
-psutil==5.9.5
-ptyprocess==0.7.0
-pure-eval==0.2.2
-pycparser==2.21
-pygments==2.15.1
-pyopenssl==23.1.1
-pyparsing==3.0.9
-pyrsistent==0.19.3
-pysocks==1.7.1
-python-dateutil==2.8.2
-python-json-logger==2.0.7
-python-slugify==8.0.1
-pytz==2023.3
-pyyaml==6.0
-pyzmq==25.0.2
-requests==2.28.2
-rfc3339-validator==0.1.4
-rfc3986-validator==0.1.1
-s3transfer==0.6.0
-scikit-learn==1.2.2
-scipy==1.10.1
-seaborn==0.12.2
-send2trash==1.8.0
-sentry-sdk==1.21.1
-setproctitle==1.3.2
-setuptools==67.6.1
-six==1.16.0
-smmap==5.0.0
-sniffio==1.3.0
-soupsieve==2.3.2.post1
-stack-data==0.6.2
-sympy==1.11.1
-terminado==0.17.1
-text-unidecode==1.3
-threadpoolctl==3.1.0
-tinycss2==1.2.1
-torch==2.1.0.dev20230419
-torchvision==0.16.0.dev20230419
-tornado==6.3
-tqdm==4.65.0
-traitlets==5.9.0
-typing-extensions==4.5.0
-tzdata==2023.3
-unicodedata2==15.0.0
-urllib3==1.26.15
-wandb==0.15.1
-wcwidth==0.2.6
-webencodings==0.5.1
-websocket-client==1.5.1
-werkzeug==2.2.3
-wheel==0.40.0
-widgetsnbextension==4.0.7
-zipp==3.15.0
\ No newline at end of file
diff --git a/wandb/run-20230602_140554-akua37dh/files/wandb-metadata.json b/wandb/run-20230602_140554-akua37dh/files/wandb-metadata.json
deleted file mode 100644
index a233be8372c392a23d418e80631ad3e49ef72769..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/files/wandb-metadata.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "os": "macOS-13.3.1-arm64-arm-64bit",
- "python": "3.9.16",
- "heartbeatAt": "2023-06-02T12:05:55.197158",
- "startedAt": "2023-06-02T12:05:54.363349",
- "docker": null,
- "cuda": null,
- "args": [
- "train",
- "/Users/gonzalo/Desktop/testing/test1"
- ],
- "state": "running",
- "program": "/Users/gonzalo/git/diffusion_project/main.py",
- "codePath": "main.py",
- "git": {
- "remote": "https://git.rwth-aachen.de/diffusion-project/diffusion_project.git",
- "commit": "7b0f0c0f49b1f1d776cd4b4e73e72095ed847521"
- },
- "email": "gonzalomartingarcia0@gmail.com",
- "root": "/Users/gonzalo/git/diffusion_project",
- "host": "Gonzalos-iMac.local",
- "username": "gonzalo",
- "executable": "/Users/gonzalo/miniconda3/envs/torch/bin/python",
- "cpu_count": 8,
- "cpu_count_logical": 8,
- "disk": {
- "total": 460.4317207336426,
- "used": 121.39056396484375
- },
- "gpuapple": {
- "type": "arm",
- "vendor": "Apple"
- },
- "memory": {
- "total": 16.0
- }
-}
diff --git a/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json b/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json
deleted file mode 100644
index 6a2353df9a39aec28b5e444685dc5d7223bc37fd..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json
+++ /dev/null
@@ -1 +0,0 @@
-{"_wandb": {"runtime": 2}}
\ No newline at end of file
diff --git a/wandb/run-20230602_140554-akua37dh/logs/debug-internal.log b/wandb/run-20230602_140554-akua37dh/logs/debug-internal.log
deleted file mode 100644
index 23b02c9c341bc65456fdea7c414c7f9607d49eab..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/logs/debug-internal.log
+++ /dev/null
@@ -1,179 +0,0 @@
-2023-06-02 14:05:54,371 INFO StreamThr :1368 [internal.py:wandb_internal():86] W&B internal server running at pid: 1368, started at: 2023-06-02 14:05:54.371486
-2023-06-02 14:05:54,372 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: status
-2023-06-02 14:05:54,377 INFO WriterThread:1368 [datastore.py:open_for_write():85] open: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/run-akua37dh.wandb
-2023-06-02 14:05:54,378 DEBUG SenderThread:1368 [sender.py:send():375] send: header
-2023-06-02 14:05:54,395 DEBUG SenderThread:1368 [sender.py:send():375] send: run
-2023-06-02 14:05:54,397 INFO SenderThread:1368 [sender.py:_maybe_setup_resume():761] checking resume status for gonzalomartingarcia0/test-project/akua37dh
-2023-06-02 14:05:55,002 INFO SenderThread:1368 [dir_watcher.py:__init__():219] watching files in: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files
-2023-06-02 14:05:55,003 INFO SenderThread:1368 [sender.py:_start_run_threads():1124] run started: akua37dh with start time 1685707554.373241
-2023-06-02 14:05:55,003 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:05:55,003 INFO SenderThread:1368 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:05:55,006 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: check_version
-2023-06-02 14:05:55,006 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: check_version
-2023-06-02 14:05:55,189 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: run_start
-2023-06-02 14:05:55,196 DEBUG HandlerThread:1368 [system_info.py:__init__():31] System info init
-2023-06-02 14:05:55,196 DEBUG HandlerThread:1368 [system_info.py:__init__():46] System info init done
-2023-06-02 14:05:55,196 INFO HandlerThread:1368 [system_monitor.py:start():181] Starting system monitor
-2023-06-02 14:05:55,196 INFO SystemMonitor:1368 [system_monitor.py:_start():145] Starting system asset monitoring threads
-2023-06-02 14:05:55,196 INFO HandlerThread:1368 [system_monitor.py:probe():201] Collecting system info
-2023-06-02 14:05:55,196 INFO SystemMonitor:1368 [interfaces.py:start():190] Started cpu monitoring
-2023-06-02 14:05:55,197 DEBUG HandlerThread:1368 [system_info.py:probe():195] Probing system
-2023-06-02 14:05:55,197 INFO SystemMonitor:1368 [interfaces.py:start():190] Started disk monitoring
-2023-06-02 14:05:55,197 INFO SystemMonitor:1368 [interfaces.py:start():190] Started gpuapple monitoring
-2023-06-02 14:05:55,199 INFO SystemMonitor:1368 [interfaces.py:start():190] Started memory monitoring
-2023-06-02 14:05:55,200 INFO SystemMonitor:1368 [interfaces.py:start():190] Started network monitoring
-2023-06-02 14:05:55,206 DEBUG HandlerThread:1368 [system_info.py:_probe_git():180] Probing git
-2023-06-02 14:05:55,231 DEBUG HandlerThread:1368 [system_info.py:_probe_git():188] Probing git done
-2023-06-02 14:05:55,231 DEBUG HandlerThread:1368 [system_info.py:probe():240] Probing system done
-2023-06-02 14:05:55,231 DEBUG HandlerThread:1368 [system_monitor.py:probe():210] {'os': 'macOS-13.3.1-arm64-arm-64bit', 'python': '3.9.16', 'heartbeatAt': '2023-06-02T12:05:55.197158', 'startedAt': '2023-06-02T12:05:54.363349', 'docker': None, 'cuda': None, 'args': ('train', '/Users/gonzalo/Desktop/testing/test1'), 'state': 'running', 'program': '/Users/gonzalo/git/diffusion_project/main.py', 'codePath': 'main.py', 'git': {'remote': 'https://git.rwth-aachen.de/diffusion-project/diffusion_project.git', 'commit': '7b0f0c0f49b1f1d776cd4b4e73e72095ed847521'}, 'email': 'gonzalomartingarcia0@gmail.com', 'root': '/Users/gonzalo/git/diffusion_project', 'host': 'Gonzalos-iMac.local', 'username': 'gonzalo', 'executable': '/Users/gonzalo/miniconda3/envs/torch/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'disk': {'total': 460.4317207336426, 'used': 121.39056396484375}, 'gpuapple': {'type': 'arm', 'vendor': 'Apple'}, 'memory': {'total': 16.0}}
-2023-06-02 14:05:55,231 INFO HandlerThread:1368 [system_monitor.py:probe():211] Finished collecting system info
-2023-06-02 14:05:55,231 INFO HandlerThread:1368 [system_monitor.py:probe():214] Publishing system info
-2023-06-02 14:05:55,231 DEBUG HandlerThread:1368 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
-2023-06-02 14:05:55,231 DEBUG HandlerThread:1368 [system_info.py:_save_pip():67] Saving pip packages done
-2023-06-02 14:05:55,232 DEBUG HandlerThread:1368 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment
-2023-06-02 14:05:56,006 INFO Thread-12 :1368 [dir_watcher.py:_on_file_created():278] file/dir created: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json
-2023-06-02 14:05:56,007 INFO Thread-12 :1368 [dir_watcher.py:_on_file_created():278] file/dir created: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml
-2023-06-02 14:05:56,007 INFO Thread-12 :1368 [dir_watcher.py:_on_file_created():278] file/dir created: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/requirements.txt
-2023-06-02 14:05:57,273 DEBUG HandlerThread:1368 [system_info.py:_save_conda():86] Saving conda packages done
-2023-06-02 14:05:57,274 INFO HandlerThread:1368 [system_monitor.py:probe():216] Finished publishing system info
-2023-06-02 14:05:57,277 DEBUG SenderThread:1368 [sender.py:send():375] send: files
-2023-06-02 14:05:57,277 INFO SenderThread:1368 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now
-2023-06-02 14:05:57,282 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:05:57,282 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:05:57,514 DEBUG SenderThread:1368 [sender.py:send():375] send: telemetry
-2023-06-02 14:05:57,514 DEBUG SenderThread:1368 [sender.py:send():375] send: telemetry
-2023-06-02 14:05:57,514 DEBUG SenderThread:1368 [sender.py:send():375] send: config
-2023-06-02 14:05:57,515 DEBUG SenderThread:1368 [sender.py:send():375] send: config
-2023-06-02 14:05:57,515 DEBUG SenderThread:1368 [sender.py:send():375] send: telemetry
-2023-06-02 14:05:57,515 DEBUG SenderThread:1368 [sender.py:send():375] send: telemetry
-2023-06-02 14:05:57,515 DEBUG SenderThread:1368 [sender.py:send():375] send: exit
-2023-06-02 14:05:57,515 INFO SenderThread:1368 [sender.py:send_exit():598] handling exit code: 1
-2023-06-02 14:05:57,515 INFO SenderThread:1368 [sender.py:send_exit():600] handling runtime: 2
-2023-06-02 14:05:57,516 INFO SenderThread:1368 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:05:57,516 INFO SenderThread:1368 [sender.py:send_exit():606] send defer
-2023-06-02 14:05:57,517 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,517 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 0
-2023-06-02 14:05:57,517 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,517 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 0
-2023-06-02 14:05:57,517 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 1
-2023-06-02 14:05:57,517 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,517 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 1
-2023-06-02 14:05:57,518 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,518 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 1
-2023-06-02 14:05:57,518 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 2
-2023-06-02 14:05:57,518 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,518 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 2
-2023-06-02 14:05:57,518 INFO HandlerThread:1368 [system_monitor.py:finish():190] Stopping system monitor
-2023-06-02 14:05:57,518 DEBUG SystemMonitor:1368 [system_monitor.py:_start():159] Starting system metrics aggregation loop
-2023-06-02 14:05:57,518 INFO HandlerThread:1368 [interfaces.py:finish():202] Joined cpu monitor
-2023-06-02 14:05:57,519 DEBUG SystemMonitor:1368 [system_monitor.py:_start():166] Finished system metrics aggregation loop
-2023-06-02 14:05:57,519 INFO HandlerThread:1368 [interfaces.py:finish():202] Joined disk monitor
-2023-06-02 14:05:57,519 DEBUG SystemMonitor:1368 [system_monitor.py:_start():170] Publishing last batch of metrics
-2023-06-02 14:05:57,519 INFO HandlerThread:1368 [interfaces.py:finish():202] Joined gpuapple monitor
-2023-06-02 14:05:57,520 INFO HandlerThread:1368 [interfaces.py:finish():202] Joined memory monitor
-2023-06-02 14:05:57,520 INFO HandlerThread:1368 [interfaces.py:finish():202] Joined network monitor
-2023-06-02 14:05:57,521 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,521 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 2
-2023-06-02 14:05:57,521 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 3
-2023-06-02 14:05:57,521 DEBUG SenderThread:1368 [sender.py:send():375] send: telemetry
-2023-06-02 14:05:57,521 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,521 DEBUG SenderThread:1368 [sender.py:send():375] send: stats
-2023-06-02 14:05:57,521 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 3
-2023-06-02 14:05:57,522 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,522 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 3
-2023-06-02 14:05:57,522 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 4
-2023-06-02 14:05:57,522 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,522 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 4
-2023-06-02 14:05:57,522 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,523 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 4
-2023-06-02 14:05:57,523 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 5
-2023-06-02 14:05:57,523 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,523 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 5
-2023-06-02 14:05:57,523 DEBUG SenderThread:1368 [sender.py:send():375] send: summary
-2023-06-02 14:05:57,524 INFO SenderThread:1368 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:05:57,524 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,524 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 5
-2023-06-02 14:05:57,524 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 6
-2023-06-02 14:05:57,524 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,524 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 6
-2023-06-02 14:05:57,524 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,524 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 6
-2023-06-02 14:05:57,527 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:05:57,725 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 7
-2023-06-02 14:05:57,726 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,726 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 7
-2023-06-02 14:05:57,726 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,726 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 7
-2023-06-02 14:05:57,726 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 8
-2023-06-02 14:05:57,726 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,726 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 8
-2023-06-02 14:05:57,727 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:05:57,727 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 8
-2023-06-02 14:05:57,735 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 9
-2023-06-02 14:05:57,735 DEBUG SenderThread:1368 [sender.py:send():375] send: artifact
-2023-06-02 14:05:57,736 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:05:57,739 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 9
-2023-06-02 14:05:57,897 INFO wandb-upload_0:1368 [upload_job.py:push():137] Uploaded file /var/folders/8r/kg68c_2j23d5ms25g3y7_z900000gn/T/tmpa6260m3cwandb/129c9et7-wandb-metadata.json
-2023-06-02 14:05:58,011 INFO Thread-12 :1368 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json
-2023-06-02 14:05:58,011 INFO Thread-12 :1368 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml
-2023-06-02 14:05:58,011 INFO Thread-12 :1368 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/config.yaml
-2023-06-02 14:05:58,012 INFO Thread-12 :1368 [dir_watcher.py:_on_file_created():278] file/dir created: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/wandb-metadata.json
-2023-06-02 14:05:58,293 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: poll_exit
-2023-06-02 14:05:59,084 INFO wandb-upload_1:1368 [upload_job.py:push():95] Uploaded file /Users/gonzalo/Library/Application Support/wandb/artifacts/staging/tmpn47ets0n
-2023-06-02 14:05:59,116 INFO wandb-upload_0:1368 [upload_job.py:push():95] Uploaded file /Users/gonzalo/Library/Application Support/wandb/artifacts/staging/tmpu_6c0hqf
-2023-06-02 14:06:00,441 INFO SenderThread:1368 [sender.py:send_artifact():1474] sent artifact job-https___git.rwth-aachen.de_diffusion-project_diffusion_project.git_main.py - {'id': 'QXJ0aWZhY3Q6NDc1MDg5MzM5', 'digest': '9d690e599e66bd3379a405d5bfcc0d79', 'state': 'PENDING', 'aliases': [], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjczNzA3Njgw', 'latestArtifact': None}, 'version': 'latest'}
-2023-06-02 14:06:00,441 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:06:00,441 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 9
-2023-06-02 14:06:00,441 INFO SenderThread:1368 [dir_watcher.py:finish():365] shutting down directory watcher
-2023-06-02 14:06:01,021 INFO SenderThread:1368 [dir_watcher.py:finish():395] scan: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files
-2023-06-02 14:06:01,021 INFO SenderThread:1368 [dir_watcher.py:finish():409] scan save: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/requirements.txt requirements.txt
-2023-06-02 14:06:01,021 INFO SenderThread:1368 [dir_watcher.py:finish():409] scan save: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/config.yaml config.yaml
-2023-06-02 14:06:01,025 INFO SenderThread:1368 [dir_watcher.py:finish():409] scan save: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json wandb-summary.json
-2023-06-02 14:06:01,029 INFO SenderThread:1368 [dir_watcher.py:finish():409] scan save: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml conda-environment.yaml
-2023-06-02 14:06:01,029 INFO SenderThread:1368 [dir_watcher.py:finish():409] scan save: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/wandb-metadata.json wandb-metadata.json
-2023-06-02 14:06:01,032 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 10
-2023-06-02 14:06:01,032 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: poll_exit
-2023-06-02 14:06:01,033 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:06:01,033 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 10
-2023-06-02 14:06:01,033 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:06:01,033 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 10
-2023-06-02 14:06:01,033 INFO SenderThread:1368 [file_pusher.py:finish():167] shutting down file pusher
-2023-06-02 14:06:01,480 INFO wandb-upload_1:1368 [upload_job.py:push():137] Uploaded file /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/requirements.txt
-2023-06-02 14:06:01,541 INFO wandb-upload_0:1368 [upload_job.py:push():137] Uploaded file /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/config.yaml
-2023-06-02 14:06:01,650 INFO wandb-upload_3:1368 [upload_job.py:push():137] Uploaded file /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/conda-environment.yaml
-2023-06-02 14:06:01,738 INFO wandb-upload_2:1368 [upload_job.py:push():137] Uploaded file /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/files/wandb-summary.json
-2023-06-02 14:06:01,942 INFO Thread-11 :1368 [sender.py:transition_state():626] send defer: 11
-2023-06-02 14:06:01,943 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:06:01,943 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 11
-2023-06-02 14:06:01,943 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:06:01,944 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 11
-2023-06-02 14:06:01,944 INFO SenderThread:1368 [file_pusher.py:join():172] waiting for file pusher
-2023-06-02 14:06:01,944 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 12
-2023-06-02 14:06:01,944 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:06:01,944 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 12
-2023-06-02 14:06:01,944 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:06:01,944 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 12
-2023-06-02 14:06:02,137 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 13
-2023-06-02 14:06:02,138 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:06:02,138 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 13
-2023-06-02 14:06:02,139 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:06:02,139 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 13
-2023-06-02 14:06:02,139 INFO SenderThread:1368 [sender.py:transition_state():626] send defer: 14
-2023-06-02 14:06:02,139 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: defer
-2023-06-02 14:06:02,140 INFO HandlerThread:1368 [handler.py:handle_request_defer():170] handle defer: 14
-2023-06-02 14:06:02,140 DEBUG SenderThread:1368 [sender.py:send():375] send: final
-2023-06-02 14:06:02,140 DEBUG SenderThread:1368 [sender.py:send():375] send: footer
-2023-06-02 14:06:02,141 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: defer
-2023-06-02 14:06:02,141 INFO SenderThread:1368 [sender.py:send_request_defer():622] handle sender defer: 14
-2023-06-02 14:06:02,143 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: poll_exit
-2023-06-02 14:06:02,143 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: server_info
-2023-06-02 14:06:02,143 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: poll_exit
-2023-06-02 14:06:02,143 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: get_summary
-2023-06-02 14:06:02,144 DEBUG SenderThread:1368 [sender.py:send_request():402] send_request: server_info
-2023-06-02 14:06:02,144 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: sampled_history
-2023-06-02 14:06:02,297 DEBUG HandlerThread:1368 [handler.py:handle_request():144] handle_request: shutdown
-2023-06-02 14:06:02,297 INFO HandlerThread:1368 [handler.py:finish():845] shutting down handler
-2023-06-02 14:06:03,149 INFO WriterThread:1368 [datastore.py:close():298] close: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/run-akua37dh.wandb
-2023-06-02 14:06:03,300 INFO SenderThread:1368 [sender.py:finish():1550] shutting down sender
-2023-06-02 14:06:03,300 INFO SenderThread:1368 [file_pusher.py:finish():167] shutting down file pusher
-2023-06-02 14:06:03,300 INFO SenderThread:1368 [file_pusher.py:join():172] waiting for file pusher
diff --git a/wandb/run-20230602_140554-akua37dh/logs/debug.log b/wandb/run-20230602_140554-akua37dh/logs/debug.log
deleted file mode 100644
index 3f5ae9b6436073a536e4d0840fe08d75464d546b..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_140554-akua37dh/logs/debug.log
+++ /dev/null
@@ -1,39 +0,0 @@
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Current SDK version is 0.15.1
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Configure stats pid to 1361
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Loading settings from /Users/gonzalo/.config/wandb/settings
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Loading settings from /Users/gonzalo/git/diffusion_project/wandb/settings
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'main.py', 'program': '/Users/gonzalo/git/diffusion_project/main.py'}
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_init.py:_log_setup():507] Logging user logs to /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/logs/debug.log
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_init.py:_log_setup():508] Logging internal logs to /Users/gonzalo/git/diffusion_project/wandb/run-20230602_140554-akua37dh/logs/debug-internal.log
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_init.py:init():547] calling init triggers
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
-config: {}
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_init.py:init():596] starting backend
-2023-06-02 14:05:54,366 INFO MainThread:1361 [wandb_init.py:init():600] setting up manager
-2023-06-02 14:05:54,369 INFO MainThread:1361 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=spawn,fork,forkserver, using: spawn
-2023-06-02 14:05:54,373 INFO MainThread:1361 [wandb_init.py:init():606] backend started and connected
-2023-06-02 14:05:54,377 INFO MainThread:1361 [wandb_init.py:init():700] updated telemetry
-2023-06-02 14:05:54,395 INFO MainThread:1361 [wandb_init.py:init():737] communicating run to backend with 60.0 second timeout
-2023-06-02 14:05:55,005 INFO MainThread:1361 [wandb_run.py:_on_init():2177] communicating current version
-2023-06-02 14:05:55,185 INFO MainThread:1361 [wandb_run.py:_on_init():2186] got version response upgrade_message: "wandb version 0.15.3 is available! To upgrade, please run:\n $ pip install wandb --upgrade"
-
-2023-06-02 14:05:55,185 INFO MainThread:1361 [wandb_init.py:init():787] starting run threads in backend
-2023-06-02 14:05:57,281 INFO MainThread:1361 [wandb_run.py:_console_start():2158] atexit reg
-2023-06-02 14:05:57,281 INFO MainThread:1361 [wandb_run.py:_redirect():2013] redirect: SettingsConsole.WRAP_RAW
-2023-06-02 14:05:57,281 INFO MainThread:1361 [wandb_run.py:_redirect():2078] Wrapping output streams.
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_run.py:_redirect():2103] Redirects installed.
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_init.py:init():829] run started, returning control to user process
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_config.py:__setitem__():151] config set learning_rate = 0.001 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x13262ad00>>
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_run.py:_config_callback():1286] config_cb learning_rate 0.001 None
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_config.py:__setitem__():151] config set optimizer = AdamW - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x13262ad00>>
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_run.py:_config_callback():1286] config_cb optimizer AdamW None
-2023-06-02 14:05:57,282 INFO MainThread:1361 [wandb_watch.py:watch():51] Watching
-2023-06-02 14:05:57,286 INFO MainThread:1361 [wandb_run.py:_finish():1893] finishing run gonzalomartingarcia0/test-project/akua37dh
-2023-06-02 14:05:57,286 INFO MainThread:1361 [wandb_run.py:_atexit_cleanup():2127] got exitcode: 1
-2023-06-02 14:05:57,286 INFO MainThread:1361 [wandb_run.py:_restore():2110] restore
-2023-06-02 14:05:57,286 INFO MainThread:1361 [wandb_run.py:_restore():2116] restore done
-2023-06-02 14:06:03,299 INFO MainThread:1361 [wandb_run.py:_footer_history_summary_info():3469] rendering history
-2023-06-02 14:06:03,300 INFO MainThread:1361 [wandb_run.py:_footer_history_summary_info():3501] rendering summary
-2023-06-02 14:06:03,305 INFO MainThread:1361 [wandb_run.py:_footer_sync_info():3428] logging synced files
diff --git a/wandb/run-20230602_140554-akua37dh/run-akua37dh.wandb b/wandb/run-20230602_140554-akua37dh/run-akua37dh.wandb
deleted file mode 100644
index 239939f77c8d446dabb8b3e4656701d770bf5e86..0000000000000000000000000000000000000000
Binary files a/wandb/run-20230602_140554-akua37dh/run-akua37dh.wandb and /dev/null differ
diff --git a/wandb/run-20230602_141242-akua37dh/files/config.yaml b/wandb/run-20230602_141242-akua37dh/files/config.yaml
deleted file mode 100644
index 02b1e24ab08e90efca07a9c7f5cc62181f1e6558..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_141242-akua37dh/files/config.yaml
+++ /dev/null
@@ -1,36 +0,0 @@
-wandb_version: 1
-
-_wandb:
- desc: null
- value:
- python_version: 3.9.16
- cli_version: 0.15.1
- framework: torch
- is_jupyter_run: false
- is_kaggle_kernel: false
- start_time: 1685707962.170081
- t:
- 1:
- - 1
- - 41
- - 55
- 2:
- - 1
- - 41
- - 55
- 3:
- - 1
- - 5
- - 19
- - 23
- 4: 3.9.16
- 5: 0.15.1
- 8:
- - 4
- - 5
-optimizer:
- desc: null
- value: AdamW
-learning_rate:
- desc: null
- value: 0.001
diff --git a/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json b/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
deleted file mode 100644
index f15d79468ec0b3654cc1b4ca811e784c0fa9c93b..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
+++ /dev/null
@@ -1 +0,0 @@
-{"loss": 0.3680811822414398, "epoch": 0, "batch": 882, "_timestamp": 1685708336.088873, "_runtime": 377.0653259754181, "_step": 883}
\ No newline at end of file
diff --git a/wandb/run-20230602_141242-akua37dh/logs/debug-internal.log b/wandb/run-20230602_141242-akua37dh/logs/debug-internal.log
deleted file mode 100644
index 7a4adcf2fbe1d7771cd6ee67e94920ff4dc35dd2..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_141242-akua37dh/logs/debug-internal.log
+++ /dev/null
@@ -1,4298 +0,0 @@
-2023-06-02 14:12:42,167 INFO StreamThr :1509 [internal.py:wandb_internal():86] W&B internal server running at pid: 1509, started at: 2023-06-02 14:12:42.167628
-2023-06-02 14:12:42,168 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status
-2023-06-02 14:12:42,174 INFO WriterThread:1509 [datastore.py:open_for_write():85] open: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/run-akua37dh.wandb
-2023-06-02 14:12:42,174 DEBUG SenderThread:1509 [sender.py:send():375] send: header
-2023-06-02 14:12:42,192 DEBUG SenderThread:1509 [sender.py:send():375] send: run
-2023-06-02 14:12:42,194 INFO SenderThread:1509 [sender.py:_maybe_setup_resume():761] checking resume status for gonzalomartingarcia0/test-project/akua37dh
-2023-06-02 14:12:42,430 INFO SenderThread:1509 [sender.py:_maybe_setup_resume():833] configured resuming with: ResumeState(resumed=True,step=1,history=0,events=1,output=0,runtime=3.146534,wandb_runtime=2,summary={'_wandb': {'runtime': 2}},config={'_wandb': {'desc': None, 'value': {'t': {'1': [1, 41, 55], '2': [1, 41, 55], '3': [1, 2, 19, 23], '4': '3.9.16', '5': '0.15.1', '8': [4, 5]}, 'framework': 'torch', 'start_time': 1685707554.373241, 'cli_version': '0.15.1', 'is_jupyter_run': False, 'python_version': '3.9.16', 'is_kaggle_kernel': False}}, 'optimizer': {'desc': None, 'value': 'AdamW'}, 'learning_rate': {'desc': None, 'value': 0.001}})
-2023-06-02 14:12:42,791 INFO SenderThread:1509 [dir_watcher.py:__init__():219] watching files in: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files
-2023-06-02 14:12:42,791 INFO SenderThread:1509 [sender.py:_start_run_threads():1124] run started: akua37dh with start time 1685707959.023547
-2023-06-02 14:12:42,791 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:42,791 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:42,794 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: check_version
-2023-06-02 14:12:42,794 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: check_version
-2023-06-02 14:12:42,908 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: run_start
-2023-06-02 14:12:42,914 DEBUG HandlerThread:1509 [system_info.py:__init__():31] System info init
-2023-06-02 14:12:42,914 DEBUG HandlerThread:1509 [system_info.py:__init__():46] System info init done
-2023-06-02 14:12:42,914 INFO HandlerThread:1509 [system_monitor.py:start():181] Starting system monitor
-2023-06-02 14:12:42,914 INFO SystemMonitor:1509 [system_monitor.py:_start():145] Starting system asset monitoring threads
-2023-06-02 14:12:42,915 INFO SystemMonitor:1509 [interfaces.py:start():190] Started cpu monitoring
-2023-06-02 14:12:42,915 INFO SystemMonitor:1509 [interfaces.py:start():190] Started disk monitoring
-2023-06-02 14:12:42,917 INFO SystemMonitor:1509 [interfaces.py:start():190] Started gpuapple monitoring
-2023-06-02 14:12:42,919 INFO SystemMonitor:1509 [interfaces.py:start():190] Started memory monitoring
-2023-06-02 14:12:42,920 INFO SystemMonitor:1509 [interfaces.py:start():190] Started network monitoring
-2023-06-02 14:12:42,926 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:12:42,926 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:12:43,082 DEBUG SenderThread:1509 [sender.py:send():375] send: telemetry
-2023-06-02 14:12:43,082 DEBUG SenderThread:1509 [sender.py:send():375] send: telemetry
-2023-06-02 14:12:43,082 DEBUG SenderThread:1509 [sender.py:send():375] send: config
-2023-06-02 14:12:43,083 DEBUG SenderThread:1509 [sender.py:send():375] send: config
-2023-06-02 14:12:43,083 DEBUG SenderThread:1509 [sender.py:send():375] send: telemetry
-2023-06-02 14:12:43,450 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:43,450 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:43,451 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:43,451 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:43,796 INFO Thread-12 :1509 [dir_watcher.py:_on_file_created():278] file/dir created: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:43,858 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:43,859 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:43,859 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:43,859 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:44,260 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:44,260 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:44,260 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:44,260 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:44,646 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:44,646 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:44,646 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:44,647 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:44,796 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:45,043 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:45,043 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:45,043 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:45,044 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:45,434 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:45,435 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:45,435 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:45,436 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:45,802 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:45,854 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:45,854 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:45,854 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:45,855 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:46,266 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:46,267 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:46,267 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:46,267 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:46,666 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:46,667 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:46,667 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:46,667 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:46,803 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:47,065 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:47,065 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:47,065 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:47,066 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:47,472 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:47,472 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:47,473 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:47,473 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:12:47,473 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:47,804 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:47,884 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:47,884 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:47,884 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:47,884 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:48,296 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:48,297 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:48,297 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:48,297 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:48,697 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:48,698 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:48,698 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:48,698 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:48,809 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:49,112 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:49,113 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:49,113 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:49,114 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:49,512 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:49,512 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:49,513 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:49,513 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:49,813 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:49,911 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:49,911 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:49,911 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:49,912 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:50,311 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:50,311 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:50,312 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:50,312 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:50,727 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:50,728 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:50,728 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:50,728 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:50,818 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:51,143 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:51,143 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:51,143 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:51,144 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:51,552 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:51,553 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:51,553 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:51,553 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:51,821 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:51,957 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:51,957 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:51,957 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:51,957 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:52,353 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:52,353 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:52,353 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:52,354 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:52,749 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:52,749 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:52,749 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:52,749 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:12:52,749 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:52,823 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:53,158 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:53,159 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:53,159 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:53,159 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:53,583 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:53,584 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:53,584 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:53,584 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:53,824 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:54,001 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:54,001 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:54,001 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:54,001 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:54,403 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:54,404 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:54,404 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:54,404 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:54,821 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:54,821 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:54,821 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:54,821 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:54,828 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:55,219 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:55,220 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:55,220 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:55,220 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:55,606 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:55,607 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:55,607 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:55,607 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:55,834 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:56,179 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:56,180 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:56,180 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:56,180 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:56,630 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:56,631 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:56,631 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:56,631 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:56,837 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:57,046 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:57,047 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:57,047 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:57,047 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:57,444 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:57,444 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:57,444 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:57,445 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:57,839 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:57,840 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:57,840 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:57,840 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:12:57,840 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:57,842 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:57,925 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:12:57,926 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:12:58,226 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:58,227 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:58,227 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:58,227 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:58,633 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:58,633 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:58,633 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:58,633 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:58,845 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:12:59,031 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:59,032 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:59,032 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:59,032 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:59,415 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:59,415 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:59,415 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:59,415 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:59,799 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:12:59,799 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:12:59,799 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:12:59,800 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:12:59,845 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:00,183 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:00,183 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:00,183 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:00,183 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:00,571 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:00,571 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:00,571 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:00,571 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:00,846 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:00,981 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:00,982 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:00,982 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:00,982 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:01,376 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:01,376 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:01,376 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:01,376 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:01,783 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:01,783 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:01,783 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:01,784 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:01,851 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:02,202 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:02,202 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:02,203 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:02,203 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:02,617 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:02,618 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:02,618 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:02,618 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:02,856 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:03,030 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:03,030 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:03,030 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:03,030 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:03,030 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:03,435 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:03,436 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:03,436 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:03,436 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:03,858 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:03,858 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:03,858 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:03,858 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:03,861 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:04,266 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:04,266 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:04,266 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:04,267 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:04,674 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:04,674 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:04,674 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:04,675 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:04,866 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:05,258 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:05,259 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:05,259 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:05,259 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:05,728 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:05,729 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:05,730 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:05,730 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:05,870 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:06,221 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:06,221 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:06,222 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:06,222 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:06,652 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:06,652 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:06,653 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:06,653 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:06,875 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:07,087 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:07,087 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:07,087 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:07,087 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:07,518 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:07,519 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:07,520 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:07,520 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:07,878 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:07,945 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:07,945 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:07,946 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:07,946 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:08,373 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:08,373 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:08,374 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:08,374 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:08,374 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:08,829 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:08,830 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:08,830 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:08,831 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:08,883 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:09,264 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:09,264 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:09,264 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:09,265 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:09,711 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:09,711 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:09,711 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:09,711 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:09,885 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:10,171 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:10,172 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:10,172 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:10,172 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:10,715 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:10,716 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:10,716 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:10,717 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:10,898 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:11,542 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:11,543 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:11,543 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:11,544 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:11,903 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:11,994 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:11,995 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:11,995 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:11,995 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:12,422 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:12,423 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:12,423 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:12,423 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:12,913 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:12,937 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:13:12,938 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:13:13,049 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:13,131 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:13,132 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:13,135 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:13,599 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:13,600 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:13,603 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:13,814 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:13,814 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:13,913 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/config.yaml
-2023-06-02 14:13:13,913 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:14,019 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:14,020 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:14,020 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:14,020 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:14,425 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:14,426 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:14,426 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:14,426 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:14,830 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:14,830 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:14,830 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:14,830 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:14,913 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:15,220 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:15,220 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:15,220 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:15,221 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:15,684 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:15,684 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:15,684 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:15,685 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:15,918 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:16,194 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:16,195 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:16,195 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:16,196 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:16,650 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:16,650 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:16,650 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:16,651 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:16,922 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:17,053 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:17,053 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:17,053 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:17,053 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:17,459 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:17,460 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:17,460 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:17,460 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:17,847 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:17,847 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:17,847 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:17,848 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:17,928 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:18,236 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:18,236 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:18,236 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:18,236 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:18,620 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:18,621 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:18,621 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:18,621 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:18,933 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:19,026 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:19,026 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:19,026 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:19,027 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:19,027 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:19,422 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:19,422 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:19,422 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:19,422 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:19,820 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:19,820 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:19,820 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:19,821 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:19,938 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:20,214 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:20,214 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:20,215 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:20,215 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:20,614 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:20,615 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:20,615 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:20,615 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:20,943 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:21,010 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:21,011 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:21,011 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:21,011 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:21,406 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:21,407 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:21,407 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:21,407 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:21,808 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:21,808 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:21,809 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:21,809 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:21,948 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:22,203 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:22,203 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:22,204 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:22,204 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:22,594 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:22,595 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:22,595 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:22,596 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:22,952 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:22,988 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:22,989 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:22,989 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:22,989 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:23,386 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:23,387 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:23,387 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:23,387 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:23,812 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:23,813 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:23,813 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:23,813 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:23,956 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:24,254 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:24,254 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:24,254 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:24,254 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:24,255 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:24,672 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:24,672 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:24,672 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:24,673 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:24,957 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:25,117 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,117 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:25,117 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:25,117 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:25,269 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,270 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,284 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,311 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,315 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,316 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,327 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,347 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,352 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,352 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,361 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,377 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,385 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,385 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,402 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,435 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,450 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,450 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,502 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,524 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,540 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,549 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,557 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,563 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,574 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,580 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,595 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,596 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,669 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:25,673 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:25,675 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:25,677 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:25,962 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:26,094 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:26,095 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:26,095 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:26,095 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:26,508 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:26,508 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:26,508 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:26,508 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:26,915 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:26,916 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:26,916 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:26,917 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:26,967 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:27,322 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:27,322 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:27,322 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:27,323 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:27,819 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:27,820 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:27,820 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:27,820 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:27,934 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:13:27,934 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:13:27,972 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:28,268 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:28,269 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:28,269 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:28,269 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:28,794 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:28,796 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:28,796 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:28,797 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:28,977 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:29,316 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:29,316 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:29,316 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:29,316 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:29,317 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:29,715 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:29,715 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:29,715 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:29,716 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:29,979 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:30,186 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:30,186 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:30,186 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:30,187 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:30,608 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:30,609 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:30,609 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:30,609 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:30,980 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:31,005 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:31,006 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:31,006 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:31,006 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:31,399 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:31,399 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:31,400 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:31,401 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:31,810 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:31,810 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:31,810 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:31,810 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:31,983 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:32,196 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:32,196 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:32,196 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:32,197 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:32,581 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:32,582 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:32,582 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:32,582 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:32,983 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:32,983 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:32,983 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:32,983 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:32,988 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:33,375 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:33,375 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:33,375 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:33,375 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:33,793 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:33,793 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:33,793 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:33,794 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:33,992 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:34,196 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:34,197 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:34,197 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:34,197 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:34,595 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:34,596 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:34,596 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:34,596 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:34,596 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:34,991 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:34,991 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:34,991 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:34,991 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:34,997 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:35,390 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:35,391 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:35,391 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:35,391 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:35,793 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:35,793 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:35,793 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:35,794 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:35,999 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:36,177 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:36,177 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:36,177 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:36,177 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:36,580 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:36,580 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:36,581 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:36,581 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:36,986 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:36,986 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:36,986 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:36,987 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:37,002 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:37,370 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:37,370 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:37,370 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:37,370 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:37,759 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:37,759 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:37,759 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:37,760 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:38,007 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:38,151 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:38,151 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:38,151 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:38,152 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:38,531 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:38,531 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:38,532 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:38,532 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:38,926 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:38,927 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:38,927 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:38,927 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:39,009 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:39,342 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:39,342 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:39,342 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:39,343 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:39,735 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:39,735 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:39,735 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:39,735 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:39,735 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:40,014 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:40,125 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:40,126 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:40,126 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:40,126 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:40,518 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:40,519 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:40,519 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:40,519 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:40,914 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:40,914 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:40,914 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:40,915 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:41,015 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:41,309 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:41,310 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:41,310 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:41,310 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:41,713 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:41,713 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:41,714 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:41,714 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:42,019 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:42,110 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:42,110 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:42,110 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:42,110 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:42,531 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:42,531 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:42,531 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:42,531 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:42,921 DEBUG SystemMonitor:1509 [system_monitor.py:_start():159] Starting system metrics aggregation loop
-2023-06-02 14:13:42,923 DEBUG SenderThread:1509 [sender.py:send():375] send: telemetry
-2023-06-02 14:13:42,923 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:13:42,934 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:13:42,937 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:13:42,952 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:43,022 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:43,083 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:43,083 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:43,084 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:43,350 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:43,350 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:43,350 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:43,350 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:43,740 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:43,740 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:43,740 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:43,740 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:44,026 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:44,176 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:44,177 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:44,177 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:44,178 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:44,713 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:44,714 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:44,714 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:44,715 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:45,032 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:45,113 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:45,113 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:45,116 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:45,313 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:45,314 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:45,513 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:45,514 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:45,514 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:45,514 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:46,034 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/config.yaml
-2023-06-02 14:13:46,037 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:46,077 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:46,077 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:46,077 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:46,078 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:46,559 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:46,560 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:46,560 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:46,561 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:46,990 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:46,990 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:46,990 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:46,990 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:47,036 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:47,399 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:47,399 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:47,399 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:47,399 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:47,810 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:47,811 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:47,811 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:47,811 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:48,036 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:48,220 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:48,221 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:48,221 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:48,221 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:48,638 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:48,638 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:48,638 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:48,639 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:49,035 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:49,036 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:49,036 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:49,037 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:49,037 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:49,452 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:49,452 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:49,452 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:49,453 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:49,843 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:49,843 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:49,843 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:49,844 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:50,042 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:50,235 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:50,236 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:50,236 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:50,236 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:50,623 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:50,624 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:50,624 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:50,624 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:50,624 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:51,013 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:51,013 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:51,013 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:51,013 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:51,043 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:51,419 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:51,419 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:51,419 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:51,419 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:51,839 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:51,840 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:51,840 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:51,840 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:52,043 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:52,243 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:52,244 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:52,244 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:52,244 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:52,639 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:52,639 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:52,639 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:52,639 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:53,041 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:53,041 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:53,041 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:53,041 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:53,045 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:53,437 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:53,438 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:53,438 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:53,438 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:53,857 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:53,857 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:53,857 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:53,858 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:54,049 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:54,274 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:54,274 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:54,274 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:54,275 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:54,707 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:54,707 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:54,707 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:54,708 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:55,054 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:55,115 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:55,115 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:55,115 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:55,115 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:55,504 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:55,504 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:55,504 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:55,504 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:55,901 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:55,901 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:55,901 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:55,902 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:13:55,902 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:56,056 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:56,308 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:56,308 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:56,308 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:56,308 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:56,732 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:56,734 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:56,735 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:56,736 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:57,059 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:57,374 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:57,374 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:57,375 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:57,375 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:57,807 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:57,808 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:57,808 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:57,808 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:57,939 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:13:57,940 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:13:58,061 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:58,241 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:58,242 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:58,242 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:58,242 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:58,647 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:58,647 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:58,647 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:58,648 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:59,042 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:59,043 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:59,043 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:59,043 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:59,066 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:13:59,444 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:59,445 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:59,445 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:59,445 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:13:59,840 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:13:59,840 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:13:59,840 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:13:59,840 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:00,068 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:00,238 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:00,238 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:00,239 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:00,239 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:00,627 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:00,627 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:00,627 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:00,627 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:01,032 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:01,032 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:01,033 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:01,033 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:01,033 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:01,070 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:01,430 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:01,431 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:01,431 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:01,431 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:01,833 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:01,833 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:01,833 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:01,834 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:02,074 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:02,254 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:02,254 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:02,254 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:02,255 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:02,665 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:02,665 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:02,666 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:02,666 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:03,053 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:03,053 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:03,053 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:03,053 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:03,079 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:03,488 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:03,488 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:03,488 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:03,488 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:03,897 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:03,898 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:03,898 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:03,898 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:04,085 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:04,322 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:04,323 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:04,323 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:04,323 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:04,768 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:04,769 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:04,769 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:04,769 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:05,089 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:05,175 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:05,175 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:05,175 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:05,175 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:05,578 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:05,579 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:05,579 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:05,579 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:06,060 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:06,060 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:06,060 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:06,061 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:06,061 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:06,093 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:06,541 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:06,541 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:06,542 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:06,542 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:07,099 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:07,123 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,124 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:07,124 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:07,124 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:07,295 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,295 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,308 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,332 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,337 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,338 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,348 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,370 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,375 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,376 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,388 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,410 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,418 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,418 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,439 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,476 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,495 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,495 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,568 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,599 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,619 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,629 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,638 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,644 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,658 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,664 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,678 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,680 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,754 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:07,758 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:07,761 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:07,763 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:08,103 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:08,229 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:08,230 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:08,230 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:08,230 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:08,735 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:08,735 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:08,735 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:08,736 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:09,108 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:09,188 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:09,188 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:09,188 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:09,188 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:09,640 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:09,640 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:09,640 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:09,641 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:10,111 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:10,181 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:10,181 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:10,181 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:10,181 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:10,691 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:10,691 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:10,692 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:10,692 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:11,114 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:11,151 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:11,152 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:11,152 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:11,152 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:11,152 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:11,561 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:11,562 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:11,562 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:11,563 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:11,991 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:11,991 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:11,992 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:11,993 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:12,118 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:12,420 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:12,420 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:12,420 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:12,422 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:12,886 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:12,886 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:12,887 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:12,887 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:12,924 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:14:12,937 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:14:12,938 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:14:13,120 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:13,281 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:13,282 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:13,282 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:13,282 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:13,697 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:13,697 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:13,697 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:13,698 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:14,126 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:14,175 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:14,175 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:14,175 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:14,176 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:14,605 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:14,606 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:14,606 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:14,606 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:15,023 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:15,023 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:15,023 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:15,024 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:15,130 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:15,423 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:15,423 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:15,423 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:15,423 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:15,817 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:15,817 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:15,818 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:15,818 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:16,132 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:16,213 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:16,213 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:16,214 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:16,214 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:16,214 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:16,618 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:16,618 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:16,618 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:16,618 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:17,014 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:17,015 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:17,015 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:17,015 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:17,135 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:17,423 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:17,424 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:17,424 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:17,424 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:17,815 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:17,816 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:17,816 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:17,816 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:18,139 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:18,217 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:18,217 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:18,217 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:18,217 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:18,613 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:18,614 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:18,614 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:18,614 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:19,013 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:19,013 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:19,013 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:19,014 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:19,141 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:19,409 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:19,410 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:19,410 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:19,411 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:19,821 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:19,821 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:19,822 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:19,822 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:20,146 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:20,218 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:20,218 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:20,218 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:20,218 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:20,611 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:20,612 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:20,612 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:20,612 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:21,019 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:21,019 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:21,019 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:21,019 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:21,149 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:21,432 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:21,433 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:21,433 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:21,433 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:21,434 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:21,830 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:21,830 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:21,830 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:21,831 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:22,154 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:22,242 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:22,243 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:22,244 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:22,244 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:22,703 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:22,704 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:22,704 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:22,704 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:23,157 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:23,190 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:23,190 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:23,190 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:23,190 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:23,669 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:23,669 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:23,669 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:23,670 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:24,083 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:24,083 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:24,083 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:24,083 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:24,162 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:24,512 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:24,513 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:24,513 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:24,514 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:25,124 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:25,125 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:25,125 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:25,125 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:25,166 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:25,543 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:25,543 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:25,543 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:25,544 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:26,069 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:26,069 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:26,069 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:26,069 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:26,169 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:26,559 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:26,559 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:26,559 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:26,559 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:26,560 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:27,026 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:27,026 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:27,026 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:27,027 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:27,174 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:27,515 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:27,516 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:27,516 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:27,517 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:27,953 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:14:27,955 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:14:28,030 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:28,119 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:28,120 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:28,120 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:28,176 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:28,512 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:28,513 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:28,513 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:28,513 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:28,925 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:28,925 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:28,925 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:28,926 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:29,176 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:29,354 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:29,355 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:29,355 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:29,355 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:29,762 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:29,763 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:29,763 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:29,763 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:30,169 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:30,169 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:30,169 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:30,169 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:30,181 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:30,593 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:30,593 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:30,593 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:30,594 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:31,090 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:31,091 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:31,091 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:31,091 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:31,185 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:31,545 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:31,546 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:31,546 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:31,546 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:31,988 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:31,988 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:31,988 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:31,988 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:31,989 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:32,188 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:32,419 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:32,419 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:32,419 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:32,420 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:32,829 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:32,829 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:32,829 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:32,829 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:33,190 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:33,242 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:33,242 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:33,242 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:33,242 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:33,634 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:33,634 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:33,635 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:33,635 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:34,010 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:34,011 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:34,011 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:34,011 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:34,192 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:34,404 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:34,404 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:34,405 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:34,405 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:34,849 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:34,849 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:34,849 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:34,850 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:35,193 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:35,263 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:35,264 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:35,264 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:35,264 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:35,661 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:35,661 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:35,661 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:35,662 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:36,055 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:36,055 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:36,056 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:36,056 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:36,196 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:36,453 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:36,454 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:36,454 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:36,454 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:36,864 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:36,865 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:36,865 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:36,865 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:37,200 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:37,295 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:37,295 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:37,296 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:37,296 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:37,296 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:37,821 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:37,822 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:37,822 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:37,822 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:38,203 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:38,267 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:38,267 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:38,267 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:38,267 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:38,681 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:38,681 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:38,681 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:38,681 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:39,106 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:39,106 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:39,106 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:39,107 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:39,208 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:39,517 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:39,517 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:39,517 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:39,518 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:39,946 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:39,947 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:39,947 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:39,947 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:40,209 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:40,389 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:40,389 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:40,389 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:40,390 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:40,849 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:40,849 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:40,849 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:40,849 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:41,212 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:41,269 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:41,270 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:41,270 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:41,270 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:41,750 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:41,751 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:41,751 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:41,751 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:42,167 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:42,167 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:42,167 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:42,168 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:42,212 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:42,636 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:42,637 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:42,637 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:42,637 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:42,637 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:42,929 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:14:42,941 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:14:42,941 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:14:43,090 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:43,091 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:43,092 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:43,092 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:43,217 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:43,499 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:43,499 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:43,499 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:43,499 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:43,912 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:43,912 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:43,912 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:43,913 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:44,222 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:44,350 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:44,351 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:44,351 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:44,351 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:44,788 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:44,788 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:44,788 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:44,789 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:45,223 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:45,227 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:45,228 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:45,228 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:45,228 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:45,660 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:45,660 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:45,661 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:45,661 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:46,091 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:46,091 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:46,091 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:46,092 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:46,224 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:46,518 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:46,518 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:46,519 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:46,519 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:46,945 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:46,946 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:46,946 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:46,946 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:47,228 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:47,381 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:47,381 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:47,381 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:47,381 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:47,827 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:47,828 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:47,828 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:47,828 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:47,828 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:48,233 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:48,277 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:48,277 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:48,277 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:48,277 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:48,727 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:48,728 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:48,728 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:48,728 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:49,132 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:49,132 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:49,132 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:49,132 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:49,238 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:49,541 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:49,541 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:49,542 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:49,542 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:49,951 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:49,952 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:49,952 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:49,952 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:50,240 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:50,374 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,374 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:50,375 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:50,375 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:50,793 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,793 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:50,794 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:50,794 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:50,938 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,938 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,952 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,975 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,980 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,980 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:50,990 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,007 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,012 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,013 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,023 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,040 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,045 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,045 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,062 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,095 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,110 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,110 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,162 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,186 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,202 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,212 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,221 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,226 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,235 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,240 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,244 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:51,255 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,257 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,326 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,330 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:51,332 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:51,334 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:51,730 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:51,730 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:51,730 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:51,730 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:52,141 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:52,141 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:52,141 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:52,142 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:52,246 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:52,549 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:52,549 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:52,549 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:52,550 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:52,977 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:52,978 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:52,978 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:52,978 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:52,978 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:53,246 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:53,380 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:53,381 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:53,381 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:53,381 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:53,786 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:53,787 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:53,787 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:53,787 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:54,192 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:54,192 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:54,193 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:54,193 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:54,248 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:54,607 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:54,607 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:54,607 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:54,608 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:55,018 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:55,018 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:55,019 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:55,019 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:55,254 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:55,445 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:55,446 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:55,446 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:55,446 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:55,860 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:55,861 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:55,861 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:55,861 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:56,260 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:56,272 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:56,273 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:56,273 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:56,273 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:56,686 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:56,686 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:56,686 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:56,686 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:57,090 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:57,091 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:57,091 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:57,091 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:57,265 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:57,501 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:57,501 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:57,501 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:57,502 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:57,928 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:57,928 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:57,928 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:57,929 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:57,945 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:14:57,946 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:14:58,096 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:14:58,268 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:58,354 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:58,354 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:58,354 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:58,354 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:58,748 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:58,748 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:58,748 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:58,749 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:59,159 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:59,159 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:59,159 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:59,160 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:59,273 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:14:59,570 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:59,571 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:59,571 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:59,571 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:14:59,981 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:14:59,981 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:14:59,981 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:14:59,981 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:00,279 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:00,403 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:00,404 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:00,404 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:00,404 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:00,833 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:00,834 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:00,834 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:00,834 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:01,230 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:01,230 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:01,230 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:01,231 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:01,284 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:01,644 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:01,644 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:01,644 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:01,644 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:02,062 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:02,062 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:02,063 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:02,063 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:02,288 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:02,476 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:02,476 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:02,476 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:02,476 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:02,904 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:02,904 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:02,904 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:02,905 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:03,292 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:03,319 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:03,320 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:03,320 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:03,320 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:03,320 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:03,730 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:03,731 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:03,731 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:03,731 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:04,128 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:04,129 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:04,129 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:04,129 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:04,294 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:04,526 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:04,527 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:04,527 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:04,527 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:04,934 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:04,934 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:04,934 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:04,935 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:05,298 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:05,386 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:05,387 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:05,387 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:05,387 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:05,830 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:05,830 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:05,830 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:05,831 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:06,262 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:06,262 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:06,262 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:06,263 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:06,300 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:06,676 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:06,676 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:06,676 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:06,676 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:07,089 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:07,090 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:07,090 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:07,090 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:07,301 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:07,503 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:07,503 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:07,503 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:07,503 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:07,923 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:07,923 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:07,923 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:07,924 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:08,307 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:08,347 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:08,348 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:08,348 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:08,348 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:08,348 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:08,761 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:08,761 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:08,761 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:08,762 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:09,172 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:09,172 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:09,173 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:09,173 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:09,308 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:09,577 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:09,578 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:09,578 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:09,579 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:09,976 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:09,976 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:09,976 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:09,976 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:10,309 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:10,396 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:10,397 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:10,397 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:10,398 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:10,817 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:10,817 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:10,817 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:10,818 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:11,219 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:11,220 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:11,220 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:11,220 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:11,315 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:11,623 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:11,624 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:11,624 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:11,624 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:12,077 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:12,078 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:12,078 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:12,079 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:12,316 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:12,526 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:12,526 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:12,526 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:12,527 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:12,931 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:15:12,949 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:15:12,949 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:15:13,068 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:13,094 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:13,095 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:13,096 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:13,321 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:13,564 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:13,564 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:13,565 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:13,565 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:13,565 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:13,985 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:13,985 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:13,985 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:13,986 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:14,326 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:14,442 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:14,442 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:14,442 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:14,442 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:14,878 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:14,878 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:14,878 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:14,879 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:15,303 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:15,304 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:15,304 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:15,304 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:15,331 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:15,725 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:15,725 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:15,725 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:15,725 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:16,138 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:16,138 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:16,138 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:16,139 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:16,334 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:16,543 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:16,544 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:16,544 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:16,544 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:16,948 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:16,949 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:16,949 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:16,949 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:17,341 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:17,357 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:17,358 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:17,358 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:17,358 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:17,761 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:17,761 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:17,761 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:17,762 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:18,235 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:18,236 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:18,236 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:18,236 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:18,342 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:18,692 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:18,693 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:18,693 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:18,693 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:18,693 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:19,346 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:19,389 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:19,390 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:19,390 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:19,390 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:19,798 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:19,798 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:19,798 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:19,799 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:20,213 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:20,213 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:20,213 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:20,213 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:20,347 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:20,635 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:20,636 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:20,636 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:20,636 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:21,060 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:21,060 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:21,060 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:21,060 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:21,349 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:21,461 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:21,461 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:21,462 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:21,462 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:21,846 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:21,847 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:21,847 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:21,847 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:22,238 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:22,239 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:22,239 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:22,239 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:22,354 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:22,629 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:22,630 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:22,630 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:22,630 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:23,038 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:23,039 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:23,039 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:23,040 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:23,358 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:23,455 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:23,455 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:23,456 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:23,456 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:23,843 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:23,843 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:23,843 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:23,843 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:23,844 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:24,226 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:24,226 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:24,226 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:24,226 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:24,362 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:24,616 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:24,617 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:24,617 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:24,617 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:25,014 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:25,014 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:25,014 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:25,014 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:25,367 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:25,465 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:25,466 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:25,466 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:25,466 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:25,893 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:25,893 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:25,893 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:25,893 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:26,304 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:26,305 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:26,305 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:26,305 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:26,372 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:26,699 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:26,699 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:26,700 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:26,700 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:27,111 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:27,112 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:27,112 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:27,112 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:27,374 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:27,517 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:27,517 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:27,517 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:27,517 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:27,909 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:27,910 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:27,910 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:27,911 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:27,949 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:15:27,950 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:15:28,319 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:28,319 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:28,319 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:28,319 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:28,374 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:28,736 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:28,736 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:28,736 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:28,736 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:29,139 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:29,139 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:29,139 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:29,139 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:29,139 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:29,378 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:29,521 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:29,522 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:29,522 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:29,522 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:29,915 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:29,915 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:29,915 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:29,916 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:30,290 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:30,290 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:30,290 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:30,290 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:30,380 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:30,711 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:30,712 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:30,712 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:30,712 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:31,120 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:31,121 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:31,121 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:31,121 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:31,386 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:31,508 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:31,509 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:31,509 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:31,509 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:31,906 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:31,906 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:31,907 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:31,907 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:32,307 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,307 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:32,307 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:32,307 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:32,389 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:32,692 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,693 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:32,693 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:32,693 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:32,842 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,842 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,858 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,880 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,886 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,886 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,898 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,914 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,919 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,919 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,930 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,946 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,952 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,953 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:32,970 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,005 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,024 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,024 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,077 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,105 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,124 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,133 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,142 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,147 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,159 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,166 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,184 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,186 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,257 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,262 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:33,264 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:33,266 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:33,391 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:33,674 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:33,675 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:33,675 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:33,675 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:34,073 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:34,074 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:34,074 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:34,074 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:34,392 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:34,480 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:34,480 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:34,480 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:34,480 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:34,480 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:34,869 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:34,869 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:34,870 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:34,870 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:35,268 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:35,268 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:35,268 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:35,268 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:35,397 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:35,689 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:35,690 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:35,690 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:35,691 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:36,092 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:36,092 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:36,092 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:36,093 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:36,403 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:36,483 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:36,483 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:36,483 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:36,484 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:36,869 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:36,869 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:36,869 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:36,870 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:37,256 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:37,257 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:37,257 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:37,257 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:37,404 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:37,648 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:37,648 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:37,648 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:37,648 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:38,041 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:38,041 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:38,041 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:38,042 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:38,411 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:38,455 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:38,455 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:38,455 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:38,455 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:38,860 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:38,860 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:38,860 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:38,861 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:39,247 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:39,248 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:39,248 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:39,248 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:39,412 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:39,652 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:39,653 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:39,653 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:39,653 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:39,653 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:40,044 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:40,044 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:40,044 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:40,045 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:40,416 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:40,456 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:40,456 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:40,456 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:40,457 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:40,877 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:40,878 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:40,878 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:40,878 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:41,272 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:41,272 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:41,272 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:41,272 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:41,419 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:41,662 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:41,663 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:41,663 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:41,663 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:42,052 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:42,052 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:42,052 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:42,053 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:42,423 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:42,477 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:42,477 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:42,477 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:42,477 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:42,866 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:42,867 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:42,867 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:42,867 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:42,935 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:15:42,950 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:15:42,951 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:15:43,281 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:43,282 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:43,282 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:43,282 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:43,427 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:43,698 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:43,698 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:43,698 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:43,698 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:44,095 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:44,096 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:44,096 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:44,096 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:44,432 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:44,481 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:44,482 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:44,482 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:44,482 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:44,883 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:44,883 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:44,883 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:44,883 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:44,884 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:45,267 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:45,267 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:45,267 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:45,268 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:45,437 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:45,670 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:45,670 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:45,670 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:45,671 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:46,068 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:46,069 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:46,069 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:46,069 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:46,440 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:46,461 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:46,462 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:46,462 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:46,462 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:46,851 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:46,851 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:46,851 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:46,851 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:47,246 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:47,247 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:47,247 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:47,247 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:47,445 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:47,628 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:47,629 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:47,629 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:47,629 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:48,020 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:48,021 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:48,021 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:48,021 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:48,436 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:48,436 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:48,436 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:48,437 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:48,449 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:48,815 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:48,815 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:48,815 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:48,816 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:49,213 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:49,213 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:49,213 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:49,213 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:49,454 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:49,610 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:49,611 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:49,611 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:49,611 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:50,016 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:50,016 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:50,016 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:50,017 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:50,017 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:50,415 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:50,415 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:50,416 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:50,416 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:50,460 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:50,829 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:50,830 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:50,830 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:50,830 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:51,234 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:51,235 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:51,235 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:51,235 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:51,462 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:51,626 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:51,627 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:51,627 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:51,627 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:52,015 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:52,016 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:52,016 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:52,016 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:52,410 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:52,410 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:52,410 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:52,410 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:52,467 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:52,849 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:52,850 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:52,850 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:52,850 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:53,263 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:53,264 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:53,264 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:53,264 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:53,472 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:53,670 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:53,671 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:53,671 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:53,671 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:54,077 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:54,077 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:54,077 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:54,077 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:54,466 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:54,466 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:54,466 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:54,466 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:54,475 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:54,854 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:54,854 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:54,854 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:54,854 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:55,244 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:55,244 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:55,244 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:55,245 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:15:55,245 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:55,481 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:55,646 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:55,647 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:55,647 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:55,648 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:56,051 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:56,052 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:56,052 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:56,052 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:56,441 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:56,442 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:56,442 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:56,442 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:56,482 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:56,837 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:56,837 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:56,837 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:56,838 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:57,242 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:57,242 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:57,242 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:57,243 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:57,483 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:57,659 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:57,659 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:57,659 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:57,660 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:57,954 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:15:57,954 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:15:58,060 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:58,101 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:58,101 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:58,102 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:58,488 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:58,635 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:58,636 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:58,636 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:58,636 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:59,152 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:59,152 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:59,153 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:59,153 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:59,487 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:15:59,567 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:59,567 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:59,567 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:59,568 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:15:59,992 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:15:59,992 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:15:59,992 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:15:59,992 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:00,418 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:00,418 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:00,418 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:00,419 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:00,419 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:00,493 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:00,886 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:00,887 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:00,887 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:00,887 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:01,349 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:01,349 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:01,349 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:01,350 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:01,496 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:01,776 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:01,776 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:01,776 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:01,776 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:02,191 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:02,191 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:02,191 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:02,192 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:02,498 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:02,645 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:02,646 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:02,646 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:02,646 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:03,288 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:03,289 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:03,290 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:03,291 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:03,506 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:03,966 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:03,966 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:03,966 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:03,967 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:04,480 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:04,480 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:04,480 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:04,481 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:04,512 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:04,981 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:04,982 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:04,982 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:04,982 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:05,465 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:05,466 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:05,466 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:05,466 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:05,466 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:05,511 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:05,887 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:05,888 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:05,888 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:05,889 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:06,318 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:06,319 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:06,319 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:06,319 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:06,516 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:06,789 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:06,790 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:06,790 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:06,790 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:07,386 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:07,389 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:07,389 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:07,398 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:07,521 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:07,893 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:07,893 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:07,893 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:07,893 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:08,351 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:08,352 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:08,352 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:08,352 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:08,525 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:08,865 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:08,866 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:08,866 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:08,866 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:09,438 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:09,439 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:09,439 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:09,440 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:09,527 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:09,916 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:09,916 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:09,916 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:09,916 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:10,315 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:10,315 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:10,315 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:10,315 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:10,529 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:10,718 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:10,718 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:10,718 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:10,719 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:10,719 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:11,104 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:11,105 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:11,105 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:11,106 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:11,521 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:11,521 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:11,521 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:11,522 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:11,531 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:11,927 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:11,927 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:11,927 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:11,927 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:12,319 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:12,319 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:12,319 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:12,319 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:12,536 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:12,718 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:12,719 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:12,719 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:12,719 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:12,939 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:16:12,957 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:16:12,958 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:16:13,113 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:13,113 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:13,113 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:13,113 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:13,511 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:13,511 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:13,511 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:13,511 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:13,537 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:13,923 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:13,924 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:13,924 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:13,924 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:14,319 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:14,319 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:14,319 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:14,319 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:14,540 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:14,719 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:14,719 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:14,719 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:14,720 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:15,208 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,211 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:15,212 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:15,213 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:15,379 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,379 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,393 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,419 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,423 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,423 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,433 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,452 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,456 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,456 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,465 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,482 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,488 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,488 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,502 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,537 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,544 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:15,554 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,555 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,610 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,634 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,651 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,659 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,667 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,673 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,685 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,690 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,704 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,706 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,780 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:15,784 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:15,786 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:15,786 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:15,788 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:16,194 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:16,195 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:16,195 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:16,196 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:16,550 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:16,741 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:16,742 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:16,742 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:16,742 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:17,193 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:17,193 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:17,194 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:17,194 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:17,552 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:17,638 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:17,638 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:17,638 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:17,639 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:18,054 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:18,055 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:18,055 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:18,055 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:18,522 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:18,523 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:18,523 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:18,523 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:18,557 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:18,942 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:18,942 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:18,943 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:18,943 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:19,432 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:19,432 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:19,432 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:19,433 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:19,565 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:19,919 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:19,920 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:19,920 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:19,920 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:20,386 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:20,386 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:20,386 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:20,386 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:20,569 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:20,835 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:20,835 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:20,835 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:20,835 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:20,837 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:21,240 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:21,241 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:21,241 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:21,241 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:21,574 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:21,659 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:21,659 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:21,659 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:21,659 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:22,055 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:22,055 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:22,055 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:22,055 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:22,461 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:22,462 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:22,462 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:22,462 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:22,575 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:22,861 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:22,862 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:22,862 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:22,862 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:23,244 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:23,245 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:23,245 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:23,245 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:23,581 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:23,621 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:23,622 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:23,622 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:23,622 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:24,025 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:24,026 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:24,026 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:24,026 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:24,429 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:24,429 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:24,430 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:24,430 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:24,583 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:24,827 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:24,827 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:24,827 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:24,827 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:25,208 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:25,208 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:25,208 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:25,209 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:25,589 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:25,611 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:25,612 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:25,612 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:25,612 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:25,993 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:25,994 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:25,994 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:25,994 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:25,994 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:26,387 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:26,388 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:26,388 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:26,388 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:26,593 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:26,865 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:26,865 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:26,865 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:26,865 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:27,391 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:27,392 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:27,392 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:27,392 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:27,595 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:27,833 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:27,833 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:27,833 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:27,834 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:27,958 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:16:27,959 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:16:28,327 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:28,328 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:28,328 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:28,329 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:28,600 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:28,779 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:28,779 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:28,780 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:28,780 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:29,211 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:29,212 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:29,212 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:29,212 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:29,604 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:29,618 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:29,619 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:29,619 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:29,619 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:30,019 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:30,019 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:30,019 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:30,019 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:30,405 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:30,406 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:30,406 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:30,407 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:30,610 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:30,873 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:30,873 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:30,873 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:30,874 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:31,281 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:31,282 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:31,282 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:31,282 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:31,282 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:31,615 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:31,699 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:31,699 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:31,699 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:31,699 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:32,258 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:32,258 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:32,258 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:32,259 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:32,620 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:32,720 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:32,720 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:32,720 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:32,721 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:33,110 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:33,110 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:33,110 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:33,110 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:33,500 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:33,500 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:33,500 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:33,501 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:33,621 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:33,930 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:33,930 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:33,931 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:33,931 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:34,455 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:34,456 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:34,456 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:34,456 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:34,627 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:34,899 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:34,899 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:34,899 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:34,900 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:35,356 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:35,357 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:35,357 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:35,357 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:35,632 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:35,797 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:35,798 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:35,798 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:35,798 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:36,199 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:36,199 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:36,199 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:36,199 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:36,618 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:36,619 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:36,619 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:36,619 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:36,619 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:36,634 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:37,031 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:37,031 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:37,031 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:37,032 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:37,419 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:37,419 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:37,419 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:37,419 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:37,635 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:37,821 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:37,821 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:37,821 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:37,821 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:38,228 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:38,228 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:38,228 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:38,228 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:38,637 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:38,642 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:38,643 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:38,643 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:38,643 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:39,042 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:39,042 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:39,043 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:39,043 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:39,463 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:39,463 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:39,463 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:39,463 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:39,638 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:39,861 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:39,861 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:39,861 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:39,862 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:40,256 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:40,257 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:40,257 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:40,257 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:40,641 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:40,659 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:40,660 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:40,660 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:40,660 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:41,041 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:41,041 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:41,041 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:41,042 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:41,439 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:41,439 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:41,439 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:41,440 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:41,643 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:41,860 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:41,860 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:41,860 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:41,860 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:41,861 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:42,257 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:42,258 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:42,258 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:42,258 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:42,646 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:42,660 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:42,660 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:42,660 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:42,661 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:42,943 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:16:42,962 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:16:42,963 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:16:43,042 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:43,138 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:43,139 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:43,139 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:43,438 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:43,438 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:43,438 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:43,438 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:43,650 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:43,832 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:43,832 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:43,832 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:43,832 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:44,235 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:44,236 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:44,236 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:44,236 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:44,630 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:44,631 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:44,631 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:44,631 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:44,654 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:45,022 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:45,022 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:45,022 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:45,023 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:45,415 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:45,415 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:45,415 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:45,416 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:45,658 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:45,824 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:45,824 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:45,824 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:45,825 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:46,210 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:46,210 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:46,210 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:46,210 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:46,624 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:46,625 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:46,625 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:46,626 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:46,659 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:47,022 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:47,022 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:47,022 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:47,023 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:47,023 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:47,437 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:47,438 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:47,438 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:47,439 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:47,665 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:47,888 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:47,888 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:47,888 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:47,888 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:48,275 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:48,275 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:48,275 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:48,275 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:48,668 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:48,686 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:48,686 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:48,686 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:48,686 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:49,091 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:49,091 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:49,091 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:49,092 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:49,631 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:49,631 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:49,631 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:49,632 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:49,672 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:50,101 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:50,102 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:50,102 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:50,102 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:50,555 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:50,555 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:50,555 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:50,556 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:50,677 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:50,989 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:50,990 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:50,990 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:50,990 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:51,413 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:51,413 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:51,414 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:51,414 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:51,679 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:51,830 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:51,830 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:51,830 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:51,830 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:52,229 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:52,229 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:52,229 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:52,229 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:52,229 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:52,620 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:52,620 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:52,620 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:52,620 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:52,684 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:53,050 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:53,050 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:53,050 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:53,051 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:53,439 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:53,439 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:53,439 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:53,440 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:53,689 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:53,832 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:53,832 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:53,832 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:53,833 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:54,316 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:54,316 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:54,317 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:54,317 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:54,695 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:54,779 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:54,779 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:54,779 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:54,780 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:55,201 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:55,202 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:55,202 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:55,202 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:55,604 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:55,604 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:55,605 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:55,605 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:55,700 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:56,001 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:56,002 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:56,002 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:56,002 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:56,509 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:56,510 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:56,510 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:56,510 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:56,706 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:57,023 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:57,023 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:57,023 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:57,023 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:57,539 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:57,540 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:57,540 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:57,540 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:16:57,540 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:57,709 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:57,968 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:16:57,968 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:16:58,094 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,127 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:58,128 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:58,129 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:58,237 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,238 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,251 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,275 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,279 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,280 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,289 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,304 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,308 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,308 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,316 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,330 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,335 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,335 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,352 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,381 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,392 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,392 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,443 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,466 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,484 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,492 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,501 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,506 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,518 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,524 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,539 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,542 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,629 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:58,634 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:58,638 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:58,641 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:58,715 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:16:59,182 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:59,183 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:59,183 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:59,183 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:59,610 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:16:59,611 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:16:59,611 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:16:59,611 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:16:59,720 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:00,029 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:00,030 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:00,030 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:00,030 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:00,524 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:00,524 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:00,524 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:00,524 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:00,725 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:00,949 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:00,949 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:00,949 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:00,950 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:01,351 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:01,351 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:01,351 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:01,352 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:01,731 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:01,760 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:01,761 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:01,761 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:01,762 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:02,186 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:02,186 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:02,186 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:02,186 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:02,581 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:02,581 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:02,581 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:02,581 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:02,581 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:02,732 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:02,968 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:02,968 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:02,968 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:02,969 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:03,365 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:03,366 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:03,366 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:03,366 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:03,736 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:03,762 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:03,763 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:03,763 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:03,763 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:04,142 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:04,142 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:04,142 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:04,142 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:04,553 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:04,553 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:04,553 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:04,554 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:04,739 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:04,959 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:04,960 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:04,960 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:04,960 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:05,352 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:05,352 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:05,352 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:05,352 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:05,745 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:05,779 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:05,780 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:05,780 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:05,780 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:06,174 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:06,174 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:06,174 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:06,175 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:06,636 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:06,637 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:06,637 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:06,638 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:06,751 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:07,154 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:07,155 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:07,155 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:07,156 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:07,572 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:07,573 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:07,573 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:07,573 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:07,754 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:07,971 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:07,971 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:07,971 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:07,971 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:07,972 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:08,371 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:08,371 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:08,372 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:08,372 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:08,755 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:08,780 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:08,781 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:08,781 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:08,781 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:09,165 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:09,166 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:09,166 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:09,166 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:09,569 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:09,570 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:09,570 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:09,570 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:09,760 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:09,983 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:09,983 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:09,983 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:09,984 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:10,378 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:10,379 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:10,379 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:10,379 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:10,761 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:10,768 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:10,768 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:10,768 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:10,769 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:11,152 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:11,153 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:11,153 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:11,153 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:11,543 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:11,543 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:11,543 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:11,543 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:11,768 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:11,948 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:11,949 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:11,949 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:11,950 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:12,355 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:12,355 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:12,356 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:12,356 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:12,749 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:12,750 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:12,750 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:12,750 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:12,771 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:12,944 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:17:12,972 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:17:12,972 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:17:13,119 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:13,159 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:13,159 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:13,160 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:13,160 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:13,550 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:13,550 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:13,550 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:13,550 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:13,775 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:13,933 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:13,934 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:13,934 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:13,934 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:14,333 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:14,334 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:14,334 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:14,334 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:14,750 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:14,750 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:14,750 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:14,751 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:14,776 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:15,149 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:15,149 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:15,149 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:15,149 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:15,544 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:15,544 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:15,544 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:15,545 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:15,779 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:15,943 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:15,943 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:15,943 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:15,944 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:16,332 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:16,332 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:16,333 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:16,333 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:16,716 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:16,716 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:16,716 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:16,717 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:16,785 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:17,123 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:17,123 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:17,123 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:17,124 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:17,514 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:17,514 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:17,514 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:17,514 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:17,785 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:17,909 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:17,910 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:17,910 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:17,910 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:18,300 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:18,301 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:18,301 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:18,301 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:18,301 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:18,694 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:18,695 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:18,695 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:18,695 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:18,787 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:19,093 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:19,093 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:19,093 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:19,094 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:19,486 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:19,487 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:19,487 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:19,487 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:19,791 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:19,880 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:19,880 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:19,880 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:19,880 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:20,267 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:20,267 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:20,267 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:20,267 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:20,658 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:20,659 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:20,659 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:20,659 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:20,795 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:21,047 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:21,047 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:21,047 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:21,047 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:21,443 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:21,444 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:21,444 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:21,444 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:21,801 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:21,852 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:21,852 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:21,852 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:21,852 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:22,273 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:22,274 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:22,274 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:22,275 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:22,683 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:22,683 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:22,683 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:22,684 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:22,802 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:23,083 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:23,083 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:23,083 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:23,084 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:23,480 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:23,480 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:23,480 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:23,480 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:23,481 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:23,804 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:23,864 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:23,865 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:23,865 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:23,865 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:24,242 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:24,243 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:24,243 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:24,243 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:24,653 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:24,654 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:24,654 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:24,654 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:24,807 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:25,049 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:25,049 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:25,050 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:25,050 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:25,436 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:25,437 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:25,437 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:25,437 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:25,813 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:25,832 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:25,832 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:25,832 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:25,833 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:26,217 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:26,218 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:26,218 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:26,218 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:26,607 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:26,607 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:26,607 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:26,608 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:26,817 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:27,015 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:27,015 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:27,016 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:27,016 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:27,420 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:27,420 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:27,420 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:27,420 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:27,812 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:27,812 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:27,812 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:27,813 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:27,819 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:27,975 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:17:27,976 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:17:28,316 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:28,316 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:28,317 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:28,317 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:28,801 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:28,801 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:28,801 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:28,801 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:28,802 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:28,822 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:29,251 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:29,253 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:29,253 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:29,254 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:29,751 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:29,751 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:29,751 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:29,751 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:29,829 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:30,210 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:30,211 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:30,211 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:30,211 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:30,644 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:30,645 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:30,645 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:30,645 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:30,833 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:31,091 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:31,091 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:31,091 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:31,092 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:31,540 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:31,540 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:31,540 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:31,540 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:31,837 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:31,998 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:31,999 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:31,999 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:32,000 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:32,416 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:32,416 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:32,416 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:32,417 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:32,827 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:32,827 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:32,827 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:32,827 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:32,839 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:33,231 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:33,232 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:33,232 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:33,232 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:33,669 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:33,670 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:33,670 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:33,670 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:33,843 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:34,078 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:34,079 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:34,079 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:34,079 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:34,079 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:34,554 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:34,555 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:34,555 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:34,556 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:34,848 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:35,065 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:35,065 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:35,065 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:35,066 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:35,463 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:35,464 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:35,464 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:35,464 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:35,850 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:35,873 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:35,873 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:35,873 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:35,873 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:36,537 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:36,538 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:36,538 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:36,539 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:36,853 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:36,944 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:36,945 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:36,945 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:36,945 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:37,362 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:37,363 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:37,363 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:37,363 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:37,823 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:37,824 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:37,824 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:37,824 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:37,857 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:38,227 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:38,228 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:38,228 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:38,228 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:38,615 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:38,615 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:38,615 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:38,615 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:38,859 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:39,017 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,017 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:39,017 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:39,017 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:39,398 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,399 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:39,399 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:39,399 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:39,399 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:39,818 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,818 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:39,818 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:39,819 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:39,860 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:39,955 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,955 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,970 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,994 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,999 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:39,999 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,010 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,026 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,032 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,032 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,043 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,059 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,066 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,066 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,084 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,117 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,131 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,131 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,183 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,210 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,227 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,236 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,247 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,251 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,262 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,268 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,284 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,286 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,355 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,359 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:40,361 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:40,362 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:40,771 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:40,772 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:40,772 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:40,772 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:40,866 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:41,171 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:41,172 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:41,172 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:41,172 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:41,564 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:41,564 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:41,564 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:41,564 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:41,869 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:41,961 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:41,961 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:41,961 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:41,961 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:42,366 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:42,366 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:42,366 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:42,367 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:42,779 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:42,779 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:42,779 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:42,779 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:42,873 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:42,947 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:17:42,977 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:17:42,977 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:17:43,158 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:43,158 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:43,158 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:43,158 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:43,541 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:43,541 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:43,541 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:43,542 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:43,874 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:43,938 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:43,938 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:43,938 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:43,938 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:44,331 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:44,331 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:44,331 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:44,331 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:44,741 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:44,742 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:44,742 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:44,742 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:44,742 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:44,878 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:45,146 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:45,146 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:45,146 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:45,146 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:45,544 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:45,544 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:45,545 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:45,545 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:45,881 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:45,966 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:45,967 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:45,967 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:45,967 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:46,386 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:46,386 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:46,386 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:46,387 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:46,787 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:46,787 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:46,787 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:46,787 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:46,885 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:47,195 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:47,196 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:47,196 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:47,196 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:47,614 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:47,614 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:47,615 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:47,615 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:47,885 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:47,998 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:47,999 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:47,999 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:47,999 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:48,399 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:48,400 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:48,400 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:48,400 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:48,787 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:48,788 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:48,788 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:48,788 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:48,890 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:49,177 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:49,178 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:49,178 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:49,178 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:49,570 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:49,570 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:49,570 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:49,570 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:49,896 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:49,986 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:49,987 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:49,987 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:49,987 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:49,987 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:50,394 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:50,394 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:50,394 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:50,395 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:50,785 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:50,785 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:50,786 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:50,786 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:50,899 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:51,186 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:51,187 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:51,187 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:51,187 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:51,590 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:51,590 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:51,590 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:51,591 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:51,904 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:51,993 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:51,993 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:51,994 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:51,994 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:52,417 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:52,418 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:52,418 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:52,419 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:52,823 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:52,823 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:52,823 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:52,824 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:52,910 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:53,224 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:53,225 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:53,225 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:53,225 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:53,623 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:53,624 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:53,624 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:53,624 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:53,915 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:54,018 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:54,019 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:54,019 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:54,019 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:54,404 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:54,405 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:54,405 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:54,405 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:54,808 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:54,808 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:54,809 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:54,809 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:54,920 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:55,214 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:55,214 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:55,214 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:55,214 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:17:55,215 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:55,613 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:55,614 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:55,614 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:55,614 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:55,923 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:56,220 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:56,220 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:56,220 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:56,221 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:56,614 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:56,614 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:56,614 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:56,614 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:56,924 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:57,070 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:57,070 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:57,070 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:57,071 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:57,486 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:57,486 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:57,487 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:57,487 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:57,918 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:57,918 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:57,918 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:57,918 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:57,926 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:57,983 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:17:57,983 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:17:58,309 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:58,310 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:58,310 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:58,310 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:58,756 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:58,756 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:58,756 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:58,756 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:58,930 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:59,166 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:59,167 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:59,167 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:59,167 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:59,576 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:59,578 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:59,578 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:59,578 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:17:59,936 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:17:59,981 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:17:59,982 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:17:59,982 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:17:59,982 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:00,388 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:00,388 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:00,388 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:00,388 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:00,389 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:00,777 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:00,777 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:00,778 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:00,778 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:00,936 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:01,164 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:01,164 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:01,164 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:01,165 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:01,546 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:01,546 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:01,547 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:01,547 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:01,940 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:01,942 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:01,942 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:01,943 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:01,943 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:02,349 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:02,350 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:02,350 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:02,350 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:02,894 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:02,895 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:02,895 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:02,896 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:02,945 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:03,405 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:03,405 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:03,406 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:03,406 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:03,838 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:03,839 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:03,839 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:03,839 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:03,947 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:04,313 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:04,314 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:04,314 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:04,314 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:04,890 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:04,891 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:04,891 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:04,892 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:04,950 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:05,484 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:05,484 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:05,484 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:05,484 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:05,484 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:05,880 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:05,881 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:05,881 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:05,881 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:05,950 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:06,276 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:06,277 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:06,277 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:06,277 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:06,666 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:06,667 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:06,667 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:06,667 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:06,952 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:07,090 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:07,090 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:07,090 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:07,091 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:07,489 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:07,490 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:07,490 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:07,490 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:07,884 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:07,884 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:07,884 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:07,884 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:07,953 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:08,279 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:08,279 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:08,279 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:08,279 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:08,691 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:08,691 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:08,691 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:08,691 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:08,955 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:09,076 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:09,076 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:09,076 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:09,077 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:09,459 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:09,459 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:09,460 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:09,460 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:09,961 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:10,114 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:10,115 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:10,115 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:10,115 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:10,574 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:10,575 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:10,575 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:10,575 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:10,575 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:10,962 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:10,984 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:10,984 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:10,984 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:10,985 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:11,390 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:11,390 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:11,390 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:11,390 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:11,802 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:11,802 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:11,802 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:11,803 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:11,962 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:12,195 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:12,195 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:12,195 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:12,195 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:12,601 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:12,601 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:12,602 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:12,602 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:12,948 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:18:12,968 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:12,983 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:18:12,984 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:18:13,011 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:13,130 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:13,131 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:13,132 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:13,408 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:13,409 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:13,409 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:13,409 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:13,799 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:13,800 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:13,800 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:13,800 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:13,974 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:14,189 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:14,189 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:14,189 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:14,189 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:14,579 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:14,579 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:14,579 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:14,579 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:14,970 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:14,971 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:14,971 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:14,971 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:14,976 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:15,372 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:15,373 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:15,373 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:15,373 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:15,785 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:15,786 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:15,786 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:15,786 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:15,786 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:15,981 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:16,188 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:16,188 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:16,189 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:16,189 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:16,580 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:16,580 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:16,580 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:16,581 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:16,971 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:16,972 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:16,972 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:16,972 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:16,984 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:17,370 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:17,370 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:17,370 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:17,370 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:17,779 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:17,779 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:17,779 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:17,780 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:17,988 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:18,178 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:18,178 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:18,178 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:18,179 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:18,565 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:18,566 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:18,566 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:18,566 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:18,963 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:18,964 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:18,964 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:18,964 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:18,988 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:19,358 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:19,358 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:19,358 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:19,358 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:19,749 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:19,749 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:19,749 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:19,750 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:19,990 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:20,151 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:20,151 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:20,152 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:20,152 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:20,561 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:20,562 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:20,562 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:20,562 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:20,956 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:20,956 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:20,956 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:20,956 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:20,957 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:20,991 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:21,353 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,354 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:21,354 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:21,354 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:21,487 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,487 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,501 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,524 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,527 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,528 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,539 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,553 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,556 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,557 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,565 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,580 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,584 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,585 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,599 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,627 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,642 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,642 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,693 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,718 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,734 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,743 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,751 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,756 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,768 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,772 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,786 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,787 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,857 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:21,861 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:21,863 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:21,865 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:21,996 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:22,236 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:22,236 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:22,237 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:22,237 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:22,645 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:22,646 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:22,646 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:22,646 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:23,004 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:23,051 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:23,052 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:23,052 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:23,052 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:23,451 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:23,451 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:23,451 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:23,451 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:23,855 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:23,856 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:23,856 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:23,856 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:24,009 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:24,262 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:24,263 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:24,263 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:24,263 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:24,662 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:24,663 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:24,663 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:24,663 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:25,015 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:25,070 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:25,071 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:25,071 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:25,071 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:25,488 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:25,488 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:25,488 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:25,489 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:25,897 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:25,897 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:25,897 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:25,898 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:26,017 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:26,296 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:26,297 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:26,297 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:26,297 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:26,297 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:26,685 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:26,686 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:26,686 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:26,686 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:27,022 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:27,078 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:27,079 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:27,079 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:27,079 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:27,484 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:27,485 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:27,485 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:27,485 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:27,896 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:27,896 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:27,896 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:27,896 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:27,987 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:18:27,987 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:18:28,028 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:28,305 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:28,306 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:28,306 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:28,306 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:28,707 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:28,707 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:28,708 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:28,708 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:29,029 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:29,102 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:29,102 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:29,102 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:29,102 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:29,504 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:29,504 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:29,504 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:29,505 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:29,913 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:29,913 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:29,913 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:29,913 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:30,033 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:30,325 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:30,326 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:30,326 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:30,326 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:30,740 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:30,741 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:30,741 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:30,741 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:31,033 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:31,137 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:31,138 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:31,138 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:31,138 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:31,529 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:31,529 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:31,529 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:31,529 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:31,530 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:31,920 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:31,920 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:31,920 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:31,920 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:32,038 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:32,301 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:32,302 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:32,302 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:32,302 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:32,705 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:32,706 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:32,706 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:32,707 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:33,040 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:33,104 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:33,105 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:33,105 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:33,106 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:33,495 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:33,496 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:33,496 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:33,496 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:33,891 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:33,892 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:33,892 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:33,892 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:34,043 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:34,290 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:34,290 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:34,290 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:34,290 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:34,676 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:34,677 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:34,677 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:34,677 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:35,051 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:35,086 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:35,087 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:35,087 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:35,087 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:35,495 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:35,495 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:35,495 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:35,495 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:35,883 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:35,884 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:35,884 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:35,884 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:36,054 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:36,280 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:36,281 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:36,281 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:36,281 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:36,677 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:36,678 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:36,678 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:36,678 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:36,678 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:37,061 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:37,076 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:37,076 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:37,076 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:37,077 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:37,472 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:37,473 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:37,473 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:37,473 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:37,894 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:37,894 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:37,894 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:37,894 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:38,063 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:38,311 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:38,311 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:38,311 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:38,312 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:38,950 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:38,951 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:38,951 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:38,951 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:39,069 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:39,482 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:39,483 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:39,483 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:39,483 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:39,933 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:39,933 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:39,933 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:39,934 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:40,073 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:40,409 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:40,410 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:40,410 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:40,410 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:40,807 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:40,807 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:40,807 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:40,808 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:41,079 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:41,249 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:41,250 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:41,250 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:41,250 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:41,641 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:41,641 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:41,641 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:41,641 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:42,034 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:42,034 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:42,034 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:42,034 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:42,034 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:42,082 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:42,626 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:42,627 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:42,627 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:42,628 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:42,952 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:18:42,989 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: stop_status
-2023-06-02 14:18:42,990 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: stop_status
-2023-06-02 14:18:43,078 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:43,087 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:43,149 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:43,149 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:43,150 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:43,492 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:43,493 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:43,493 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:43,493 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:43,899 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:43,899 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:43,899 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:43,899 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:44,090 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:44,309 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:44,309 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:44,309 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:44,310 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:44,699 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:44,700 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:44,700 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:44,700 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:45,085 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:45,085 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:45,085 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:45,085 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:45,094 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:45,486 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:45,487 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:45,487 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:45,488 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:45,886 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:45,886 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:45,886 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:45,887 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:46,098 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:46,270 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:46,271 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:46,271 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:46,271 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:46,670 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:46,670 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:46,670 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:46,671 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:47,074 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:47,075 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:47,075 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:47,075 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:47,075 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:47,104 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:47,475 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:47,476 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:47,476 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:47,476 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:47,894 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:47,895 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:47,895 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:47,895 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:48,105 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:48,319 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:48,320 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:48,320 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:48,320 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:48,711 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:48,711 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:48,711 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:48,711 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:49,108 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:49,118 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:49,118 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:49,119 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:49,119 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:49,520 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:49,521 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:49,521 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:49,521 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:49,910 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:49,910 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:49,910 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:49,910 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:50,112 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:50,320 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:50,321 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:50,321 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:50,321 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:50,722 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:50,722 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:50,722 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:50,722 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:51,117 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:51,135 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:51,135 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:51,135 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:51,135 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:51,543 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:51,543 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:51,543 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:51,543 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:51,953 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:51,953 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:51,954 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:51,954 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:52,122 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:52,363 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:52,364 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:52,364 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:52,364 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:52,364 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:52,781 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:52,782 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:52,782 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:52,782 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:53,124 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:53,195 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:53,196 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:53,196 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:53,196 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:53,594 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:53,594 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:53,594 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:53,594 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:54,061 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:54,061 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:54,061 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:54,061 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:54,128 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:54,462 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:54,462 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:54,462 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:54,462 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:54,861 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:54,862 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:54,862 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:54,862 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:55,129 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:55,271 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:55,272 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:55,272 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:55,272 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:55,686 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:55,686 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:55,686 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:55,686 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:56,089 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: partial_history
-2023-06-02 14:18:56,089 DEBUG SenderThread:1509 [sender.py:send():375] send: history
-2023-06-02 14:18:56,089 DEBUG SenderThread:1509 [sender.py:send_request():402] send_request: summary_record
-2023-06-02 14:18:56,089 INFO SenderThread:1509 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
-2023-06-02 14:18:56,130 INFO Thread-12 :1509 [dir_watcher.py:_on_file_modified():295] file/dir modified: /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/files/wandb-summary.json
-2023-06-02 14:18:57,564 INFO memory :1509 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited.
-2023-06-02 14:18:57,565 DEBUG SystemMonitor:1509 [system_monitor.py:_start():166] Finished system metrics aggregation loop
-2023-06-02 14:18:57,565 DEBUG SystemMonitor:1509 [system_monitor.py:_start():170] Publishing last batch of metrics
-2023-06-02 14:18:57,566 DEBUG SenderThread:1509 [sender.py:send():375] send: stats
-2023-06-02 14:18:57,566 DEBUG HandlerThread:1509 [handler.py:handle_request():144] handle_request: status_report
-2023-06-02 14:18:58,149 INFO MainThread:1509 [internal.py:handle_exit():76] Internal process exited
diff --git a/wandb/run-20230602_141242-akua37dh/logs/debug.log b/wandb/run-20230602_141242-akua37dh/logs/debug.log
deleted file mode 100644
index b4872e7c3e20cb9d005a9f7d202bae4b46a9a81b..0000000000000000000000000000000000000000
--- a/wandb/run-20230602_141242-akua37dh/logs/debug.log
+++ /dev/null
@@ -1,33 +0,0 @@
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Current SDK version is 0.15.1
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Configure stats pid to 1498
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Loading settings from /Users/gonzalo/.config/wandb/settings
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Loading settings from /Users/gonzalo/git/diffusion_project/wandb/settings
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'main.py', 'program': '/Users/gonzalo/git/diffusion_project/main.py'}
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_init.py:_log_setup():507] Logging user logs to /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/logs/debug.log
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_init.py:_log_setup():508] Logging internal logs to /Users/gonzalo/git/diffusion_project/wandb/run-20230602_141242-akua37dh/logs/debug-internal.log
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_init.py:init():547] calling init triggers
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
-config: {}
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_init.py:init():596] starting backend
-2023-06-02 14:12:42,162 INFO MainThread:1498 [wandb_init.py:init():600] setting up manager
-2023-06-02 14:12:42,165 INFO MainThread:1498 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=spawn,fork,forkserver, using: spawn
-2023-06-02 14:12:42,169 INFO MainThread:1498 [wandb_init.py:init():606] backend started and connected
-2023-06-02 14:12:42,174 INFO MainThread:1498 [wandb_init.py:init():700] updated telemetry
-2023-06-02 14:12:42,191 INFO MainThread:1498 [wandb_init.py:init():737] communicating run to backend with 60.0 second timeout
-2023-06-02 14:12:42,783 INFO MainThread:1498 [wandb_init.py:init():780] run resumed
-2023-06-02 14:12:42,794 INFO MainThread:1498 [wandb_run.py:_on_init():2177] communicating current version
-2023-06-02 14:12:42,904 INFO MainThread:1498 [wandb_run.py:_on_init():2186] got version response upgrade_message: "wandb version 0.15.3 is available! To upgrade, please run:\n $ pip install wandb --upgrade"
-
-2023-06-02 14:12:42,904 INFO MainThread:1498 [wandb_init.py:init():787] starting run threads in backend
-2023-06-02 14:12:42,925 INFO MainThread:1498 [wandb_run.py:_console_start():2158] atexit reg
-2023-06-02 14:12:42,925 INFO MainThread:1498 [wandb_run.py:_redirect():2013] redirect: SettingsConsole.WRAP_RAW
-2023-06-02 14:12:42,925 INFO MainThread:1498 [wandb_run.py:_redirect():2078] Wrapping output streams.
-2023-06-02 14:12:42,925 INFO MainThread:1498 [wandb_run.py:_redirect():2103] Redirects installed.
-2023-06-02 14:12:42,926 INFO MainThread:1498 [wandb_init.py:init():829] run started, returning control to user process
-2023-06-02 14:12:42,926 INFO MainThread:1498 [wandb_config.py:__setitem__():151] config set learning_rate = 0.001 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x12d084c70>>
-2023-06-02 14:12:42,926 INFO MainThread:1498 [wandb_run.py:_config_callback():1286] config_cb learning_rate 0.001 None
-2023-06-02 14:12:42,926 INFO MainThread:1498 [wandb_config.py:__setitem__():151] config set optimizer = AdamW - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x12d084c70>>
-2023-06-02 14:12:42,926 INFO MainThread:1498 [wandb_run.py:_config_callback():1286] config_cb optimizer AdamW None
-2023-06-02 14:12:42,926 INFO MainThread:1498 [wandb_watch.py:watch():51] Watching
diff --git a/wandb/run-20230602_141242-akua37dh/run-akua37dh.wandb b/wandb/run-20230602_141242-akua37dh/run-akua37dh.wandb
deleted file mode 100644
index 9f9fe88fad19ad5dfb29691c7082878670169a34..0000000000000000000000000000000000000000
Binary files a/wandb/run-20230602_141242-akua37dh/run-akua37dh.wandb and /dev/null differ
diff --git a/wandb/wandb-resume.json b/wandb/wandb-resume.json
deleted file mode 100644
index 88c9472398446686ea574588b5ad53ac80bd3c0f..0000000000000000000000000000000000000000
--- a/wandb/wandb-resume.json
+++ /dev/null
@@ -1 +0,0 @@
-{"run_id": "akua37dh"}
\ No newline at end of file