Skip to content
Snippets Groups Projects
Commit 934372c0 authored by Gonzalo Martin Garcia's avatar Gonzalo Martin Garcia
Browse files

add CelebAHQ dataloader

parent 910ae792
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 39973 additions and 28 deletions
dataloader/load.py
+ 35
− 23
View file @ 934372c0
...@@ -20,55 +20,67 @@ class UnconditionalDataset(Dataset): ...@@ -20,55 +20,67 @@ class UnconditionalDataset(Dataset):
""" """
### Create DataFrame ### Create DataFrame
file_list = [] #file_list = []
for root, dirs, files in os.walk(fpath, topdown=False): #for root, dirs, files in os.walk(fpath, topdown=False):
for name in sorted(files): # for name in sorted(files):
file_list.append(os.path.join(root, name)) # file_list.append(os.path.join(root, name))
#
df = pd.DataFrame({"Filepath":file_list},) #df = pd.DataFrame({"Filepath":file_list},)
self.df = df[df["Filepath"].str.endswith(ext)] #self.df = df[df["Filepath"].str.endswith(ext)]
if skip_first_n: if skip_first_n:
self.df = self.df[skip_first_n:] self.df = self.df[skip_first_n:]
print(fpath)
if train: if train:
df_train = self.df.sample(frac=frac,random_state=2) fpath = os.path.join(fpath, 'train')
self.df = df_train
else: else:
df_train = self.df.sample(frac=frac,random_state=2) fpath = os.path.join(fpath, 'valid')
df_test = df.drop(df_train.index)
self.df = df_test file_list =[]
for root, dirs, files in os.walk(fpath, topdown=False):
for name in sorted(files):
file_list.append(os.path.join(root, name))
df = pd.DataFrame({"Filepath":file_list},)
self.df = df[df["Filepath"].str.endswith(ext)]
if transform: if transform:
# for training
intermediate_size = 150 intermediate_size = 150
theta = np.pi/4 -np.arccos(intermediate_size/(np.sqrt(2)*img_size)) #Check dataloading.ipynb in analysis-depot for more details theta = np.pi/4 -np.arccos(intermediate_size/(np.sqrt(2)*img_size)) #Check dataloading.ipynb in analysis-depot for more details
transform_rotate = transforms.Compose([transforms.ToTensor(),transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5)), transform_rotate = transforms.Compose([transforms.ToTensor(),
transforms.Resize(intermediate_size,antialias=True), transforms.Resize(intermediate_size,antialias=True),
transforms.RandomRotation(theta/np.pi*180,interpolation=transforms.InterpolationMode.BILINEAR), transforms.RandomRotation(theta/np.pi*180,interpolation=transforms.InterpolationMode.BILINEAR),
transforms.CenterCrop(img_size),transforms.RandomHorizontalFlip(p=0.5)]) transforms.CenterCrop(img_size),transforms.RandomHorizontalFlip(p=0.5),
transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5))])
transform_randomcrop = transforms.Compose([transforms.ToTensor(),transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5)), transform_randomcrop = transforms.Compose([transforms.ToTensor(),
transforms.Resize(intermediate_size),transforms.RandomCrop(img_size),transforms.RandomHorizontalFlip(p=0.5)]) transforms.Resize(intermediate_size, antialias=True),
transforms.RandomCrop(img_size),transforms.RandomHorizontalFlip(p=0.5),
transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5))])
self.transform = transforms.RandomChoice([transform_rotate,transform_randomcrop]) self.transform = transforms.RandomChoice([transform_rotate,transform_randomcrop])
else : else :
# for evaluation
self.transform = transforms.Compose([transforms.ToTensor(), self.transform = transforms.Compose([transforms.ToTensor(),
transforms.Lambda(lambda x: (x * 255).type(torch.uint8)), transforms.Lambda(lambda x: (x * 255).type(torch.uint8)),
transforms.Resize(img_size)]) transforms.Resize(img_size)])
if train==False:
# for testing
self.transform = transforms.Compose([transforms.ToTensor(),
transforms.Resize(intermediate_size, antialias=True),
transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5))])
def __len__(self): def __len__(self):
return len(self.df) return len(self.df)
def __getitem__(self,idx): def __getitem__(self,idx):
path = self.df.iloc[idx].Filepath path = self.df.iloc[idx].Filepath
img = Image.open(path) img = Image.open(path)
if img.mode == 'RGBA':
background = Image.new('RGB', img.size, (255, 255, 255))
background.paste(img, mask=img.split()[3])
img = background
return self.transform(img),0 return self.transform(img),0
def tensor2PIL(self,img): def tensor2PIL(self,img):
......
models/all_unets.py
+ 8
− 5
View file @ 934372c0
...@@ -44,7 +44,7 @@ class UNet_Res(nn.Module): ...@@ -44,7 +44,7 @@ class UNet_Res(nn.Module):
self.up1 = UpsampleBlock_Res(fctr[1],fctr[0],time_dim) self.up1 = UpsampleBlock_Res(fctr[1],fctr[0],time_dim)
self.up2 = UpsampleBlock_Res(fctr[2],fctr[1],time_dim) self.up2 = UpsampleBlock_Res(fctr[2],fctr[1],time_dim)
self.up3 = UpsampleBlock_Res(fctr[3],fctr[2],time_dim,attention=attention) self.up3 = UpsampleBlock_Res(fctr[3],fctr[2],time_dim,attention=attention)
self.up4 = UpsampleBlock_Res(fctr[4],fctr[3],time_dim) self.up4 = UpsampleBlock_Res(fctr[4],fctr[3],time_dim,attention=attention)
# final 1x1 conv # final 1x1 conv
self.end_conv = nn.Conv2d(fctr[0], channels_out, kernel_size=1,bias=True) self.end_conv = nn.Conv2d(fctr[0], channels_out, kernel_size=1,bias=True)
...@@ -154,22 +154,25 @@ class ConvBlock_Res(nn.Module): ...@@ -154,22 +154,25 @@ class ConvBlock_Res(nn.Module):
self.attlayer = MHABlock(channels_in=channels_out) self.attlayer = MHABlock(channels_in=channels_out)
# Convolution layer 1 # Convolution layer 1
self.conv1 = nn.Conv2d(channels_in, channels_out, kernel_size=3, padding='same', bias=True) self.conv1 = nn.Conv2d(channels_in, channels_out, kernel_size=3, padding='same', bias=False)
self.gn1 = nn.GroupNorm(num_groups, channels_out) self.gn1 = nn.GroupNorm(num_groups, channels_out)
self.act1 = nn.SiLU() self.act1 = nn.SiLU()
# Convolution layer 2 # Convolution layer 2
self.conv2 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding='same', bias=True) self.conv2 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding='same', bias=False)
self.gn2 = nn.GroupNorm(num_groups, channels_out) self.gn2 = nn.GroupNorm(num_groups, channels_out)
self.act2 = nn.SiLU() self.act2 = nn.SiLU()
# Convolution layer 3 # Convolution layer 3
self.conv3 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding='same', bias=True) self.conv3 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding='same', bias=False)
self.gn3 = nn.GroupNorm(num_groups, channels_out) self.gn3 = nn.GroupNorm(num_groups, channels_out)
self.act3 = nn.SiLU() self.act3 = nn.SiLU()
#Convolution skip #Convolution skip
self.res_skip = nn.Identity()
if channels_in != channels_out:
self.res_skip = nn.Conv2d(channels_in, channels_out, kernel_size=1) self.res_skip = nn.Conv2d(channels_in, channels_out, kernel_size=1)
#self.res_skip = nn.Conv2d(channels_in,channels_out,kernel_size=1)
nn.init.xavier_normal_(self.conv1.weight) nn.init.xavier_normal_(self.conv1.weight)
nn.init.xavier_normal_(self.conv2.weight) nn.init.xavier_normal_(self.conv2.weight)
......
wandb/debug-cli.id929844.log 0 → 100644
+ 0
− 0
View file @ 934372c0
wandb/debug-internal.log 0 → 120000
+ 1
− 0
View file @ 934372c0
run-20230709_160458-humans/logs/debug-internal.log
\ No newline at end of file
wandb/debug.log 0 → 120000
+ 1
− 0
View file @ 934372c0
run-20230709_160458-humans/logs/debug.log
\ No newline at end of file
wandb/latest-run 0 → 120000
+ 1
− 0
View file @ 934372c0
run-20230709_160458-humans
\ No newline at end of file
wandb/run-20230708_232745-humans/files/config.yaml 0 → 100644
+ 36
− 0
View file @ 934372c0
wandb_version: 1
_wandb:
desc: null
value:
python_version: 3.10.4
cli_version: 0.15.3
framework: torch
is_jupyter_run: false
is_kaggle_kernel: false
start_time: 1688851665.535903
t:
1:
- 1
- 41
- 55
2:
- 1
- 41
- 55
3:
- 2
- 13
- 14
- 19
- 23
4: 3.10.4
5: 0.15.3
8:
- 5
learning_rate:
desc: null
value: 0.0001
optimizer:
desc: null
value: AdamW
wandb/run-20230708_232745-humans/files/requirements.txt 0 → 100644
+ 175
− 0
View file @ 934372c0
alabaster==0.7.12
appdirs==1.4.4
asn1crypto==1.5.1
atomicwrites==1.4.0
attrs==21.4.0
babel==2.10.1
backports.entry-points-selectable==1.1.1
backports.functools-lru-cache==1.6.4
bcrypt==3.2.2
bitstring==3.1.9
blist==1.3.6
cachecontrol==0.12.11
cachy==0.3.0
certifi==2021.10.8
cffi==1.15.0
chardet==4.0.0
charset-normalizer==2.0.12
cleo==0.8.1
click==8.1.3
clikit==0.6.2
cmake==3.26.3
colorama==0.4.4
contourpy==1.0.7
crashtest==0.3.1
cryptography==37.0.1
cycler==0.11.0
cython==0.29.28
decorator==5.1.1
diffusers==0.16.1
distlib==0.3.4
docker-pycreds==0.4.0
docopt==0.6.2
docutils==0.17.1
ecdsa==0.17.0
editables==0.3
einops==0.6.1
filelock==3.6.0
flit-core==3.7.1
flit==3.7.1
fonttools==4.39.4
fsspec==2022.3.0
future==0.18.2
gitdb==4.0.10
gitpython==3.1.31
glob2==0.7
h5py==3.8.0
html5lib==1.1
huggingface-hub==0.15.1
idna==3.3
imagesize==1.3.0
importlib-metadata==4.11.3
importlib-resources==5.7.1
iniconfig==1.1.1
intervaltree==3.1.0
intreehooks==1.0
ipaddress==1.0.23
jeepney==0.8.0
jinja2==3.1.2
joblib==1.1.0
jsonschema==4.4.0
keyring==23.5.0
keyrings.alt==4.1.0
kiwisolver==1.4.4
liac-arff==2.5.0
lit==16.0.5
lockfile==0.12.2
markupsafe==2.1.1
matplotlib==3.7.1
mock==4.0.3
more-itertools==8.12.0
mpmath==1.3.0
msgpack==1.0.3
netaddr==0.8.0
netifaces==0.11.0
networkx==3.1
numpy==1.24.3
nvidia-cublas-cu11==11.10.3.66
nvidia-cuda-cupti-cu11==11.7.101
nvidia-cuda-nvrtc-cu11==11.7.99
nvidia-cuda-runtime-cu11==11.7.99
nvidia-cudnn-cu11==8.5.0.96
nvidia-cufft-cu11==10.9.0.58
nvidia-curand-cu11==10.2.10.91
nvidia-cusolver-cu11==11.4.0.1
nvidia-cusparse-cu11==11.7.4.91
nvidia-nccl-cu11==2.14.3
nvidia-nvtx-cu11==11.7.91
packaging==20.9
pandas==2.0.2
paramiko==2.10.4
pastel==0.2.1
pathlib2==2.3.7.post1
pathspec==0.9.0
pathtools==0.1.2
pbr==5.8.1
pexpect==4.8.0
pillow==9.5.0
pip==22.0.4
pkginfo==1.8.2
platformdirs==2.4.1
pluggy==1.0.0
poetry-core==1.0.8
poetry==1.1.13
protobuf==4.23.2
psutil==5.9.0
ptyprocess==0.7.0
py-expression-eval==0.3.14
py==1.11.0
pyasn1==0.4.8
pycparser==2.21
pycrypto==2.6.1
pygments==2.12.0
pylev==1.4.0
pynacl==1.5.0
pyparsing==3.0.8
pyrsistent==0.18.1
pytest==7.1.2
python-dateutil==2.8.2
pytoml==0.1.21
pytz==2022.1
pyyaml==6.0
regex==2022.4.24
requests-toolbelt==0.9.1
requests==2.27.1
scandir==1.10.0
scipy==1.10.1
secretstorage==3.3.2
semantic-version==2.9.0
sentry-sdk==1.25.0
setproctitle==1.3.2
setuptools-rust==1.3.0
setuptools-scm==6.4.2
setuptools==62.1.0
shellingham==1.4.0
simplegeneric==0.8.1
simplejson==3.17.6
six==1.16.0
smmap==5.0.0
snowballstemmer==2.2.0
sortedcontainers==2.4.0
sphinx-bootstrap-theme==0.8.1
sphinx==4.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
sphinxcontrib-websupport==1.2.4
sympy==1.12
tabulate==0.8.9
threadpoolctl==3.1.0
toml==0.10.2
tomli-w==1.0.0
tomli==2.0.1
tomlkit==0.10.2
torch-fidelity==0.3.0
torch==2.0.1
torchaudio==2.0.2
torchmetrics==0.11.4
torchvision==0.15.2
tqdm==4.65.0
triton==2.0.0
typing-extensions==4.2.0
tzdata==2023.3
ujson==5.2.0
urllib3==1.26.16
virtualenv==20.14.1
wandb==0.15.3
wcwidth==0.2.5
webencodings==0.5.1
wheel==0.37.1
xlrd==2.0.1
zipfile36==0.1.3
zipp==3.8.0
\ No newline at end of file
wandb/run-20230708_232745-humans/files/wandb-metadata.json 0 → 100644
+ 292
− 0
View file @ 934372c0
{
"os": "Linux-4.18.0-425.19.2.el8_7.x86_64-x86_64-with-glibc2.28",
"python": "3.10.4",
"heartbeatAt": "2023-07-08T21:27:46.318010",
"startedAt": "2023-07-08T21:27:45.403194",
"docker": null,
"cuda": null,
"args": [
"train",
"/home/id929844/experiments_gonzalo/humans/settings"
],
"state": "running",
"program": "/rwthfs/rz/cluster/home/id929844/diffusion_project/main.py",
"codePath": "main.py",
"git": {
"remote": "https://git.rwth-aachen.de/diffusion-project/diffusion_project.git",
"commit": "910ae79222394141930cd1bf6eafb783b4bd397e"
},
"email": "gonzalomartingarcia0@gmail.com",
"root": "/rwthfs/rz/cluster/home/id929844/diffusion_project",
"host": "login18-g-1.hpc.itc.rwth-aachen.de",
"username": "id929844",
"executable": "/cvmfs/software.hpc.rwth.de/Linux/RH8/x86_64/intel/skylake_avx512/software/Python/3.10.4-GCCcore-11.3.0/bin/python",
"cpu_count": 48,
"cpu_count_logical": 48,
"cpu_freq": {
"current": 3.6861249999999983,
"min": 3700.0,
"max": 3700.0
},
"cpu_freq_per_core": [
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.377,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
},
{
"current": 3.7,
"min": 3700.0,
"max": 3700.0
}
],
"disk": {
"total": 39.043426513671875,
"used": 18.819751739501953
},
"gpu": "Tesla V100-SXM2-16GB",
"gpu_count": 2,
"gpu_devices": [
{
"name": "Tesla V100-SXM2-16GB",
"memory_total": 17179869184
},
{
"name": "Tesla V100-SXM2-16GB",
"memory_total": 17179869184
}
],
"memory": {
"total": 376.3241539001465
}
}
wandb/run-20230708_232745-humans/files/wandb-summary.json 0 → 100644
+ 1
− 0
View file @ 934372c0
{"loss": 0.07890284061431885, "learning_rate": 9.999996060526846e-05, "epoch": 0, "batch": 118, "_timestamp": 1688851816.1800227, "_runtime": 150.64411973953247, "_step": 118, "_wandb": {"runtime": 150}}
\ No newline at end of file
wandb/run-20230708_232745-humans/logs/debug-internal.log 0 → 100644
+ 817
− 0
View file @ 934372c0
2023-07-08 23:27:45,440 INFO StreamThr :24434 [internal.py:wandb_internal():86] W&B internal server running at pid: 24434, started at: 2023-07-08 23:27:45.437961
2023-07-08 23:27:45,441 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status
2023-07-08 23:27:45,541 INFO WriterThread:24434 [datastore.py:open_for_write():85] open: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/run-humans.wandb
2023-07-08 23:27:45,544 DEBUG SenderThread:24434 [sender.py:send():375] send: header
2023-07-08 23:27:45,578 DEBUG SenderThread:24434 [sender.py:send():375] send: run
2023-07-08 23:27:45,584 INFO SenderThread:24434 [sender.py:_maybe_setup_resume():761] checking resume status for deep-lab-/Unconditional Landscapes/humans
2023-07-08 23:27:46,179 INFO SenderThread:24434 [dir_watcher.py:__init__():219] watching files in: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files
2023-07-08 23:27:46,180 INFO SenderThread:24434 [sender.py:_start_run_threads():1124] run started: humans with start time 1688851665.535903
2023-07-08 23:27:46,180 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:27:46,183 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: check_version
2023-07-08 23:27:46,184 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:27:46,184 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: check_version
2023-07-08 23:27:46,258 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: run_start
2023-07-08 23:27:46,263 DEBUG HandlerThread:24434 [system_info.py:__init__():31] System info init
2023-07-08 23:27:46,264 DEBUG HandlerThread:24434 [system_info.py:__init__():46] System info init done
2023-07-08 23:27:46,264 INFO HandlerThread:24434 [system_monitor.py:start():181] Starting system monitor
2023-07-08 23:27:46,264 INFO SystemMonitor:24434 [system_monitor.py:_start():145] Starting system asset monitoring threads
2023-07-08 23:27:46,264 INFO HandlerThread:24434 [system_monitor.py:probe():201] Collecting system info
2023-07-08 23:27:46,264 INFO SystemMonitor:24434 [interfaces.py:start():190] Started cpu monitoring
2023-07-08 23:27:46,265 INFO SystemMonitor:24434 [interfaces.py:start():190] Started disk monitoring
2023-07-08 23:27:46,265 INFO SystemMonitor:24434 [interfaces.py:start():190] Started gpu monitoring
2023-07-08 23:27:46,266 INFO SystemMonitor:24434 [interfaces.py:start():190] Started memory monitoring
2023-07-08 23:27:46,266 INFO SystemMonitor:24434 [interfaces.py:start():190] Started network monitoring
2023-07-08 23:27:46,317 DEBUG HandlerThread:24434 [system_info.py:probe():195] Probing system
2023-07-08 23:27:46,330 DEBUG HandlerThread:24434 [system_info.py:_probe_git():180] Probing git
2023-07-08 23:27:46,361 DEBUG HandlerThread:24434 [system_info.py:_probe_git():188] Probing git done
2023-07-08 23:27:46,361 DEBUG HandlerThread:24434 [system_info.py:probe():240] Probing system done
2023-07-08 23:27:46,361 DEBUG HandlerThread:24434 [system_monitor.py:probe():210] {'os': 'Linux-4.18.0-425.19.2.el8_7.x86_64-x86_64-with-glibc2.28', 'python': '3.10.4', 'heartbeatAt': '2023-07-08T21:27:46.318010', 'startedAt': '2023-07-08T21:27:45.403194', 'docker': None, 'cuda': None, 'args': ('train', '/home/id929844/experiments_gonzalo/humans/settings'), 'state': 'running', 'program': '/rwthfs/rz/cluster/home/id929844/diffusion_project/main.py', 'codePath': 'main.py', 'git': {'remote': 'https://git.rwth-aachen.de/diffusion-project/diffusion_project.git', 'commit': '910ae79222394141930cd1bf6eafb783b4bd397e'}, 'email': 'gonzalomartingarcia0@gmail.com', 'root': '/rwthfs/rz/cluster/home/id929844/diffusion_project', 'host': 'login18-g-1.hpc.itc.rwth-aachen.de', 'username': 'id929844', 'executable': '/cvmfs/software.hpc.rwth.de/Linux/RH8/x86_64/intel/skylake_avx512/software/Python/3.10.4-GCCcore-11.3.0/bin/python', 'cpu_count': 48, 'cpu_count_logical': 48, 'cpu_freq': {'current': 3.6861249999999983, 'min': 3700.0, 'max': 3700.0}, 'cpu_freq_per_core': [{'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.377, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}, {'current': 3.7, 'min': 3700.0, 'max': 3700.0}], 'disk': {'total': 39.043426513671875, 'used': 18.819751739501953}, 'gpu': 'Tesla V100-SXM2-16GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla V100-SXM2-16GB', 'memory_total': 17179869184}, {'name': 'Tesla V100-SXM2-16GB', 'memory_total': 17179869184}], 'memory': {'total': 376.3241539001465}}
2023-07-08 23:27:46,361 INFO HandlerThread:24434 [system_monitor.py:probe():211] Finished collecting system info
2023-07-08 23:27:46,361 INFO HandlerThread:24434 [system_monitor.py:probe():214] Publishing system info
2023-07-08 23:27:46,361 DEBUG HandlerThread:24434 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
2023-07-08 23:27:46,367 DEBUG HandlerThread:24434 [system_info.py:_save_pip():67] Saving pip packages done
2023-07-08 23:27:46,373 INFO HandlerThread:24434 [system_monitor.py:probe():216] Finished publishing system info
2023-07-08 23:27:46,487 DEBUG SenderThread:24434 [sender.py:send():375] send: files
2023-07-08 23:27:46,487 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now
2023-07-08 23:27:46,495 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:27:46,495 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:27:46,655 DEBUG SenderThread:24434 [sender.py:send():375] send: telemetry
2023-07-08 23:27:46,656 DEBUG SenderThread:24434 [sender.py:send():375] send: telemetry
2023-07-08 23:27:46,656 DEBUG SenderThread:24434 [sender.py:send():375] send: config
2023-07-08 23:27:46,656 DEBUG SenderThread:24434 [sender.py:send():375] send: config
2023-07-08 23:27:46,930 INFO wandb-upload_0:24434 [upload_job.py:push():137] Uploaded file /tmp/id929844/login18-g-1_16644/tmp8yq9kkl_wandb/1uktm71e-wandb-metadata.json
2023-07-08 23:27:47,182 INFO Thread-12 :24434 [dir_watcher.py:_on_file_created():278] file/dir created: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-metadata.json
2023-07-08 23:27:47,182 INFO Thread-12 :24434 [dir_watcher.py:_on_file_created():278] file/dir created: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:27:47,182 INFO Thread-12 :24434 [dir_watcher.py:_on_file_created():278] file/dir created: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/requirements.txt
2023-07-08 23:27:50,657 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:27:55,658 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:00,658 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:01,495 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:28:01,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:28:06,628 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:09,297 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:09,298 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:09,298 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:09,326 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:10,197 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:11,681 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:11,681 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:11,681 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:11,681 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:11,685 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:12,198 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:12,715 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:12,716 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:12,716 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:12,720 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:13,198 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:13,877 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:13,878 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:13,878 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:13,885 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:14,199 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:14,877 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:14,878 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:14,878 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:14,883 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:15,200 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:15,971 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:15,971 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:15,971 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:15,979 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:16,200 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:16,495 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:28:16,495 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:28:16,901 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:16,901 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:16,906 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:17,102 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:17,107 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:17,201 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/config.yaml
2023-07-08 23:28:17,201 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:17,950 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:17,950 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:17,951 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:17,959 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:18,201 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:19,039 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:19,040 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:19,040 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:19,045 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:19,202 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:20,046 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:20,046 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:20,046 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:20,055 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:20,202 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:21,248 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:21,249 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:21,249 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:21,257 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:22,204 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:22,257 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:22,375 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:22,376 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:22,376 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:22,383 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:23,204 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:23,363 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:23,364 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:23,364 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:23,372 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:24,205 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:24,421 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:24,422 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:24,422 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:24,427 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:25,205 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:25,418 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:25,419 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:25,419 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:25,428 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:26,206 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:26,397 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:26,398 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:26,398 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:26,402 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:27,207 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:27,402 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:27,547 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:27,548 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:27,548 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:27,555 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:28,207 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:28,644 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:28,645 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:28,645 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:28,685 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:29,208 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:29,727 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:29,727 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:29,728 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:29,767 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:30,208 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:30,925 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:30,926 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:30,926 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:30,930 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:31,209 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:31,495 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:28:31,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:28:31,845 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:31,846 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:31,846 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:31,850 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:32,210 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:32,816 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:32,817 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:32,817 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:32,817 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:32,821 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:33,211 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:33,765 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:33,765 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:33,766 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:33,771 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:34,211 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:34,771 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:34,772 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:34,772 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:34,778 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:35,212 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:35,686 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:35,687 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:35,687 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:35,695 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:36,212 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:36,784 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:36,785 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:36,785 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:36,793 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:37,213 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:37,882 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:37,882 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:37,882 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:37,883 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:37,887 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:38,214 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:38,980 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:38,981 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:38,981 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:38,985 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:39,214 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:39,955 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:39,956 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:39,956 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:39,960 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:40,215 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:40,929 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:40,929 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:40,929 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:40,934 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:41,216 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:42,300 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:42,301 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:42,301 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:42,307 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:43,217 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:43,290 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:43,290 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:43,290 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:43,290 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:43,298 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:44,217 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:44,342 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:44,343 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:44,343 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:44,353 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:45,218 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:45,335 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:45,336 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:45,336 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:45,357 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:46,218 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:46,266 DEBUG SystemMonitor:24434 [system_monitor.py:_start():159] Starting system metrics aggregation loop
2023-07-08 23:28:46,267 DEBUG SenderThread:24434 [sender.py:send():375] send: stats
2023-07-08 23:28:46,368 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:46,369 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:46,369 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:46,375 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:46,495 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:28:46,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:28:47,219 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:47,542 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:47,543 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:47,543 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:47,561 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:48,220 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:48,561 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:48,671 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:48,672 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:48,672 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:48,700 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:49,220 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:49,832 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:49,833 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:49,833 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:49,844 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:50,221 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:50,912 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:50,912 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:50,912 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:50,931 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:51,221 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:51,938 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:51,938 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:51,938 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:51,944 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:52,222 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:52,860 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:52,860 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:52,861 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:52,868 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:53,223 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:53,868 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:53,975 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:53,976 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:53,976 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:53,982 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:54,223 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:54,932 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:54,933 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:54,933 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:54,941 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:55,224 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:55,980 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:55,981 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:55,981 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:55,988 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:56,224 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:57,150 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:57,151 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:57,151 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:57,155 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:57,225 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:28:58,269 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:58,269 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:58,270 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:58,273 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:59,199 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:28:59,200 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:28:59,200 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:28:59,200 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:28:59,205 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:28:59,226 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:00,278 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:00,279 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:00,279 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:00,283 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:01,161 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:01,162 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:01,162 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:01,165 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:01,228 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:01,496 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:29:01,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:29:02,122 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:02,122 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:02,123 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:02,131 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:02,228 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:03,218 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:03,218 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:03,219 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:03,231 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:03,231 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:04,231 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:04,260 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:04,260 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:04,261 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:04,273 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:05,232 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:05,318 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:05,318 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:05,318 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:05,326 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:06,233 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:06,440 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:06,441 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:06,441 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:06,449 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:07,234 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:07,471 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:07,471 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:07,472 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:07,477 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:08,234 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:08,628 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:08,628 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:08,628 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:08,651 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:09,235 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:09,652 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:10,089 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:10,089 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:10,090 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:10,128 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:10,235 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:11,278 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:11,279 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:11,279 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:11,303 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:12,237 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:12,574 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:12,575 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:12,575 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:12,588 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:13,237 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:13,612 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:13,612 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:13,612 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:13,617 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:14,238 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:14,805 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:14,806 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:14,806 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:14,806 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:14,811 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:15,238 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:15,788 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:15,788 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:15,788 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:15,792 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:16,239 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:16,268 DEBUG SenderThread:24434 [sender.py:send():375] send: stats
2023-07-08 23:29:16,496 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:29:16,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:29:16,815 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:16,815 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:16,815 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:16,820 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:17,240 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:17,814 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:17,815 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:17,815 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:17,820 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:18,241 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:19,000 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:19,001 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:19,001 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:19,005 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:19,241 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:20,006 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:20,120 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:20,121 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:20,121 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:20,125 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:20,242 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:21,203 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:21,203 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:21,203 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:21,209 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:21,243 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:22,355 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:22,356 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:22,356 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:22,365 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:23,244 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:23,464 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:23,465 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:23,465 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:23,490 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:24,245 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:24,469 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:24,470 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:24,470 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:24,496 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:25,245 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:25,492 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:25,493 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:25,493 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:25,493 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:25,499 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:26,246 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:26,459 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:26,460 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:26,460 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:26,467 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:27,247 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:27,512 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:27,513 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:27,513 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:27,520 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:28,247 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:28,544 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:28,544 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:28,544 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:28,569 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:29,248 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:29,662 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:29,663 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:29,663 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:29,704 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:30,249 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:30,704 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:30,788 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:30,788 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:30,789 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:30,806 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:31,249 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:31,496 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:29:31,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:29:31,865 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:31,866 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:31,866 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:31,873 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:32,250 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:32,921 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:32,922 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:32,922 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:32,927 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:33,250 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:34,064 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:34,064 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:34,064 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:34,071 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:34,251 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:34,993 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:34,994 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:34,994 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:34,998 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:35,252 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:35,998 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:36,032 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:36,032 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:36,033 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:36,037 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:36,252 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:36,944 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:36,945 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:36,945 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:36,951 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:37,253 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:37,997 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:37,998 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:37,998 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:38,002 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:38,253 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:39,074 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:39,075 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:39,075 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:39,079 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:39,254 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:40,191 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:40,192 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:40,192 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:40,198 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:40,255 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:41,198 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:41,286 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:41,286 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:41,287 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:41,297 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:42,256 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:42,457 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:42,458 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:42,458 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:42,473 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:43,257 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:43,476 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:43,476 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:43,476 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:43,482 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:44,257 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:44,429 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:44,430 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:44,430 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:44,437 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:45,258 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:45,443 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:45,444 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:45,444 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:45,451 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:46,258 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:46,270 DEBUG SenderThread:24434 [sender.py:send():375] send: stats
2023-07-08 23:29:46,270 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:46,496 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:29:46,496 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:29:46,499 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:46,637 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:46,637 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:46,641 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:47,259 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:47,485 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:47,486 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:47,486 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:47,493 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:48,259 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:48,527 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:48,527 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:48,527 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:48,560 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:49,260 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:49,675 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:49,675 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:49,675 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:49,705 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:50,260 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:51,046 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:51,047 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:51,047 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:51,051 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:51,261 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:52,052 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:52,119 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:52,119 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:52,119 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:52,127 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:52,261 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:53,220 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:53,220 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:53,221 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:53,226 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:53,262 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:54,178 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:54,179 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:54,179 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:54,189 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:54,262 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:55,142 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:55,143 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:55,143 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:55,147 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:55,263 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:56,104 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:56,105 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:56,105 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:56,109 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:56,264 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:57,091 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:57,092 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:57,092 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:57,092 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:29:57,098 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:57,264 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:58,044 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:58,044 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:58,045 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:58,051 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:58,265 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:29:58,988 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:29:58,988 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:29:58,989 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:29:59,004 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:29:59,266 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:00,084 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:00,085 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:00,085 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:00,091 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:00,266 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:01,007 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:01,008 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:01,008 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:01,013 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:01,267 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:01,496 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:30:01,497 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:30:02,108 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:02,108 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:02,109 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:02,109 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:30:02,112 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:02,268 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:03,263 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:03,264 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:03,264 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:03,269 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:03,269 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:04,291 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:04,291 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:04,291 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:04,297 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:05,270 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:05,329 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:05,329 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:05,330 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:05,336 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:06,271 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:06,356 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:06,356 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:06,356 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:06,361 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:07,272 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:07,362 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:30:07,455 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:07,456 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:07,456 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:07,460 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:08,272 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:08,523 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:08,523 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:08,524 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:08,530 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:09,273 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:09,828 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:09,828 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:09,829 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:09,875 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:10,274 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:10,934 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:10,935 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:10,935 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:10,940 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:11,274 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:11,986 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:11,986 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:11,987 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:11,991 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:12,275 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:12,898 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:12,899 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:12,899 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:12,899 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:30:12,903 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:13,275 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:13,974 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:13,975 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:13,975 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:13,981 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:14,276 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:15,034 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:15,034 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:15,035 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:15,041 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:15,276 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:16,180 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: partial_history
2023-07-08 23:30:16,181 DEBUG SenderThread:24434 [sender.py:send():375] send: history
2023-07-08 23:30:16,181 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: summary_record
2023-07-08 23:30:16,187 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:16,270 DEBUG SenderThread:24434 [sender.py:send():375] send: stats
2023-07-08 23:30:16,277 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:16,496 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: stop_status
2023-07-08 23:30:16,497 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: stop_status
2023-07-08 23:30:16,662 DEBUG SenderThread:24434 [sender.py:send():375] send: telemetry
2023-07-08 23:30:16,663 DEBUG SenderThread:24434 [sender.py:send():375] send: exit
2023-07-08 23:30:16,663 INFO SenderThread:24434 [sender.py:send_exit():598] handling exit code: 1
2023-07-08 23:30:16,663 INFO SenderThread:24434 [sender.py:send_exit():600] handling runtime: 150
2023-07-08 23:30:16,682 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:16,682 INFO SenderThread:24434 [sender.py:send_exit():606] send defer
2023-07-08 23:30:16,683 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,683 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 0
2023-07-08 23:30:16,683 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,683 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 0
2023-07-08 23:30:16,683 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 1
2023-07-08 23:30:16,683 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,683 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 1
2023-07-08 23:30:16,683 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,683 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 1
2023-07-08 23:30:16,683 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 2
2023-07-08 23:30:16,683 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,683 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 2
2023-07-08 23:30:16,683 INFO HandlerThread:24434 [system_monitor.py:finish():190] Stopping system monitor
2023-07-08 23:30:16,684 INFO HandlerThread:24434 [interfaces.py:finish():202] Joined cpu monitor
2023-07-08 23:30:16,684 DEBUG SystemMonitor:24434 [system_monitor.py:_start():166] Finished system metrics aggregation loop
2023-07-08 23:30:16,684 INFO HandlerThread:24434 [interfaces.py:finish():202] Joined disk monitor
2023-07-08 23:30:16,684 DEBUG SystemMonitor:24434 [system_monitor.py:_start():170] Publishing last batch of metrics
2023-07-08 23:30:16,966 INFO HandlerThread:24434 [interfaces.py:finish():202] Joined gpu monitor
2023-07-08 23:30:16,966 INFO HandlerThread:24434 [interfaces.py:finish():202] Joined memory monitor
2023-07-08 23:30:16,966 INFO HandlerThread:24434 [interfaces.py:finish():202] Joined network monitor
2023-07-08 23:30:16,967 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,967 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 2
2023-07-08 23:30:16,967 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 3
2023-07-08 23:30:16,967 DEBUG SenderThread:24434 [sender.py:send():375] send: stats
2023-07-08 23:30:16,967 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,967 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 3
2023-07-08 23:30:16,967 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,967 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 3
2023-07-08 23:30:16,967 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 4
2023-07-08 23:30:16,967 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,967 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 4
2023-07-08 23:30:16,968 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,968 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 4
2023-07-08 23:30:16,968 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 5
2023-07-08 23:30:16,968 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,968 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 5
2023-07-08 23:30:16,968 DEBUG SenderThread:24434 [sender.py:send():375] send: summary
2023-07-08 23:30:16,973 INFO SenderThread:24434 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
2023-07-08 23:30:16,973 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,973 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 5
2023-07-08 23:30:16,973 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 6
2023-07-08 23:30:16,973 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:16,973 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 6
2023-07-08 23:30:16,973 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:16,973 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 6
2023-07-08 23:30:16,978 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: status_report
2023-07-08 23:30:17,199 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 7
2023-07-08 23:30:17,199 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:17,199 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 7
2023-07-08 23:30:17,199 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:17,199 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 7
2023-07-08 23:30:17,199 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 8
2023-07-08 23:30:17,199 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:17,199 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 8
2023-07-08 23:30:17,199 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:17,199 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 8
2023-07-08 23:30:17,277 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/config.yaml
2023-07-08 23:30:17,277 INFO Thread-12 :24434 [dir_watcher.py:_on_file_modified():295] file/dir modified: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:17,479 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 9
2023-07-08 23:30:17,479 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:17,479 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 9
2023-07-08 23:30:17,479 DEBUG SenderThread:24434 [sender.py:send():375] send: artifact
2023-07-08 23:30:17,664 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: poll_exit
2023-07-08 23:30:19,119 INFO wandb-upload_1:24434 [upload_job.py:push():95] Uploaded file /home/id929844/.local/share/wandb/artifacts/staging/tmp6dqsurz8
2023-07-08 23:30:19,146 INFO wandb-upload_0:24434 [upload_job.py:push():95] Uploaded file /home/id929844/.local/share/wandb/artifacts/staging/tmpfyhv2y8t
2023-07-08 23:30:20,289 INFO SenderThread:24434 [sender.py:send_artifact():1474] sent artifact job-https___git.rwth-aachen.de_diffusion-project_diffusion_project.git_main.py - {'id': 'QXJ0aWZhY3Q6NTA4NTQ3MjI2', 'digest': 'bf1568722af4a74888c40644c1b5c7ca', 'state': 'PENDING', 'aliases': [], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjc4MzI2MzYx', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NDk4MzI2NjMw', 'versionIndex': 3}}, 'version': 'latest'}
2023-07-08 23:30:20,289 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:20,289 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 9
2023-07-08 23:30:20,289 INFO SenderThread:24434 [dir_watcher.py:finish():365] shutting down directory watcher
2023-07-08 23:30:21,279 INFO SenderThread:24434 [dir_watcher.py:finish():395] scan: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files
2023-07-08 23:30:21,279 INFO SenderThread:24434 [dir_watcher.py:finish():409] scan save: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/config.yaml config.yaml
2023-07-08 23:30:21,280 INFO SenderThread:24434 [dir_watcher.py:finish():409] scan save: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-metadata.json wandb-metadata.json
2023-07-08 23:30:21,280 INFO SenderThread:24434 [dir_watcher.py:finish():409] scan save: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/requirements.txt requirements.txt
2023-07-08 23:30:21,280 INFO SenderThread:24434 [dir_watcher.py:finish():409] scan save: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json wandb-summary.json
2023-07-08 23:30:21,283 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 10
2023-07-08 23:30:21,286 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: poll_exit
2023-07-08 23:30:21,287 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:21,291 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 10
2023-07-08 23:30:21,291 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:21,291 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 10
2023-07-08 23:30:21,291 INFO SenderThread:24434 [file_pusher.py:finish():167] shutting down file pusher
2023-07-08 23:30:21,738 INFO wandb-upload_0:24434 [upload_job.py:push():137] Uploaded file /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/requirements.txt
2023-07-08 23:30:21,799 INFO wandb-upload_2:24434 [upload_job.py:push():137] Uploaded file /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/wandb-summary.json
2023-07-08 23:30:21,916 INFO wandb-upload_1:24434 [upload_job.py:push():137] Uploaded file /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/files/config.yaml
2023-07-08 23:30:22,116 INFO Thread-11 (_thread_body):24434 [sender.py:transition_state():626] send defer: 11
2023-07-08 23:30:22,117 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:22,117 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 11
2023-07-08 23:30:22,117 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:22,117 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 11
2023-07-08 23:30:22,117 INFO SenderThread:24434 [file_pusher.py:join():172] waiting for file pusher
2023-07-08 23:30:22,117 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 12
2023-07-08 23:30:22,117 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:22,117 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 12
2023-07-08 23:30:22,117 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:22,117 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 12
2023-07-08 23:30:22,306 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 13
2023-07-08 23:30:22,306 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:22,306 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 13
2023-07-08 23:30:22,306 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:22,306 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 13
2023-07-08 23:30:22,306 INFO SenderThread:24434 [sender.py:transition_state():626] send defer: 14
2023-07-08 23:30:22,306 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: defer
2023-07-08 23:30:22,307 DEBUG SenderThread:24434 [sender.py:send():375] send: final
2023-07-08 23:30:22,307 INFO HandlerThread:24434 [handler.py:handle_request_defer():170] handle defer: 14
2023-07-08 23:30:22,307 DEBUG SenderThread:24434 [sender.py:send():375] send: footer
2023-07-08 23:30:22,307 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: defer
2023-07-08 23:30:22,307 INFO SenderThread:24434 [sender.py:send_request_defer():622] handle sender defer: 14
2023-07-08 23:30:22,308 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: poll_exit
2023-07-08 23:30:22,308 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: server_info
2023-07-08 23:30:22,308 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: get_summary
2023-07-08 23:30:22,308 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: poll_exit
2023-07-08 23:30:22,308 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: sampled_history
2023-07-08 23:30:22,308 DEBUG SenderThread:24434 [sender.py:send_request():402] send_request: server_info
2023-07-08 23:30:22,444 DEBUG HandlerThread:24434 [handler.py:handle_request():144] handle_request: shutdown
2023-07-08 23:30:22,444 INFO HandlerThread:24434 [handler.py:finish():842] shutting down handler
2023-07-08 23:30:23,308 INFO WriterThread:24434 [datastore.py:close():298] close: /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/run-humans.wandb
2023-07-08 23:30:23,443 INFO SenderThread:24434 [sender.py:finish():1550] shutting down sender
2023-07-08 23:30:23,443 INFO SenderThread:24434 [file_pusher.py:finish():167] shutting down file pusher
2023-07-08 23:30:23,443 INFO SenderThread:24434 [file_pusher.py:join():172] waiting for file pusher
wandb/run-20230708_232745-humans/logs/debug.log 0 → 100644
+ 38
− 0
View file @ 934372c0
2023-07-08 23:27:45,427 INFO MainThread:16924 [wandb_setup.py:_flush():76] Current SDK version is 0.15.3
2023-07-08 23:27:45,427 INFO MainThread:16924 [wandb_setup.py:_flush():76] Configure stats pid to 16924
2023-07-08 23:27:45,427 INFO MainThread:16924 [wandb_setup.py:_flush():76] Loading settings from /home/id929844/.config/wandb/settings
2023-07-08 23:27:45,428 INFO MainThread:16924 [wandb_setup.py:_flush():76] Loading settings from /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/settings
2023-07-08 23:27:45,428 INFO MainThread:16924 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
2023-07-08 23:27:45,428 INFO MainThread:16924 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
2023-07-08 23:27:45,428 INFO MainThread:16924 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'main.py', 'program': '/rwthfs/rz/cluster/home/id929844/diffusion_project/main.py'}
2023-07-08 23:27:45,428 INFO MainThread:16924 [wandb_init.py:_log_setup():507] Logging user logs to /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/logs/debug.log
2023-07-08 23:27:45,428 INFO MainThread:16924 [wandb_init.py:_log_setup():508] Logging internal logs to /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_232745-humans/logs/debug-internal.log
2023-07-08 23:27:45,429 INFO MainThread:16924 [wandb_init.py:init():547] calling init triggers
2023-07-08 23:27:45,429 INFO MainThread:16924 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
config: {}
2023-07-08 23:27:45,429 INFO MainThread:16924 [wandb_init.py:init():596] starting backend
2023-07-08 23:27:45,429 INFO MainThread:16924 [wandb_init.py:init():600] setting up manager
2023-07-08 23:27:45,435 INFO MainThread:16924 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
2023-07-08 23:27:45,535 INFO MainThread:16924 [wandb_init.py:init():606] backend started and connected
2023-07-08 23:27:45,540 INFO MainThread:16924 [wandb_init.py:init():700] updated telemetry
2023-07-08 23:27:45,577 INFO MainThread:16924 [wandb_init.py:init():737] communicating run to backend with 60.0 second timeout
2023-07-08 23:27:46,182 INFO MainThread:16924 [wandb_run.py:_on_init():2177] communicating current version
2023-07-08 23:27:46,252 INFO MainThread:16924 [wandb_run.py:_on_init():2186] got version response upgrade_message: "wandb version 0.15.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade"
2023-07-08 23:27:46,252 INFO MainThread:16924 [wandb_init.py:init():787] starting run threads in backend
2023-07-08 23:27:46,495 INFO MainThread:16924 [wandb_run.py:_console_start():2158] atexit reg
2023-07-08 23:27:46,495 INFO MainThread:16924 [wandb_run.py:_redirect():2013] redirect: SettingsConsole.WRAP_RAW
2023-07-08 23:27:46,495 INFO MainThread:16924 [wandb_run.py:_redirect():2078] Wrapping output streams.
2023-07-08 23:27:46,495 INFO MainThread:16924 [wandb_run.py:_redirect():2103] Redirects installed.
2023-07-08 23:27:46,496 INFO MainThread:16924 [wandb_init.py:init():829] run started, returning control to user process
2023-07-08 23:27:46,496 INFO MainThread:16924 [wandb_config.py:__setitem__():151] config set learning_rate = 0.0001 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1474039d78e0>>
2023-07-08 23:27:46,496 INFO MainThread:16924 [wandb_run.py:_config_callback():1286] config_cb learning_rate 0.0001 None
2023-07-08 23:27:46,496 INFO MainThread:16924 [wandb_config.py:__setitem__():151] config set optimizer = AdamW - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1474039d78e0>>
2023-07-08 23:27:46,496 INFO MainThread:16924 [wandb_run.py:_config_callback():1286] config_cb optimizer AdamW None
2023-07-08 23:30:16,662 INFO MainThread:16924 [wandb_run.py:_finish():1893] finishing run deep-lab-/Unconditional Landscapes/humans
2023-07-08 23:30:16,662 INFO MainThread:16924 [wandb_run.py:_atexit_cleanup():2127] got exitcode: 1
2023-07-08 23:30:16,662 INFO MainThread:16924 [wandb_run.py:_restore():2110] restore
2023-07-08 23:30:16,663 INFO MainThread:16924 [wandb_run.py:_restore():2116] restore done
2023-07-08 23:30:23,445 INFO MainThread:16924 [wandb_run.py:_footer_history_summary_info():3469] rendering history
2023-07-08 23:30:23,445 INFO MainThread:16924 [wandb_run.py:_footer_history_summary_info():3501] rendering summary
2023-07-08 23:30:23,450 INFO MainThread:16924 [wandb_run.py:_footer_sync_info():3428] logging synced files
wandb/run-20230708_232745-humans/run-humans.wandb 0 → 100644
+ 0
− 0
View file @ 934372c0
File added
wandb/run-20230708_233359-humans/files/config.yaml 0 → 100644
+ 37
− 0
View file @ 934372c0
wandb_version: 1
_wandb:
desc: null
value:
python_version: 3.10.4
cli_version: 0.15.3
framework: torch
is_jupyter_run: false
is_kaggle_kernel: false
start_time: 1688852039.597313
t:
1:
- 1
- 41
- 55
2:
- 1
- 41
- 55
3:
- 2
- 5
- 13
- 14
- 19
- 23
4: 3.10.4
5: 0.15.3
8:
- 5
optimizer:
desc: null
value: AdamW
learning_rate:
desc: null
value: 0.0001
wandb/run-20230708_233359-humans/files/wandb-summary.json 0 → 100644
+ 1
− 0
View file @ 934372c0
{"_step": 8568, "batch": 843, "epoch": 9, "_runtime": 3589.605794429779, "_timestamp": 1688855478.0541804, "learning_rate": 9.979864405611205e-05, "loss": 0.006437242031097412, "running_loss": 0.012675670010337863, "_wandb": {"runtime": 3588}}
\ No newline at end of file
wandb/run-20230708_233359-humans/logs/debug-internal.log 0 → 100644
+ 38453
− 0
View file @ 934372c0
Source diff could not be displayed: it is too large. Options to address this: view the blob.
wandb/run-20230708_233359-humans/logs/debug.log 0 → 100644
+ 39
− 0
View file @ 934372c0
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Current SDK version is 0.15.3
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Configure stats pid to 232873
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Loading settings from /home/id929844/.config/wandb/settings
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Loading settings from /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/settings
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'main.py', 'program': '/rwthfs/rz/cluster/home/id929844/diffusion_project/main.py'}
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_init.py:_log_setup():507] Logging user logs to /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_233359-humans/logs/debug.log
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_init.py:_log_setup():508] Logging internal logs to /rwthfs/rz/cluster/home/id929844/diffusion_project/wandb/run-20230708_233359-humans/logs/debug-internal.log
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_init.py:init():547] calling init triggers
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
config: {}
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_init.py:init():596] starting backend
2023-07-08 23:33:59,577 INFO MainThread:232873 [wandb_init.py:init():600] setting up manager
2023-07-08 23:33:59,584 INFO MainThread:232873 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
2023-07-08 23:33:59,596 INFO MainThread:232873 [wandb_init.py:init():606] backend started and connected
2023-07-08 23:33:59,604 INFO MainThread:232873 [wandb_init.py:init():700] updated telemetry
2023-07-08 23:33:59,689 INFO MainThread:232873 [wandb_init.py:init():737] communicating run to backend with 60.0 second timeout
2023-07-08 23:34:00,210 INFO MainThread:232873 [wandb_init.py:init():780] run resumed
2023-07-08 23:34:00,237 INFO MainThread:232873 [wandb_run.py:_on_init():2177] communicating current version
2023-07-08 23:34:00,307 INFO MainThread:232873 [wandb_run.py:_on_init():2186] got version response upgrade_message: "wandb version 0.15.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade"
2023-07-08 23:34:00,307 INFO MainThread:232873 [wandb_init.py:init():787] starting run threads in backend
2023-07-08 23:34:00,355 INFO MainThread:232873 [wandb_run.py:_console_start():2158] atexit reg
2023-07-08 23:34:00,356 INFO MainThread:232873 [wandb_run.py:_redirect():2013] redirect: SettingsConsole.WRAP_RAW
2023-07-08 23:34:00,356 INFO MainThread:232873 [wandb_run.py:_redirect():2078] Wrapping output streams.
2023-07-08 23:34:00,356 INFO MainThread:232873 [wandb_run.py:_redirect():2103] Redirects installed.
2023-07-08 23:34:00,356 INFO MainThread:232873 [wandb_init.py:init():829] run started, returning control to user process
2023-07-08 23:34:00,356 INFO MainThread:232873 [wandb_config.py:__setitem__():151] config set learning_rate = 0.0001 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x150015bcbb50>>
2023-07-08 23:34:00,356 INFO MainThread:232873 [wandb_run.py:_config_callback():1286] config_cb learning_rate 0.0001 None
2023-07-08 23:34:00,357 INFO MainThread:232873 [wandb_config.py:__setitem__():151] config set optimizer = AdamW - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x150015bcbb50>>
2023-07-08 23:34:00,357 INFO MainThread:232873 [wandb_run.py:_config_callback():1286] config_cb optimizer AdamW None
2023-07-09 00:31:19,283 INFO MainThread:232873 [wandb_run.py:_finish():1893] finishing run deep-lab-/Unconditional Landscapes/humans
2023-07-09 00:31:19,283 INFO MainThread:232873 [wandb_run.py:_atexit_cleanup():2127] got exitcode: 1
2023-07-09 00:31:19,283 INFO MainThread:232873 [wandb_run.py:_restore():2110] restore
2023-07-09 00:31:19,284 INFO MainThread:232873 [wandb_run.py:_restore():2116] restore done
2023-07-09 00:31:22,244 INFO MainThread:232873 [wandb_run.py:_footer_history_summary_info():3469] rendering history
2023-07-09 00:31:22,244 INFO MainThread:232873 [wandb_run.py:_footer_history_summary_info():3501] rendering summary
2023-07-09 00:31:22,260 INFO MainThread:232873 [wandb_run.py:_footer_sync_info():3428] logging synced files
wandb/run-20230708_233359-humans/run-humans.wandb 0 → 100644
+ 0
− 0
View file @ 934372c0
File added
wandb/run-20230709_160458-humans/files/config.yaml 0 → 100644
+ 37
− 0
View file @ 934372c0
wandb_version: 1
_wandb:
desc: null
value:
python_version: 3.10.4
cli_version: 0.15.3
framework: torch
is_jupyter_run: false
is_kaggle_kernel: false
start_time: 1688911498.388557
t:
1:
- 1
- 41
- 55
2:
- 1
- 41
- 55
3:
- 2
- 5
- 13
- 14
- 19
- 23
4: 3.10.4
5: 0.15.3
8:
- 5
optimizer:
desc: null
value: AdamW
learning_rate:
desc: null
value: 0.0001
wandb/run-20230709_160458-humans/files/wandb-summary.json 0 → 100644
+ 1
− 0
View file @ 934372c0
{"loss": 0.01867721416056156, "_runtime": 4059.905727148056, "_timestamp": 1688911967.4497662, "running_loss": 0.047499893636495684, "learning_rate": 9.999798939653173e-05, "_step": 9413, "batch": 843, "epoch": 0, "_wandb": {"runtime": 4056}}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment