update flexibility with diff input shape, output directories, readme

evaluation/eval_full/evaluate_full.py

@@ -10,6 +10,7 @@ from torchvision.models import resnet50
 from kNN import *
 from metrics import *
 
+
 if __name__ == '__main__':
 
     #device = "mps" if torch.backends.mps.is_available() else "cpu"
@@ -23,8 +24,10 @@ if __name__ == '__main__':
                         help='path to generated images', type=str)
     parser.add_argument('-d', '--data', nargs='?', const='lhq', default='lhq',
                         help='choose between "lhq" and "face" dataset', type=str)
-    parser.add_argument('-a', '--arch', nargs='?', const='cnn', default='cnn', 
-                        help='choose between "clip" and "cnn", default "cnn"', type=str)
+    parser.add_argument('--size', nargs='?', const=128, default=128,
+                        help='resolution of image the model was trained on, default 128 (int)', type=int)
+    parser.add_argument('-a', '--arch', nargs='?', const='clip', default='clip', 
+                        help='choose between "clip" and "cnn", default "clip"', type=str)
     parser.add_argument('-m', '--mode', nargs='?', const='kNN', default='kNN', 
                         help='choose between "kNN" and "pairs" for closest_pairs, default "kNN"', type=str)
     parser.add_argument('-k', '--k', nargs='?', const=3, default=3,
@@ -46,22 +49,25 @@ if __name__ == '__main__':
     sample = args['sample']
     name_appendix = args['name']
     fid_bool = args['fid']
+    size = args['size']
     print('Start')
+    output_path = Path(os.path.join(os.getcwd(),'output'))
+    if not output_path.is_dir():
+        os.mkdir(output_path)
     txt_filename = 'output/evaluation_' + dataset + '_' + arch + '_' + mode + '-' + name_appendix + '.txt'
     with open(txt_filename, 'w') as f:
         f.write(f'Path to real images: {path_to_real_images}\n')
         f.write(f'Path to generated images: {path_to_generated_images}\n')
-        f.write(f'Experiment on {dataset} dataset\n')
+        f.write(f'Experiment on {dataset} dataset with images of resolution {size}x{size}\n')
         f.write(f'Using {arch} model to extract features\n')
         f.write(f'Plot of {mode} on {sample} samples\n')
+        f.write(f'Quantitative metrics computed: {fid_bool}\n')
+        
     
     # load data
     path_to_training_images = os.path.join(path_to_real_images, 'train')
     path_to_test_images = os.path.join(path_to_real_images, 'test')
     if fid_bool == 'yes':
-        # load data
-        #path_to_training_images = os.path.join(path_to_real_images, 'train')
-        #path_to_test_images = os.path.join(path_to_real_images, 'test')
 
         # metrics eval
         eval_images = image_to_tensor(path_to_test_images)
@@ -92,23 +98,19 @@ if __name__ == '__main__':
     # kNN-based eval
     if dataset == 'lhq':
         print('Dataset ', dataset)
-        #pth = '/Users/roy/Desktop/Workspace/RWTH/SoSe 2023/Deep Learning Lab/DLL_vsc/data/features/lhq_features'
         pth = '/home/wn455752/repo/evaluation/features/lhq'
         # load pretrained ResNet50 
         if arch == 'cnn':
-            #path_to_pretrained_weights = '/Users/roy/Desktop/Workspace/RWTH/SoSe 2023/Deep Learning Lab/DLL_vsc/pretrained/resnet50_places365_pretrained/resnet50_places365_weights.pth'
-            print('loading model...')
+            print('Loading pretrained ResNet50...')
             path_to_pretrained_weights = '/home/wn455752/repo/evaluation/pretrained/resnet50_places365_pretrained/resnet50_places365_weights.pth'
-            print('loading weights...')
             weights = torch.load(path_to_pretrained_weights)
             model = resnet50().to(device)
-            print('initializing model with pretrained weights')
             model.load_state_dict(weights)
-            transform = transforms.Compose([transforms.ToTensor(), 
-                                            transforms.Lambda(lambda x: x * 255)])
+            transform = transforms.Compose([transforms.ToTensor(),                       # transform PIL.Image to torch.Tensor
+                                            transforms.Lambda(lambda x: x * 255)])       # scale values to VGG input range
             with torch.no_grad():
                 model.eval()
-            print('checking for saved dataset features')
+            print('Checking for existing training dataset features...')
             # check for saved dataset features
             name_pth = Path(os.path.join(pth, 'resnet_features/real_name_list'))
             if name_pth.is_file():
@@ -116,22 +118,23 @@ if __name__ == '__main__':
                     real_names = pickle.load(fp)
             feature_pth = Path(os.path.join(pth, 'resnet_features/real_image_features.pt'))
             if name_pth.is_file():
-                print('Loading ResNet features of real images...')
+                print('Loading existing training dataset features...')
                 real_features = torch.load(feature_pth, map_location="cpu")
                 real_features = real_features.to(device)
                 feature_flag = True
         # load CLIP
         elif arch == 'clip':
-            print('loading model...')
+            print('Loading pretrained CLIP...')
             model, transform = clip.load("ViT-B/32", device=device)
             # check for saved dataset features
+            print('Checking for existing training dataset features...')
             name_pth = Path(os.path.join(pth, 'clip_features/real_name_list'))
             if name_pth.is_file():
                 with open(name_pth, 'rb') as fp:
                     real_names = pickle.load(fp)
             feature_pth = Path(os.path.join(pth, 'clip_features/real_image_features.pt'))
             if name_pth.is_file():
-                print('Loading CLIP features of real images...')
+                print('Loading existing training dataset features...')
                 real_features = torch.load(feature_pth, map_location="cpu")
                 real_features = real_features.to(device)
                 feature_flag = True
@@ -140,45 +143,45 @@ if __name__ == '__main__':
 
     elif dataset == 'faces':
         print('Dataset ', dataset)
-        #pth = '/Users/roy/Desktop/Workspace/RWTH/SoSe 2023/Deep Learning Lab/DLL_vsc/data/features/face_features'
         pth = '/home/wn455752/repo/evaluation/features/faces' 
         # load pretrained VGGFace
         if arch == 'cnn':
-            print('loading model...')
-            #path_to_pretrained_weights = '/Users/roy/Desktop/Workspace/RWTH/SoSe 2023/Deep Learning Lab/DLL_vsc/pretrained/vggface_pretrained/VGG_FACE.t7'
+            print('Loading pretrained VGGFace...')
             path_to_pretrained_weights = '/home/wn455752/repo/evaluation/pretrained/vggface_pretrained/VGG_FACE.t7'
             model = VGG_16().to(device)
             model.load_weights(path=path_to_pretrained_weights)
-            transform = transforms.Compose([transforms.ToTensor(),
-                                        transforms.Resize((224,224)),
-                                        transforms.Lambda(lambda x: x * 255)])
+            transform = transforms.Compose([transforms.ToTensor(),                   # transform PIL.Image to torch.Tensor
+                                        transforms.Resize((224,224)),                # resize to VGG input shape
+                                        transforms.Lambda(lambda x: x * 255)])       # scale values to VGG input range
             with torch.no_grad():
                 model.eval()
             
             # check for saved dataset features
+            print('Checking for existing training dataset features...')
             name_pth = Path(os.path.join(pth, 'vggface_features/real_name_list'))
             if name_pth.is_file():
                 with open(name_pth, 'rb') as fp:
                     real_names = pickle.load(fp)
             feature_pth = Path(os.path.join(pth, 'vggface_features/real_image_features.pt'))
             if name_pth.is_file():
-                print('Loading VGGFace features of real images...')
+                print('Loading existing training dataset features...')
                 real_features = torch.load(feature_pth, map_location="cpu")
                 real_features = real_features.to(device)
                 feature_flag = True
 
         # load CLIP
         elif arch == 'clip':
-            print('loading model...')
+            print('Loading pretrained CLIP...')
             model, transform = clip.load("ViT-B/32", device=device)
             # check for saved dataset features
+            print('Checking for existing training dataset features...')
             name_pth = Path(os.path.join(pth, 'clip_features/real_name_list'))
             if name_pth.is_file():
                 with open(name_pth, 'rb') as fp:
                     real_names = pickle.load(fp)
             feature_pth = Path(os.path.join(pth, 'clip_features/real_image_features.pt'))
             if name_pth.is_file():
-                print('Loading CLIP features of real images...')
+                print('Loading existing training dataset features...')
                 real_features = torch.load(feature_pth, map_location="cpu")
                 real_features = real_features.to(device)
                 feature_flag = True
@@ -186,7 +189,7 @@ if __name__ == '__main__':
     knn = kNN()
     # get images
     if not feature_flag:
-        print('Collecting real images...')
+        print('Collecting training images...')
         real_names, real_tensor = knn.get_images(path_to_training_images, transform)
         with open(name_pth, 'wb') as fp:
             pickle.dump(real_names, fp)
@@ -195,7 +198,7 @@ if __name__ == '__main__':
 
     # extract features
     if not feature_flag:
-        print('Extracting features from real images...')
+        print('Extracting features from training images...')
         real_features = knn.feature_extractor(real_tensor, model, device)
         torch.save(real_features, feature_pth)
     print('Extracting features from generated images...')
@@ -206,7 +209,6 @@ if __name__ == '__main__':
     else:
         sample_size = int(sample)
 
-
     if mode == 'kNN':
         print('Finding kNNs...')
         knn.kNN(real_names, generated_names, 
@@ -214,11 +216,14 @@ if __name__ == '__main__':
             path_to_training_images, path_to_generated_images, 
             k=k_kNN, 
             sample=sample_size, 
+            size=size,
             name_appendix=name_appendix)
     elif mode == 'pairs':
+        print('Finding closest pairs...')
         knn.nearest_neighbor(real_names, generated_names, 
                          real_features, generated_features, 
                          path_to_training_images, path_to_generated_images, 
                          sample=sample_size, 
+                         size=size,
                          name_appendix=name_appendix)
     print('Finish!')

evaluation/eval_full/evaluation_readme.md

@@ -3,6 +3,7 @@
 We conduct two types of evaluation - qualitative and quantitative.
 
 ### Quantitative evaluations -
+<pre>
 Quantitative evaluations are carried out to compare different backbone architectures of our unconditional diffusion model. 
 A set of 10,000 generated samples from each model variant is compared with the test set of the real dataset. 
 These evaluations include - 
@@ -10,36 +11,42 @@ These evaluations include -
     2. Inception score 
     3. Clean FID score (with CLIP) 
     4. FID infinity and IS infinity scores 
+</pre>
 
 ### Qualitative evaluations -
-The aim of this set of evaluations is to qualitatively inspect whether our model has overfit to the training images. For this,
-the entire set of 10,000 generated samples from the best performing model from quanititative evaluation is compared with the
-training set of the real dataset. Additionally, the quality check is also done on a hand-selected subset of best generations.
+<pre>
+The aim of this set of evaluations is to qualitatively inspect whether our model has overfit to the training images. 
+For this, the entire set of 10,000 generated samples from the best performing model from quanititative evaluation is 
+compared with the training set of the real dataset. 
+Additionally, the quality check is also done on a hand-selected subset of best generations. 
 
-The comparison is implemented as MSE values between features of the generated and training samples. The features are extracted
-by using a pretrained model (ResNet50-Places365/VGGFace or CLIP). Based on the MSE scores we compute -
+The comparison is implemented as MSE values between features of the generated and training samples. The features are 
+extracted by using a pretrained model (ResNet50-Places365/VGGFace or CLIP). Based on the MSE scores we compute - 
     1. kNN - plot the k nearest neighbors of the generated samples 
     2. Closest pairs - plot the top pairs with smallest MSE value 
+</pre>
 
-
+### Argumnets - 
+<pre>
 Execution starts with evaluate_full.py file. Input arguments are - 
-
-* -rp, --realpath : Path to real images (string)
-* -gp, --genpath  : Path to generated images (string)
-* -d, --data      : Choose between 'lhq' (for LHQ landscape dataset) and 'faces' (for CelebAHQ faces dataset). 
-                    Default = 'lhq' (string)
-* -a, --arch      : Choose between 'cnn' and 'clip'. Chosen pretrained model is used to extract features from the images.
+</pre>
+* <pre>-rp, --realpath : Path to real images (string) </pre>
+* <pre>-gp, --genpath  : Path to generated images (string) </pre>
+* <pre>-d, --data      : Choose between 'lhq' (for LHQ landscape dataset) and 'faces' (for CelebAHQ faces dataset). 
+                         Default = 'lhq' (string)</pre>
+* <pre>--size          : Resolution of images the model was trained on, default 128 (int) </pre>                  
+* <pre>-a, --arch      : Choose between 'cnn' and 'clip'. Chosen pretrained model is used to extract features from the images. </pre>
                          If 'cnn' is selected, for LHQ dataset the model is a ResNet50 pretrained on Places365 dataset and for 
-                    CelebAHQ dataset the model is a pretrained VGGFace. Default = 'cnn' (string)
-* -m, --mode      : Choose between 'kNN' and 'pairs' (for closest pairs), default = 'kNN' (string)
-* -k, --k         : k value for kNN, default = 3 (int)
-* -s, --sample    : Choose between an int and 'all'. If mode is 'kNN', plot kNN for this many samples (first s samples 
+                         CelebAHQ dataset the model is a pretrained VGGFace. Not relevant in computing FID, IS scores. Default = 'clip' (string) </pre>
+* <pre>-m, --mode      : Choose between 'kNN' and 'pairs' (for closest pairs), default = 'kNN' (string) </pre>
+* <pre>-k, --k         : k value for kNN, default = 3 (int) </pre>
+* <pre>-s, --sample    : Choose between an int and 'all'. If mode is 'kNN', plot kNN for this many samples (first s samples 
                          in the directory of generated images). If mode is 'pairs', plot the top s closest pairs from entire 
-                    directory of generated images. Default 10 (int or 'all')
-* -n, --name      : Name appendix (string)
-* --fid           : Choose between 'yes' and 'no'. Compute FID, Inception score and upgraded FID scores. Default 'no' (string)   
-
+                         directory of generated images. Default 10 (int or 'all') </pre>
+* <pre>-n, --name      : Name appendix (string) </pre>
+* <pre>--fid           : Choose between 'yes' and 'no'. Compute FID, Inception score and upgraded FID scores. Default 'no' (string)   </pre>
 
+<pre>
 Path to real images leads to a directory with two sub-directories - train and test.
 
 data 
@@ -50,8 +57,9 @@ data
 |    |_ train 
 |    |_ test 
 
-CLIP and CNN (ResNet50 or VGGFace) features of training images are saved after the first execution. This alleviates the need
+CLIP and CNN (ResNet50 or VGGFace) features of training images are saved after the first execution. This alleviates the need \
 to recompute features of real images for different sets of generated samples.
+</pre>
 
 ### Links
 1. ResNet50 pretrained on Places365 - https://github.com/CSAILVision/places365

evaluation/eval_full/kNN.py

 import os
+from pathlib import Path
 import torch
 import torchvision.transforms as transforms
 from torch.utils.data import DataLoader
@@ -12,7 +13,7 @@ class kNN():
     def __init__(self):
         pass
 
-    def get_images(self, path, transform, *args, **kwargs):
+    def get_images(self, path, transform, size=128, *args, **kwargs):
         '''
         returns 
         names: list of filenames
@@ -30,8 +31,8 @@ class kNN():
                 filepath = os.path.join(path, file)
                 names.append(file)
                 im = Image.open(filepath)
-                if im.size[0] != 128:
-                    im = im.resize((128,128))              # DDPM was trained on 128x128 images
+                if im.size[0] != size:
+                    im = im.resize((size,size))              # DDPM was trained on 128x128 images
                 im = transform(im)  
                 images_list.append(im)
         
@@ -68,12 +69,14 @@ class kNN():
             real_features, generated_features, 
             path_to_real_images, path_to_generated_images, 
             k=3, 
-            sample=10, 
+            sample=10, size=128,
             name_appendix='',
             *args, **kwargs):
         '''
         creates a plot with (generated image: k nearest real images) pairs
         '''
+        if sample > 50:
+            print('Cannot plot for more than 50 samples! sample <= 50')
         fig, ax = plt.subplots(sample, k+1, figsize=((k+1)*3,sample*2))
 
         for i in range(len(generated_features)):
@@ -94,6 +97,8 @@ class kNN():
             # draw the k real images
             for idx in knn.indices:
                 im = Image.open(os.path.join(path_to_real_images, real_names[idx.item()]))
+                if im.size[0] != size:
+                    im = im.resize((size,size))
                 ax[i, j].imshow(im)
                 ax[i, j].set_xticks([])
                 ax[i, j].set_yticks([])
@@ -103,27 +108,32 @@ class kNN():
                 break
         
         # savefig
-        
+        output_path = Path(os.path.join(os.getcwd(),'output'))
+        if not output_path.is_dir():
+            os.mkdir(output_path)
         plot_name = f'{k}NN_{sample}_samples'
         if name_appendix != '':
-            plot_name = plot_name + name_appendix 
-        fig.savefig('output/' + plot_name + '.png')
+            plot_name = plot_name + '_' + name_appendix + '.png'
+        fig.savefig(os.path.join(output_path, plot_name))
 
     def nearest_neighbor(self, real_names, generated_names, 
                     real_features, generated_features, 
                     path_to_real_images, path_to_generated_images, 
-                    sample=10, 
+                    sample=10, size=128,
                     name_appendix='',
                     *args, **kwargs):
         
         print('Computing nearest neighbors...')
+        if sample > 50:
+            print('Cannot plot for more than 50 samples! sample <= 50')
         fig, ax = plt.subplots(sample, 2, figsize=(2*3,sample*2))
         nn_dict = OrderedDict()
         
         for i in range(len(generated_features)):
             # l2 norm of one generated feature and all real features
-            #dist = torch.linalg.vector_norm(real_features - generated_features[i], ord=2, dim=1)
-            dist = torch.norm(real_features - generated_features[i], dim=1, p=2)
+            #dist = torch.linalg.vector_norm(real_features - generated_features[i], ord=2, dim=1)   # no mps support
+            dist = torch.norm(real_features - generated_features[i], dim=1, p=2)                    # soon to be deprecated
+            
             # nearest neighbor of the generated image
             knn = dist.topk(1, largest=False)
             # insert to the dict: generated_image: (distance, index of the nearest neighbor)
@@ -145,13 +155,19 @@ class kNN():
             # draw the real image
             knn_score, real_img_idx = nn_dict_sorted[gen_names[i]]
             im = Image.open(os.path.join(path_to_real_images, real_names[real_img_idx]))
+            if im.size[0] != size:
+                im = im.resize((size,size))
             ax[i, 1].imshow(im)
             ax[i, 1].set_xticks([])
             ax[i, 1].set_yticks([])
             ax[i, 1].set_title(f'{real_names[real_img_idx][:-4]}, {knn_score:.2f}', fontsize=8)
                 
         #savefig
+        output_path = Path(os.path.join(os.getcwd(),'output'))
+        if not output_path.is_dir():
+            os.mkdir(output_path)
         plot_name = f'closest_pairs_top_{sample}'
         if name_appendix != '':
-            plot_name = plot_name + name_appendix
-        fig.savefig('output/' + plot_name + '.png')
+            plot_name = plot_name + '_' + name_appendix + '.png'
+        fig.savefig(os.path.join(output_path, plot_name))
\ No newline at end of file
+        
\ No newline at end of file