init

.gitattributes

@@ -29,3 +29,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

Data_custom_generator.py

@@ -0,0 +1,56 @@
+from skimage.io import imread
+from skimage.transform import resize
+import numpy as np
+import tensorflow as tf
+import keras
+from skimage.transform import resize
+from tf_record_utility import TFRecordUtility
+from configuration import DatasetName, DatasetType, \
+    AffectnetConf, D300wConf, W300Conf, InputDataSize, LearningConfig
+from numpy import save, load, asarray
+
+
+class CustomHeatmapGenerator(keras.utils.Sequence):
+
+    def __init__(self, is_single, image_filenames, label_filenames, batch_size, n_outputs, accuracy=100):
+        self.image_filenames = image_filenames
+        self.label_filenames = label_filenames
+        self.batch_size = batch_size
+        self.n_outputs = n_outputs
+        self.is_single = is_single
+        self.accuracy = accuracy
+
+    def __len__(self):
+        _len = np.ceil(len(self.image_filenames) // float(self.batch_size))
+        return int(_len)
+
+    def __getitem__(self, idx):
+        img_path = D300wConf.train_images_dir
+        tr_path_85 = D300wConf.train_hm_dir_85
+        tr_path_90 = D300wConf.train_hm_dir_90
+        tr_path_97 = D300wConf.train_hm_dir_97
+        tr_path = D300wConf.train_hm_dir
+
+        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]
+        batch_y = self.label_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]
+
+        img_batch = np.array([imread(img_path + file_name) for file_name in batch_x])
+
+        if self.is_single:
+            if self.accuracy == 85:
+                lbl_batch = np.array([load(tr_path_85 + file_name) for file_name in batch_y])
+            elif self.accuracy == 90:
+                lbl_batch = np.array([load(tr_path_90 + file_name) for file_name in batch_y])
+            elif self.accuracy == 97:
+                lbl_batch = np.array([load(tr_path_97 + file_name) for file_name in batch_y])
+            else:
+                lbl_batch = np.array([load(tr_path + file_name) for file_name in batch_y])
+
+            lbl_out_array = lbl_batch
+        else:
+            lbl_batch_85 = np.array([load(tr_path_85 + file_name) for file_name in batch_y])
+            lbl_batch_90 = np.array([load(tr_path_90 + file_name) for file_name in batch_y])
+            lbl_batch_97 = np.array([load(tr_path_97 + file_name) for file_name in batch_y])
+            lbl_out_array = [lbl_batch_85, lbl_batch_90, lbl_batch_97]
+
+        return img_batch, lbl_out_array

README.md

@@ -1,3 +1,150 @@
----
-license: mit
----
+	
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/facial-landmark-points-detection-using/face-alignment-on-cofw)](https://paperswithcode.com/sota/face-alignment-on-cofw?p=facial-landmark-points-detection-using)
+
+#
+Facial Landmark Points Detection Using Knowledge Distillation-Based Neural Networks
+
+
+
+#### Link to the paper:
+Google Scholar:
+https://scholar.google.com/citations?view_op=view_citation&hl=en&user=96lS6HIAAAAJ&citation_for_view=96lS6HIAAAAJ:zYLM7Y9cAGgC
+
+Elsevier:
+https://www.sciencedirect.com/science/article/pii/S1077314221001582
+
+Arxiv:
+https://arxiv.org/abs/2111.07047
+
+#### Link to the paperswithcode.com:
+https://paperswithcode.com/paper/facial-landmark-points-detection-using
+
+```diff
+@@plaese STAR the repo if you like it.@@
+```
+
+```
+Please cite this work as:
+
+     @article{fard2022facial,
+  title={Facial landmark points detection using knowledge distillation-based neural networks},
+  author={Fard, Ali Pourramezan and Mahoor, Mohammad H},
+  journal={Computer Vision and Image Understanding},
+  volume={215},
+  pages={103316},
+  year={2022},
+  publisher={Elsevier}
+}
+
+```
+
+## Introduction
+Facial landmark detection is a vital step for numerous facial image analysis applications. Although some deep learning-based methods have achieved good performances in this task, they are often not suitable for running on mobile devices. Such methods rely on networks with many parameters, which makes the training and inference time-consuming. Training lightweight neural networks such as MobileNets are often challenging, and the models might have low accuracy. Inspired by knowledge distillation (KD), this paper presents a novel loss function to train a lightweight Student network (e.g., MobileNetV2) for facial landmark detection. We use two Teacher networks, a Tolerant-Teacher and a Tough-Teacher in conjunction with the Student network. The Tolerant-Teacher is trained using Soft-landmarks created by active shape models, while the Tough-Teacher is trained using the ground truth (aka Hard-landmarks) landmark points. To utilize the facial landmark points predicted by the Teacher networks, we define an Assistive Loss (ALoss) for each Teacher network. Moreover, we define a loss function called KD-Loss that utilizes the facial landmark points predicted by the two pre-trained Teacher networks (EfficientNet-b3) to guide the lightweight Student network towards predicting the Hard-landmarks. Our experimental results on three challenging facial datasets show that the proposed architecture will result in a better-trained Student network that can extract facial landmark points with high accuracy.
+
+
+##Architecture
+
+We train the Tough-Teacher, and the Tolerant-Teacher networks independently using the Hard-landmarks and the Soft-landmarks respectively utilizing the L2 loss:
+
+![teacher_arch](https://github.com/aliprf/KD-Loss/blob/master/samples/teacher_arch-1.jpg?raw=true)
+
+
+Proposed KD-based architecture for training the Student network. KDLoss uses the knowledge of the previously trained Teacher networks by utilizing the assistive loss functions ALossT ou and ALossT ol, to improve the performance the face alignment task:
+
+![general_framework](https://github.com/aliprf/KD-Loss/blob/master/samples/general_framework-1.jpg?raw=true)
+
+
+## Evaluation
+
+Following are some samples in order to show the visual performance of KD-Loss on 300W, COFW and WFLW datasets:
+
+300W:
+![KD_300W_samples](https://github.com/aliprf/KD-Loss/blob/master/samples/KD_300W_samples-1.jpg?raw=true)
+
+COFW:
+![KD_cofw_samples](https://github.com/aliprf/KD-Loss/blob/master/samples/KD_cofw_samples-1.jpg?raw=true)
+
+WFLW:
+![KD_WFLW_samples](https://github.com/aliprf/KD-Loss/blob/master/samples/KD_WFLW_samples-1.jpg?raw=true)
+
+----------------------------------------------------------------------------------------------------------------------------------
+## Installing the requirements
+In order to run the code you need to install python >= 3.5. 
+The requirements and the libraries needed to run the code can be installed using the following command:
+
+```
+  pip install -r requirements.txt
+```
+
+
+## Using the pre-trained models
+You can test and use the preetrained models using the following codes which are available in the test.py:
+The pretrained student model are also located in "models/students".
+  
+```
+  cnn = CNNModel()
+        model = cnn.get_model(arch=arch, input_tensor=None, output_len=self.output_len)
+
+        model.load_weights(weight_fname)
+
+        img = None # load a cropped image
+
+        image_utility = ImageUtility()
+        pose_predicted = []
+        image = np.expand_dims(img, axis=0)
+
+        pose_predicted = model.predict(image)[1][0]
+```
+
+
+## Training Network from scratch
+
+
+### Preparing Data
+Data needs to be normalized and saved in npy format. 
+
+### Training 
+
+### Training Teacher Networks: 
+
+The training implementation is located in teacher_trainer.py class. You can use the following code to start the training for the teacher networks:
+
+```
+  '''train Teacher Networks'''
+    trainer = TeacherTrainer(dataset_name=DatasetName.w300)
+    trainer.train(arch='efficientNet',weight_path=None)
+```
+
+### Training Student Networks: 
+After Training the teacher networks, you can use the trained teachers to train the student network. The implemetation of training of the student network is provided in teacher_trainer.py . You can use the following code to start the training for the student networks:
+
+```
+ st_trainer = StudentTrainer(dataset_name=DatasetName.w300, use_augmneted=True)
+    st_trainer.train(arch_student='mobileNetV2', weight_path_student=None,
+                     loss_weight_student=2.0,
+                     arch_tough_teacher='efficientNet', weight_path_tough_teacher='./models/teachers/ds_300w_ef_tou.h5',
+                     loss_weight_tough_teacher=1,
+                     arch_tol_teacher='efficientNet', weight_path_tol_teacher='./models/teachers/ds_300w_ef_tol.h5',
+                     loss_weight_tol_teacher=1)
+```
+
+
+
+```
+Please cite this work as:
+
+     @article{fard2022facial,
+  title={Facial landmark points detection using knowledge distillation-based neural networks},
+  author={Fard, Ali Pourramezan and Mahoor, Mohammad H},
+  journal={Computer Vision and Image Understanding},
+  volume={215},
+  pages={103316},
+  year={2022},
+  publisher={Elsevier}
+}
+
+```  
+
+```diff
+@@plaese STAR the repo if you like it.@@
+```

cnn_model.py

@@ -0,0 +1,94 @@
+from configuration import DatasetName, DatasetType, \
+    D300wConf, InputDataSize, LearningConfig
+
+import tensorflow as tf
+# tf.compat.v1.disable_eager_execution()
+
+from tensorflow import keras
+from skimage.transform import resize
+from keras.regularizers import l2
+
+# tf.logging.set_verbosity(tf.logging.ERROR)
+from tensorflow.keras.models import Model
+from tensorflow.keras.applications import mobilenet_v2, mobilenet, resnet50, densenet
+
+from tensorflow.keras.layers import Dense, MaxPooling2D, Conv2D, Flatten, \
+    BatchNormalization, Activation, GlobalAveragePooling2D, DepthwiseConv2D, Dropout, ReLU, Concatenate, Input, Conv2DTranspose
+
+from keras.callbacks import ModelCheckpoint
+from keras import backend as K
+
+from keras.optimizers import Adam
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+from keras.callbacks import CSVLogger
+from datetime import datetime
+
+import cv2
+import os.path
+from keras.utils.vis_utils import plot_model
+from scipy.spatial import distance
+import scipy.io as sio
+
+import efficientnet.tfkeras as efn
+
+
+class CNNModel:
+    def get_model(self, arch, input_tensor, output_len,
+                  inp_shape=[InputDataSize.image_input_size, InputDataSize.image_input_size, 3],
+                  weight_path=None):
+        if arch == 'efficientNet':
+            model = self.create_efficientNet(inp_shape=inp_shape, input_tensor=input_tensor, output_len=output_len)
+        elif arch == 'mobileNetV2':
+            model = self.create_MobileNet(inp_shape=inp_shape, inp_tensor=input_tensor)
+        return model
+
+    def create_MobileNet(self, inp_shape, inp_tensor):
+
+        mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
+                                                   alpha=1.0,
+                                                   include_top=True,
+                                                   weights=None,
+                                                   input_tensor=inp_tensor,
+                                                   pooling=None)
+
+        inp = mobilenet_model.input
+        out_landmarks = mobilenet_model.get_layer('O_L').output
+        revised_model = Model(inp, [out_landmarks])
+        model_json = revised_model.to_json()
+        with open("mobileNet_v2_stu.json", "w") as json_file:
+            json_file.write(model_json)
+        return revised_model
+
+    def create_efficientNet(self, inp_shape, input_tensor, output_len, is_teacher=True):
+        if is_teacher:  # for teacher we use a heavier network
+            eff_net = efn.EfficientNetB3(include_top=True,
+                                         weights=None,
+                                         input_tensor=None,
+                                         input_shape=[InputDataSize.image_input_size, InputDataSize.image_input_size,
+                                                      3],
+                                         pooling=None,
+                                         classes=output_len)
+        else:  # for student we use the small network
+            eff_net = efn.EfficientNetB0(include_top=True,
+                                         weights=None,
+                                         input_tensor=None,
+                                         input_shape=inp_shape,
+                                         pooling=None,
+                                         classes=output_len)  # or weights='noisy-student'
+
+        eff_net.layers.pop()
+        inp = eff_net.input
+
+        x = eff_net.get_layer('top_activation').output
+        x = GlobalAveragePooling2D()(x)
+        x = keras.layers.Dropout(rate=0.5)(x)
+        output = Dense(output_len, activation='linear', name='out')(x)
+
+        eff_net = Model(inp, output)
+
+        eff_net.summary()
+
+        return eff_net
+

configuration.py

@@ -0,0 +1,77 @@
+class DatasetName:
+    w300 = 'w300'
+    cofw = 'cofw'
+    wflw = 'wflw'
+
+class DatasetType:
+    data_type_train = 0
+    data_type_validation = 1
+    data_type_test = 2
+
+class LearningConfig:
+    batch_size = 70
+    epochs = 150
+
+class InputDataSize:
+    image_input_size = 224
+
+class WflwConf:
+    Wflw_prefix_path = './data/wflw/'  # --> local
+
+    '''     augmented version'''
+    augmented_train_pose = Wflw_prefix_path + 'training_set/augmented/pose/'
+    augmented_train_annotation = Wflw_prefix_path + 'training_set/augmented/annotations/'
+    augmented_train_atr = Wflw_prefix_path + 'training_set/augmented/atrs/'
+    augmented_train_image = Wflw_prefix_path + 'training_set/augmented/images/'
+    augmented_train_tf_path = Wflw_prefix_path + 'training_set/augmented/tf/'
+    '''     original version'''
+    no_aug_train_annotation = Wflw_prefix_path + 'training_set/no_aug/annotations/'
+    no_aug_train_atr = Wflw_prefix_path + 'training_set/no_aug/atrs/'
+    no_aug_train_pose = Wflw_prefix_path + 'training_set/no_aug/pose/'
+    no_aug_train_image = Wflw_prefix_path + 'training_set/no_aug/images/'
+    no_aug_train_tf_path = Wflw_prefix_path + 'training_set/no_aug/tf/'
+
+    orig_number_of_training = 7500
+    orig_number_of_test = 2500
+
+    augmentation_factor = 4  # create . image from 1
+    augmentation_factor_rotate = 15  # create . image from 1
+    num_of_landmarks = 98
+
+class CofwConf:
+    Cofw_prefix_path = './data/cofw/'  # --> local
+
+    augmented_train_pose = Cofw_prefix_path + 'training_set/augmented/pose/'
+    augmented_train_annotation = Cofw_prefix_path + 'training_set/augmented/annotations/'
+    augmented_train_image = Cofw_prefix_path + 'training_set/augmented/images/'
+    augmented_train_tf_path = Cofw_prefix_path + 'training_set/augmented/tf/'
+
+    orig_number_of_training = 1345
+    orig_number_of_test = 507
+
+    augmentation_factor = 10
+    num_of_landmarks = 29
+
+
+class D300wConf:
+    w300w_prefix_path = './data/300W/'  # --> local
+
+    orig_300W_train = w300w_prefix_path + 'orig_300W_train/'
+    augmented_train_pose = w300w_prefix_path + 'training_set/augmented/pose/'
+    augmented_train_annotation = w300w_prefix_path + 'training_set/augmented/annotations/'
+    augmented_train_image = w300w_prefix_path + 'training_set/augmented/images/'
+    augmented_train_tf_path = w300w_prefix_path + 'training_set/augmented/tf/'
+
+    no_aug_train_annotation = w300w_prefix_path + 'training_set/no_aug/annotations/'
+    no_aug_train_pose = w300w_prefix_path + 'training_set/no_aug/pose/'
+    no_aug_train_image = w300w_prefix_path + 'training_set/no_aug/images/'
+    no_aug_train_tf_path = w300w_prefix_path + 'training_set/no_aug/tf/'
+
+    orig_number_of_training = 3148
+    orig_number_of_test_full = 689
+    orig_number_of_test_common = 554
+    orig_number_of_test_challenging = 135
+
+    augmentation_factor = 4  # create . image from 1
+    num_of_landmarks = 68
+

custom_Losses.py

@@ -0,0 +1,63 @@
+import math
+
+import numpy as np
+import tensorflow as tf
+
+# tf.compat.v1.disable_eager_execution()
+# tf.compat.v1.enable_eager_execution()
+
+from PIL import Image
+from tensorflow.keras import backend as K
+from scipy.spatial import distance
+
+from cnn_model import CNNModel
+from configuration import DatasetName, D300wConf, LearningConfig
+from image_utility import ImageUtility
+from pca_utility import PCAUtility
+
+print(tf.__version__)
+
+
+class Custom_losses:
+
+    def MSE(self, x_pr, x_gt):
+        return tf.losses.mean_squared_error(x_pr, x_gt)
+
+
+    def kd_loss(self, x_pr, x_gt, x_tough, x_tol, alpha_tough, alpha_mi_tough, alpha_tol, alpha_mi_tol,
+                main_loss_weight, tough_loss_weight, tol_loss_weight):
+        """km"""
+        '''los KD'''
+        # we revise teachers for reflection:
+        x_tough = x_gt + tf.sign(x_pr - x_gt) * tf.abs(x_tough - x_gt)
+        b_tough = x_gt + tf.sign(x_pr - x_gt) * tf.abs(x_tough - x_gt) * 0.15
+        x_tol = x_gt + tf.sign(x_pr - x_gt) * tf.abs(x_tol - x_gt)
+        b_tol = x_gt + tf.sign(x_pr - x_gt) * tf.abs(x_tol - x_gt) * 0.15
+        # Region A: from T -> +inf
+        tou_pos_map = tf.where(tf.sign(x_pr - x_tough) * tf.sign(x_tough - x_gt) > 0, alpha_tough, 0.0)
+        tou_neg_map = tf.where(tf.sign(x_tough - x_pr) * tf.sign(x_pr - b_tough) >= 0, alpha_mi_tough, 0.0)
+        # tou_red_map = tf.where(tf.sign(tf.abs(b_tough) - tf.abs(x_pr))*tf.sign(tf.abs(x_pr) - tf.abs(x_gt)) > 0, 0.1, 0.0)
+        tou_map = tou_pos_map + tou_neg_map  # + tou_red_map
+
+        tol_pos_map = tf.where(tf.sign(x_pr - x_tol) * tf.sign(x_tol - x_gt) > 0, alpha_tol, 0.0)
+        tol_neg_map = tf.where(tf.sign(x_tol - x_pr) * tf.sign(x_pr - b_tol) >= 0, alpha_mi_tol, 0.0)
+        # tol_red_map = tf.where(tf.sign(tf.abs(b_tol) - tf.abs(x_pr))*tf.sign(tf.abs(x_pr) - tf.abs(x_gt)) > 0, 0.1, 0.0)
+        tol_map = tol_pos_map + tol_neg_map  # + tou_red_map
+
+        '''calculate dif map for linear and non-linear part'''
+        low_diff_main_map = tf.where(tf.abs(x_gt - x_pr) <= tf.abs(x_gt - x_tol), 1.0, 0.0)
+        high_diff_main_map = tf.where(tf.abs(x_gt - x_pr) > tf.abs(x_gt - x_tol), 1.0, 0.0)
+
+        '''calculate loss'''
+        loss_main_high_dif = tf.reduce_mean(
+            high_diff_main_map * (tf.square(x_gt - x_pr) + (3 * tf.abs(x_gt - x_tol)) - tf.square(x_gt - x_tol)))
+        loss_main_low_dif = tf.reduce_mean(low_diff_main_map * (3 * tf.abs(x_gt - x_pr)))
+        loss_main = main_loss_weight * (loss_main_high_dif + loss_main_low_dif)
+
+        loss_tough_assist = tough_loss_weight * tf.reduce_mean(tou_map * tf.abs(x_tough - x_pr))
+        loss_tol_assist = tol_loss_weight * tf.reduce_mean(tol_map * tf.abs(x_tol - x_pr))
+
+        '''dif loss:'''
+        loss_total = loss_main + loss_tough_assist + loss_tol_assist
+
+        return loss_total, loss_main, loss_tough_assist, loss_tol_assist

image_utility.py

@@ -0,0 +1,585 @@
+import random
+import numpy as np
+
+import matplotlib
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
+
+import math
+from skimage.transform import warp, AffineTransform
+import cv2
+from scipy import misc
+from skimage.transform import rotate
+from PIL import Image
+from PIL import ImageOps
+from skimage.transform import resize
+from skimage import transform
+from skimage.transform import SimilarityTransform, AffineTransform
+import random
+
+class ImageUtility:
+
+    def random_rotate(self, _image, _label, file_name, fn):
+
+        xy_points, x_points, y_points = self.create_landmarks(landmarks=_label,
+                                                              scale_factor_x=1, scale_factor_y=1)
+        _image, _label = self.cropImg_2time(_image, x_points, y_points)
+
+        scale = (np.random.uniform(0.7, 1.3), np.random.uniform(0.7, 1.3))
+        # scale = (1, 1)
+
+        rot = np.random.uniform(-1 * 0.55, 0.55)
+        translation = (0, 0)
+        shear = 0
+
+        tform = AffineTransform(
+            scale=scale,  # ,
+            rotation=rot,
+            translation=translation,
+            shear=np.deg2rad(shear)
+        )
+
+        output_img = transform.warp(_image, tform.inverse, mode='symmetric')
+
+        sx, sy = scale
+        t_matrix = np.array([
+            [sx * math.cos(rot), -sy * math.sin(rot + shear), 0],
+            [sx * math.sin(rot), sy * math.cos(rot + shear), 0],
+            [0, 0, 1]
+        ])
+        landmark_arr_xy, landmark_arr_x, landmark_arr_y = self.create_landmarks(_label, 1, 1)
+        label = np.array(landmark_arr_x + landmark_arr_y).reshape([2, 68])
+        marging = np.ones([1, 68])
+        label = np.concatenate((label, marging), axis=0)
+
+        label_t = np.dot(t_matrix, label)
+        lbl_flat = np.delete(label_t, 2, axis=0).reshape([136])
+
+        t_label = self.__reorder(lbl_flat)
+
+        '''crop data: we add a small margin to the images'''
+        xy_points, x_points, y_points = self.create_landmarks(landmarks=t_label,
+                                                              scale_factor_x=1, scale_factor_y=1)
+        img_arr, points_arr = self.cropImg(output_img, x_points, y_points, no_padding=False)
+        # img_arr = output_img
+        # points_arr = t_label
+        '''resize image to 224*224'''
+        resized_img = resize(img_arr,
+                             (224, 224, 3),
+                             anti_aliasing=True)
+        dims = img_arr.shape
+        height = dims[0]
+        width = dims[1]
+        scale_factor_y = 224 / height
+        scale_factor_x = 224 / width
+
+        '''rescale and retrieve landmarks'''
+        landmark_arr_xy, landmark_arr_x, landmark_arr_y = \
+            self.create_landmarks(landmarks=points_arr,
+                                  scale_factor_x=scale_factor_x,
+                                  scale_factor_y=scale_factor_y)
+
+        if not(min(landmark_arr_x) < 0.0 or min(landmark_arr_y) < 0.0 or max(landmark_arr_x) > 224 or max(landmark_arr_y) > 224):
+
+            # self.print_image_arr(fn, resized_img, landmark_arr_x, landmark_arr_y)
+
+            im = Image.fromarray((resized_img * 255).astype(np.uint8))
+            im.save(str(file_name) + '.jpg')
+
+            pnt_file = open(str(file_name) + ".pts", "w")
+            pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"]
+            pnt_file.writelines(pre_txt)
+            points_txt = ""
+            for i in range(0, len(landmark_arr_xy), 2):
+                points_txt += str(landmark_arr_xy[i]) + " " + str(landmark_arr_xy[i + 1]) + "\n"
+
+            pnt_file.writelines(points_txt)
+            pnt_file.write("} \n")
+            pnt_file.close()
+
+        return t_label, output_img
+
+
+
+    def random_rotate_m(self, _image, _label_img, file_name):
+
+        rot = random.uniform(-80.9, 80.9)
+
+        output_img = rotate(_image, rot, resize=True)
+        output_img_lbl = rotate(_label_img, rot, resize=True)
+
+        im = Image.fromarray((output_img * 255).astype(np.uint8))
+        im_lbl = Image.fromarray((output_img_lbl * 255).astype(np.uint8))
+
+        im_m = ImageOps.mirror(im)
+        im_lbl_m = ImageOps.mirror(im_lbl)
+
+        im.save(str(file_name)+'.jpg')
+        # im_lbl.save(str(file_name)+'_lbl.jpg')
+
+        im_m.save(str(file_name) + '_m.jpg')
+        # im_lbl_m.save(str(file_name) + '_m_lbl.jpg')
+
+        im_lbl_ar = np.array(im_lbl)
+        im_lbl_m_ar = np.array(im_lbl_m)
+
+        self.__save_label(im_lbl_ar, file_name, np.array(im))
+        self.__save_label(im_lbl_m_ar, file_name+"_m", np.array(im_m))
+
+
+    def __save_label(self, im_lbl_ar, file_name, img_arr):
+
+        im_lbl_point = []
+        for i in range(im_lbl_ar.shape[0]):
+            for j in range(im_lbl_ar.shape[1]):
+                if im_lbl_ar[i, j] != 0:
+                    im_lbl_point.append(j)
+                    im_lbl_point.append(i)
+
+        pnt_file = open(str(file_name)+".pts", "w")
+
+        pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"]
+        pnt_file.writelines(pre_txt)
+        points_txt = ""
+        for i in range(0, len(im_lbl_point), 2):
+            points_txt += str(im_lbl_point[i]) + " " + str(im_lbl_point[i+1]) + "\n"
+
+        pnt_file.writelines(points_txt)
+        pnt_file.write("} \n")
+        pnt_file.close()
+
+        '''crop data: we add a small margin to the images'''
+        xy_points, x_points, y_points = self.create_landmarks(landmarks=im_lbl_point,
+                                                                       scale_factor_x=1, scale_factor_y=1)
+        img_arr, points_arr = self.cropImg(img_arr, x_points, y_points)
+
+        '''resize image to 224*224'''
+        resized_img = resize(img_arr,
+                             (224, 224, 3),
+                             anti_aliasing=True)
+        dims = img_arr.shape
+        height = dims[0]
+        width = dims[1]
+        scale_factor_y = 224 / height
+        scale_factor_x = 224 / width
+
+        '''rescale and retrieve landmarks'''
+        landmark_arr_xy, landmark_arr_x, landmark_arr_y = \
+            self.create_landmarks(landmarks=points_arr,
+                                           scale_factor_x=scale_factor_x,
+                                           scale_factor_y=scale_factor_y)
+
+        im = Image.fromarray((resized_img * 255).astype(np.uint8))
+        im.save(str(im_lbl_point[0])+'.jpg')
+        # self.print_image_arr(im_lbl_point[0], resized_img, landmark_arr_x, landmark_arr_y)
+
+
+    def augment(self, _image, _label):
+
+        # face = misc.face(gray=True)
+        #
+        # rotate_face = ndimage.rotate(_image, 45)
+        # self.print_image_arr(_label[0], rotate_face, [],[])
+
+        # hue_img = tf.image.random_hue(_image, max_delta=0.1)  # max_delta must be in the interval [0, 0.5].
+        # sat_img = tf.image.random_saturation(hue_img, lower=0.0, upper=3.0)
+        #
+        # sat_img = K.eval(sat_img)
+        #
+        _image = self.__noisy(_image)
+
+        shear = 0
+
+        # rot = 0.0
+        # scale = (1, 1)
+
+        rot = np.random.uniform(-1 * 0.009, 0.009)
+
+        scale = (random.uniform(0.9, 1.00), random.uniform(0.9, 1.00))
+
+        tform = AffineTransform(scale=scale, rotation=rot, shear=shear,
+                                translation=(0, 0))
+
+        output_img = warp(_image, tform.inverse, output_shape=(_image.shape[0], _image.shape[1]))
+
+        sx, sy = scale
+        t_matrix = np.array([
+            [sx * math.cos(rot), -sy * math.sin(rot + shear), 0],
+            [sx * math.sin(rot), sy * math.cos(rot + shear), 0],
+            [0, 0, 1]
+        ])
+        landmark_arr_xy, landmark_arr_x, landmark_arr_y = self.create_landmarks(_label, 1, 1)
+        label = np.array(landmark_arr_x + landmark_arr_y).reshape([2, 68])
+        marging = np.ones([1, 68])
+        label = np.concatenate((label, marging), axis=0)
+
+        label_t = np.dot(t_matrix, label)
+        lbl_flat = np.delete(label_t, 2, axis=0).reshape([136])
+
+        t_label = self.__reorder(lbl_flat)
+        return t_label, output_img
+
+    def __noisy(self, image):
+        noise_typ = random.randint(0, 3)
+        if noise_typ == 0 :#"gauss":
+            row, col, ch = image.shape
+            mean = 0
+            var = 0.1
+            sigma = var ** 0.5
+            gauss = np.random.normal(mean, sigma, (row, col, ch))
+            gauss = gauss.reshape(row, col, ch)
+            noisy = image + gauss
+            return noisy
+        elif noise_typ == 1 :# "s&p":
+            row, col, ch = image.shape
+            s_vs_p = 0.5
+            amount = 0.004
+            out = np.copy(image)
+            # Salt mode
+            num_salt = np.ceil(amount * image.size * s_vs_p)
+            coords = [np.random.randint(0, i - 1, int(num_salt))
+                      for i in image.shape]
+            out[coords] = 1
+
+            # Pepper mode
+            num_pepper = np.ceil(amount * image.size * (1. - s_vs_p))
+            coords = [np.random.randint(0, i - 1, int(num_pepper))
+                      for i in image.shape]
+            out[coords] = 0
+            return out
+
+        elif noise_typ == 2: #"speckle":
+            row, col, ch = image.shape
+            gauss = np.random.randn(row, col, ch)
+            gauss = gauss.reshape(row, col, ch)
+            noisy = image + image * gauss
+            return noisy
+        else:
+            return image
+
+    def __reorder(self, input_arr):
+        out_arr = []
+        for i in range(68):
+            out_arr.append(input_arr[i])
+            k = 68 + i
+            out_arr.append(input_arr[k])
+        return np.array(out_arr)
+
+    def print_image_arr_heat(self, k, image):
+        plt.figure()
+        plt.imshow(image)
+        implot = plt.imshow(image)
+        plt.axis('off')
+        plt.savefig('heat' + str(k) + '.png', bbox_inches='tight')
+        plt.clf()
+
+    def print_image_arr(self, k, image, landmarks_x, landmarks_y):
+        plt.figure()
+        plt.imshow(image)
+        implot = plt.imshow(image)
+
+        plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='black', s=20)
+        plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='white', s=15)
+        plt.axis('off')
+        plt.savefig('sss' + str(k) + '.png',  bbox_inches='tight')
+        # plt.show()
+        plt.clf()
+
+    def create_landmarks_from_normalized_original_img(self, img, landmarks, width, height, x_center, y_center, x1, y1, scale_x, scale_y):
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+
+        for j in range(0, len(landmarks), 2):
+            x = ((x_center - float(landmarks[j]) * width)*scale_x) + x1
+            y = ((y_center - float(landmarks[j + 1]) * height)*scale_y) + y1
+
+            landmark_arr_xy.append(x)
+            landmark_arr_xy.append(y)
+
+            landmark_arr_x.append(x)
+            landmark_arr_y.append(y)
+
+            img = cv2.circle(img, (int(x), int(y)), 2, (255, 14, 74), 2)
+            img = cv2.circle(img, (int(x), int(y)), 1, (0, 255, 255), 1)
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y, img
+
+
+    def create_landmarks_from_normalized(self, landmarks, width, height, x_center, y_center):
+
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+
+        for j in range(0, len(landmarks), 2):
+            x = x_center - float(landmarks[j]) * width
+            y = y_center - float(landmarks[j + 1]) * height
+
+            landmark_arr_xy.append(x)
+            landmark_arr_xy.append(y)  # [ x1, y1, x2,y2 ]
+
+            landmark_arr_x.append(x)  # [x1, x2]
+            landmark_arr_y.append(y)  # [y1, y2]
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y
+
+    def create_landmarks(self, landmarks, scale_factor_x=1, scale_factor_y=1):
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+        for j in range(0, len(landmarks), 2):
+
+            x = float(landmarks[j]) * scale_factor_x
+            y = float(landmarks[j + 1]) * scale_factor_y
+
+            landmark_arr_xy.append(x)
+            landmark_arr_xy.append(y)  # [ x1, y1, x2,y2 ]
+
+            landmark_arr_x.append(x)  # [x1, x2]
+            landmark_arr_y.append(y)  # [y1, y2]
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y
+
+    def create_landmarks_aflw(self, landmarks, scale_factor_x, scale_factor_y):
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+        for j in range(0, len(landmarks), 2):
+            if landmarks[j][0] == 1:
+                x = float(landmarks[j][1]) * scale_factor_x
+                y = float(landmarks[j][2]) * scale_factor_y
+
+                landmark_arr_xy.append(x)
+                landmark_arr_xy.append(y)  # [ x1, y1, x2,y2 ]
+
+                landmark_arr_x.append(x)  # [x1, x2]
+                landmark_arr_y.append(y)  # [y1, y2]
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y
+
+    def random_augmentation(self, lbl, img):
+        # a = random.randint(0, 1)
+        # if a == 0:
+        #     img, lbl = self.__add_margin(img, img.shape[0], lbl)
+
+        img, lbl = self.__add_margin(img, img.shape[0], lbl)
+
+        # else:
+        #     img, lbl = self.__negative_crop(img, lbl)
+
+        # i = random.randint(0, 2)
+        # if i == 0:
+        #     img, lbl = self.__rotate(img, lbl, 90, img.shape[0], img.shape[1])
+        # elif i == 1:
+        #     img, lbl = self.__rotate(img, lbl, 180, img.shape[0], img.shape[1])
+        # else:
+        #     img, lbl = self.__rotate(img, lbl, 270, img.shape[0], img.shape[1])
+
+        k = random.randint(0, 3)
+        if k > 0:
+            img = self.__change_color(img)
+
+        lbl = np.reshape(lbl, [136])
+        return lbl, img
+
+
+    def cropImg_2time(self, img, x_s, y_s):
+        min_x = max(0, int(min(x_s) - 150))
+        max_x = int(max(x_s) + 150)
+        min_y = max(0, int(min(y_s) - 150))
+        max_y = int(max(y_s) + 150)
+
+        crop = img[min_y:max_y, min_x:max_x]
+
+        new_x_s = []
+        new_y_s = []
+        new_xy_s = []
+
+        for i in range(len(x_s)):
+            new_x_s.append(x_s[i] - min_x)
+            new_y_s.append(y_s[i] - min_y)
+            new_xy_s.append(x_s[i] - min_x)
+            new_xy_s.append(y_s[i] - min_y)
+        return crop, new_xy_s
+
+    def cropImg(self, img, x_s, y_s, no_padding=False):
+        margin1 = random.randint(5, 20)
+        margin2 = random.randint(5, 20)
+        margin3 = random.randint(5, 20)
+        margin4 = random.randint(5, 20)
+
+        if no_padding:
+            min_x = max(0, int(min(x_s)))
+            max_x = int(max(x_s))
+            min_y = max(0, int(min(y_s)))
+            max_y = int(max(y_s))
+        else:
+            min_x = max(0, int(min(x_s) - margin1))
+            max_x = int(max(x_s) + margin2)
+            min_y = max(0, int(min(y_s) - margin3))
+            max_y = int(max(y_s) + margin4)
+
+        crop = img[min_y:max_y, min_x:max_x]
+
+        new_x_s = []
+        new_y_s = []
+        new_xy_s = []
+
+        for i in range(len(x_s)):
+            new_x_s.append(x_s[i] - min_x)
+            new_y_s.append(y_s[i] - min_y)
+            new_xy_s.append(x_s[i] - min_x)
+            new_xy_s.append(y_s[i] - min_y)
+
+        # imgpr.print_image_arr(k, crop, new_x_s, new_y_s)
+        # imgpr.print_image_arr_2(i, img, x_s, y_s, [min_x, max_x], [min_y, max_y])
+
+        return crop, new_xy_s
+
+    def __negative_crop(self, img, landmarks):
+
+        landmark_arr_xy, x_s, y_s = self.create_landmarks(landmarks, 1, 1)
+        min_x = img.shape[0] // random.randint(5, 15)
+        max_x = img.shape[0] - (img.shape[0] // random.randint(15, 20))
+        min_y = img.shape[0] // random.randint(5, 15)
+        max_y = img.shape[0] - (img.shape[0] // random.randint(15, 20))
+
+        crop = img[min_y:max_y, min_x:max_x]
+
+        new_x_s = []
+        new_y_s = []
+        new_xy_s = []
+
+        for i in range(len(x_s)):
+            new_x_s.append(x_s[i] - min_x)
+            new_y_s.append(y_s[i] - min_y)
+            new_xy_s.append(x_s[i] - min_x)
+            new_xy_s.append(y_s[i] - min_y)
+
+        # imgpr.print_image_arr(crop.shape[0], crop, new_x_s, new_y_s)
+        # imgpr.print_image_arr_2(crop.shape[0], crop, x_s, y_s, [min_x, max_x], [min_y, max_y])
+
+        return crop, new_xy_s
+
+    def __add_margin(self, img, img_w, lbl):
+        marging_width = img_w // random.randint(15, 20)
+        direction = random.randint(0, 4)
+
+        if direction == 1:
+            margings = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((img, margings), axis=1)
+
+        if direction == 2:
+            margings_1 = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((img, margings_1), axis=1)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img_w + int(marging_width), 3])
+            img = np.concatenate((img, margings_2), axis=0)
+
+        if direction == 3:  # need chane labels
+            margings_1 = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((margings_1, img), axis=1)
+            lbl = self.__transfer_lbl(int(marging_width), lbl, [1, 0])
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img_w + int(marging_width), 3])
+            img = np.concatenate((margings_2, img), axis=0)
+            lbl = self.__transfer_lbl(int(marging_width_1), lbl, [0, 1])
+
+        if direction == 4:  # need chane labels
+            margings_1 = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((margings_1, img), axis=1)
+            lbl = self.__transfer_lbl(int(marging_width), lbl, [1, 0])
+            img_w1 = img_w + int(marging_width)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img_w1, 3])
+            img = np.concatenate((margings_2, img), axis=0)
+            lbl = self.__transfer_lbl(int(marging_width_1), lbl, [0, 1])
+            img_w2 = img_w + int(marging_width_1)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_1 = np.random.random([img_w2, int(marging_width_1), 3])
+            img = np.concatenate((img, margings_1), axis=1)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img.shape[1], 3])
+            img = np.concatenate((img, margings_2), axis=0)
+
+        return img, lbl
+
+    def __void_image(self, img, img_w, ):
+        marging_width = int(img_w / random.randint(7, 16))
+        direction = random.randint(0, 1)
+        direction = 0
+        if direction == 0:
+            np.delete(img, 100, 1)
+            # img[:, 0:marging_width, :] = 0
+        elif direction == 1:
+            img[img_w - marging_width:img_w, :, :] = 0
+        if direction == 2:
+            img[:, img_w - marging_width:img_w, :] = 0
+
+        return img
+
+    def __change_color(self, img):
+        # color_arr = np.random.random([img.shape[0], img.shape[1]])
+        color_arr = np.zeros([img.shape[0], img.shape[1]])
+        axis = random.randint(0, 4)
+
+        if axis == 0:  # red
+            img_mono = img[:, :, 0]
+            new_img = np.stack([img_mono, color_arr, color_arr], axis=2)
+        elif axis == 1:  # green
+            img_mono = img[:, :, 1]
+            new_img = np.stack([color_arr, img_mono, color_arr], axis=2)
+        elif axis == 2:  # blue
+            img_mono = img[:, :, 1]
+            new_img = np.stack([color_arr, img_mono, color_arr], axis=2)
+        elif axis == 3:  # gray scale
+            img_mono = img[:, :, 0]
+            new_img = np.stack([img_mono, img_mono, img_mono], axis=2)
+        else:  # random noise
+            color_arr = np.random.random([img.shape[0], img.shape[1]])
+            img_mono = img[:, :, 0]
+            new_img = np.stack([img_mono, img_mono, color_arr], axis=2)
+
+        return new_img
+
+    def __rotate_origin_only(self, xy_arr, radians, xs, ys):
+        """Only rotate a point around the origin (0, 0)."""
+        rotated = []
+        for xy in xy_arr:
+            x, y = xy
+            xx = x * math.cos(radians) + y * math.sin(radians)
+            yy = -x * math.sin(radians) + y * math.cos(radians)
+            rotated.append([xx + xs, yy + ys])
+        return np.array(rotated)
+
+    def __rotate(self, img, landmark_old, degree, img_w, img_h):
+        landmark_old = np.reshape(landmark_old, [68, 2])
+
+        theta = math.radians(degree)
+
+        if degree == 90:
+            landmark = self.__rotate_origin_only(landmark_old, theta, 0, img_h)
+            return np.rot90(img, 3, axes=(-2, 0)), landmark
+        elif degree == 180:
+            landmark = self.__rotate_origin_only(landmark_old, theta, img_h, img_w)
+            return np.rot90(img, 2, axes=(-2, 0)), landmark
+        elif degree == 270:
+            landmark = self.__rotate_origin_only(landmark_old, theta, img_w, 0)
+            return np.rot90(img, 1, axes=(-2, 0)), landmark
+
+    def __transfer_lbl(self, marging_width_1, lbl, axis_arr):
+        new_lbl = []
+        for i in range(0, len(lbl), 2):
+            new_lbl.append(lbl[i] + marging_width_1 * axis_arr[0])
+            new_lbl.append(lbl[i + 1] + marging_width_1 * axis_arr[1])
+        return np.array(new_lbl)

img_printer.py

@@ -0,0 +1,81 @@
+
+import matplotlib
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
+from pathlib import Path
+
+
+def print_image(image_name, landmarks_x, landmarks_y):
+
+    my_file = Path(image_name)
+    if my_file.is_file():
+        im = plt.imread(image_name)
+        implot = plt.imshow(im)
+
+        plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='r', s=10)
+        plt.show()
+
+
+def print_image_arr_heat(k, image, print_single=False):
+    import numpy as np
+    for i in range(image.shape[2]):
+        img = np.sum(image, axis=2)
+        if print_single:
+            plt.figure()
+            plt.imshow(image[:, :, i])
+            # implot = plt.imshow(image[:, :, i])
+            plt.axis('off')
+            plt.savefig('single_heat_' + str(i+(k*100)) + '.png', bbox_inches='tight')
+            plt.clf()
+
+    plt.figure()
+    plt.imshow(img, vmin=0, vmax=1)
+    plt.axis('off')
+    plt.savefig('heat_' + str(k) + '.png', bbox_inches='tight')
+    plt.clf()
+
+
+def print_image_arr(k, image, landmarks_x, landmarks_y):
+    plt.figure()
+    plt.imshow(image)
+    implot = plt.imshow(image)
+
+    # for i in range(len(landmarks_x)):
+    #     plt.text(landmarks_x[i], landmarks_y[i], str(i), fontsize=12, c='red',
+    #              horizontalalignment='center', verticalalignment='center',
+    #              bbox={'facecolor': 'blue', 'alpha': 0.3, 'pad': 0.0})
+
+    plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='#000000', s=15)
+    plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='#fddb3a', s=8)
+    # plt.axis('off')
+    plt.savefig('z_' + str(k) + '.png', bbox_inches='tight')
+    # plt.show()
+    plt.clf()
+
+
+def print_image_arr_2(k, image, landmarks_x, landmarks_y, xs, ys):
+    plt.figure()
+    plt.imshow(image)
+    implot = plt.imshow(image)
+
+    # plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='b', s=5)
+    # for i in range(68):
+    #     plt.annotate(str(i), (landmarks_x[i], landmarks_y[i]), fontsize=6)
+
+    plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='b', s=5)
+    plt.scatter(x=xs, y=ys, c='r', s=5)
+    plt.savefig('sss'+str(k)+'.png')
+    # plt.show()
+    plt.clf()
+
+
+def print_two_landmarks(image, landmarks_1, landmarks_2):
+    plt.figure()
+    plt.imshow(image)
+    implot = plt.imshow(image)
+
+    plt.scatter(x=landmarks_1[:68], y=landmarks_1[68:], c='b', s=10)
+    plt.scatter(x=landmarks_2[:68], y=landmarks_2[68:], c='r', s=10)
+    # plt.savefig('a'+str(landmarks_x[0])+'.png')
+    plt.show()
+    plt.clf()

main.py

@@ -0,0 +1,45 @@
+
+from configuration import DatasetName, D300wConf, InputDataSize, CofwConf, WflwConf
+from cnn_model import CNNModel
+from pca_utility import PCAUtility
+from image_utility import ImageUtility
+from student_train import StudentTrainer
+from test import Test
+from teacher_trainer import TeacherTrainer
+
+if __name__ == '__main__':
+
+    '''test models'''
+
+    '''train Teacher Networks'''
+    trainer = TeacherTrainer(dataset_name=DatasetName.w300)
+    trainer.train(arch='efficientNet',weight_path=None)
+
+
+    '''Training Student Network'''
+    '''300W'''
+    st_trainer = StudentTrainer(dataset_name=DatasetName.w300, use_augmneted=True)
+    st_trainer.train(arch_student='mobileNetV2', weight_path_student=None,
+                     loss_weight_student=2.0,
+                     arch_tough_teacher='efficientNet', weight_path_tough_teacher='./models/teachers/ds_300w_ef_tou.h5',
+                     loss_weight_tough_teacher=1,
+                     arch_tol_teacher='efficientNet', weight_path_tol_teacher='./models/teachers/ds_300w_ef_tol.h5',
+                     loss_weight_tol_teacher=1)
+
+    '''COFW'''
+    st_trainer = StudentTrainer(dataset_name=DatasetName.cofw, use_augmneted=True)
+    st_trainer.train(arch_student='mobileNetV2', weight_path_student=None,
+                     loss_weight_student=2.0,
+                     arch_tough_teacher='efficientNet', weight_path_tough_teacher='./models/teachers/ds_cofw_ef_tou.h5',
+                     loss_weight_tough_teacher=1,
+                     arch_tol_teacher='efficientNet', weight_path_tol_teacher='./models/teachers/ds_cofw_ef_tol.h5',
+                     loss_weight_tol_teacher=1)
+
+    '''WFLW'''
+    st_trainer = StudentTrainer(dataset_name=DatasetName.wflw, use_augmneted=True)
+    st_trainer.train(arch_student='mobileNetV2', weight_path_student=None,
+                     loss_weight_student=2.0,
+                     arch_tough_teacher='efficientNet', weight_path_tough_teacher='./models/teachers/ds_wflw_ef_tou.h5',
+                     loss_weight_tough_teacher=1,
+                     arch_tol_teacher='efficientNet', weight_path_tol_teacher='./models/teachers/ds_wflw_ef_tol.h5',
+                     loss_weight_tol_teacher=1)

models/students/student_Net_300w.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc9c077037f4f7cc3df664b95f1f98397a11a2b298e121dc5e79f46837cb240
+size 10197672

models/students/student_Net_COFW.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3e8cc30f7f6bc3327cf3e59332552b2a321142376b27f14e3b957ffb94fb1da
+size 9799400

models/students/student_Net_WFLW.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61f3ee0c6188856377bd9e38109fdfa0aa97170086cfbdd2bc3c3f355f0f82cd
+size 10504872

pca_utility.py

@@ -0,0 +1,286 @@
+from configuration import DatasetName, DatasetType,\
+    AffectnetConf, D300wConf, W300Conf, InputDataSize, LearningConfig, CofwConf, WflwConf
+from image_utility import ImageUtility
+from sklearn.decomposition import PCA, IncrementalPCA
+from sklearn.decomposition import TruncatedSVD
+import numpy as np
+import pickle
+import os
+from tqdm import tqdm
+from numpy import save, load
+import math
+from PIL import Image
+
+
+class PCAUtility:
+    eigenvalues_prefix = "_eigenvalues_"
+    eigenvectors_prefix = "_eigenvectors_"
+    meanvector_prefix = "_meanvector_"
+
+    def create_pca_from_npy(self, dataset_name, pca_postfix):
+        lbl_arr = []
+        path = None
+        if dataset_name == DatasetName.ibug:
+            path = D300wConf.normalized_point  # normalized
+        # elif dataset_name == DatasetName.cofw:
+        #     path = CofwConf.normalized_points_npy_dir  # normalized
+        # elif dataset_name == DatasetName.wflw:
+        #     path = WflwConf.normalized_points_npy_dir  # normalized
+
+        lbl_arr = []
+        for file in tqdm(os.listdir(path)):
+            if file.endswith(".npy"):
+                npy_file = os.path.join(path, file)
+                lbl_arr.append(load(npy_file))
+
+        lbl_arr = np.array(lbl_arr)
+
+        print('PCA calculation started')
+
+        ''' no normalization is needed, since we want to generate hm'''
+        reduced_lbl_arr, eigenvalues, eigenvectors = self._func_PCA(lbl_arr, pca_postfix)
+        mean_lbl_arr = np.mean(lbl_arr, axis=0)
+        eigenvectors = eigenvectors.T
+        #
+        # self.__save_obj(eigenvalues, dataset_name + self.__eigenvalues_prefix + str(pca_postfix))
+        # self.__save_obj(eigenvectors, dataset_name + self.__eigenvectors_prefix + str(pca_postfix))
+        # self.__save_obj(mean_lbl_arr, dataset_name + self.__meanvector_prefix + str(pca_postfix))
+        #
+        save('pca_obj/' + dataset_name + self.eigenvalues_prefix + str(pca_postfix), eigenvalues)
+        save('pca_obj/' + dataset_name + self.eigenvectors_prefix + str(pca_postfix), eigenvectors)
+        save('pca_obj/' + dataset_name + self.meanvector_prefix + str(pca_postfix), mean_lbl_arr)
+
+    def create_pca_from_points(self, dataset_name, pca_postfix):
+        lbl_arr = []
+        path = None
+        if dataset_name == DatasetName.ibug:
+            path = D300wConf.rotated_img_path_prefix  # rotated is ok, since advs_aug is the same as rotated
+            num_of_landmarks = D300wConf.num_of_landmarks
+        elif dataset_name == DatasetName.cofw:
+            path = CofwConf.rotated_img_path_prefix
+            num_of_landmarks = CofwConf.num_of_landmarks
+        elif dataset_name == DatasetName.wflw:
+            path = WflwConf.rotated_img_path_prefix
+            num_of_landmarks = WflwConf.num_of_landmarks
+
+        for file in tqdm(os.listdir(path)):
+            if file.endswith(".pts"):
+                pts_file = os.path.join(path, file)
+
+                points_arr = []
+                with open(pts_file) as fp:
+                    line = fp.readline()
+                    cnt = 1
+                    while line:
+                        if 3 < cnt <= num_of_landmarks+3:
+                            x_y_pnt = line.strip()
+                            x = float(x_y_pnt.split(" ")[0])
+                            y = float(x_y_pnt.split(" ")[1])
+                            points_arr.append(x)
+                            points_arr.append(y)
+                        line = fp.readline()
+                        cnt += 1
+                lbl_arr.append(points_arr)
+
+        lbl_arr = np.array(lbl_arr)
+
+        print('PCA calculation started')
+
+        ''' no normalization is needed, since we want to generate hm'''
+        reduced_lbl_arr, eigenvalues, eigenvectors = self._func_PCA(lbl_arr, pca_postfix)
+        mean_lbl_arr = np.mean(lbl_arr, axis=0)
+        eigenvectors = eigenvectors.T
+        #
+        # self.__save_obj(eigenvalues, dataset_name + self.__eigenvalues_prefix + str(pca_postfix))
+        # self.__save_obj(eigenvectors, dataset_name + self.__eigenvectors_prefix + str(pca_postfix))
+        # self.__save_obj(mean_lbl_arr, dataset_name + self.__meanvector_prefix + str(pca_postfix))
+        #
+        save('pca_obj/' + dataset_name + self.eigenvalues_prefix + str(pca_postfix), eigenvalues)
+        save('pca_obj/' + dataset_name + self.eigenvectors_prefix + str(pca_postfix), eigenvectors)
+        save('pca_obj/' + dataset_name + self.meanvector_prefix + str(pca_postfix), mean_lbl_arr)
+
+    def test_pca_validity(self, dataset_name, pca_postfix):
+        image_utility = ImageUtility()
+
+        eigenvalues = load('pca_obj/' + dataset_name + self.eigenvalues_prefix + str(pca_postfix)+".npy")
+        eigenvectors = load('pca_obj/' + dataset_name + self.eigenvectors_prefix + str(pca_postfix)+".npy")
+        meanvector = load('pca_obj/' + dataset_name + self.meanvector_prefix + str(pca_postfix)+".npy")
+
+        '''load data: '''
+        lbl_arr = []
+        img_arr = []
+        path = None
+
+        if dataset_name == DatasetName.ibug:
+            path = D300wConf.rotated_img_path_prefix  # rotated is ok, since advs_aug is the same as rotated
+            num_of_landmarks = D300wConf.num_of_landmarks
+        elif dataset_name == DatasetName.cofw:
+            path = CofwConf.rotated_img_path_prefix
+            num_of_landmarks = CofwConf.num_of_landmarks
+        elif dataset_name == DatasetName.wflw:
+            path = WflwConf.rotated_img_path_prefix
+            num_of_landmarks = WflwConf.num_of_landmarks
+
+        for file in tqdm(os.listdir(path)):
+            if file.endswith(".pts"):
+                pts_file = os.path.join(path, file)
+                img_file = pts_file[:-3] + "jpg"
+                if not os.path.exists(img_file):
+                    continue
+
+                points_arr = []
+                with open(pts_file) as fp:
+                    line = fp.readline()
+                    cnt = 1
+                    while line:
+                        if 3 < cnt <= num_of_landmarks+3:
+                            x_y_pnt = line.strip()
+                            x = float(x_y_pnt.split(" ")[0])
+                            y = float(x_y_pnt.split(" ")[1])
+                            points_arr.append(x)
+                            points_arr.append(y)
+                        line = fp.readline()
+                        cnt += 1
+                lbl_arr.append(points_arr)
+                img_arr.append(Image.open(img_file))
+
+        for i in range(30):
+            b_vector_p = self.calculate_b_vector(lbl_arr[i], True, eigenvalues, eigenvectors, meanvector)
+            lbl_new = meanvector + np.dot(eigenvectors, b_vector_p)
+            lbl_new = lbl_new.tolist()
+
+            labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \
+                create_landmarks(lbl_arr[i], 1, 1)
+
+            labels_true_transformed_pca, landmark_arr_x_pca, landmark_arr_y_pca = image_utility. \
+                create_landmarks(lbl_new, 1, 1)
+
+            image_utility.print_image_arr(i, img_arr[i], landmark_arr_x_t, landmark_arr_y_t)
+            image_utility.print_image_arr(i * 1000, img_arr[i], landmark_arr_x_pca, landmark_arr_y_pca)
+
+    def calculate_b_vector(self, predicted_vector, correction, eigenvalues, eigenvectors, meanvector):
+        tmp1 = predicted_vector - meanvector
+        b_vector = np.dot(eigenvectors.T, tmp1)
+
+        # put b in -3lambda =>
+        if correction:
+            i = 0
+            for b_item in b_vector:
+                lambda_i_sqr = 3 * math.sqrt(eigenvalues[i])
+
+                if b_item > 0:
+                    b_item = min(b_item, lambda_i_sqr)
+                else:
+                    b_item = max(b_item, -1 * lambda_i_sqr)
+                b_vector[i] = b_item
+                i += 1
+
+        return b_vector
+
+    def create_pca(self, dataset_name, pca_postfix):
+        tf_record_util = TFRecordUtility()
+
+        lbl_arr = []
+        pose_arr = []
+        if dataset_name == DatasetName.ibug:
+            lbl_arr, img_arr, pose_arr = tf_record_util.retrieve_tf_record(D300wConf.tf_train_path,
+                                                                           D300wConf.sum_of_train_samples,
+                                                                           only_label=True, only_pose=True)
+        lbl_arr = np.array(lbl_arr)
+
+        print('PCA-retrieved')
+
+        '''need to be normalized based on the hyper face paper?'''
+
+        # reduced_lbl_arr, eigenvalues, eigenvectors = self.__svd_func(lbl_arr, pca_postfix)
+        reduced_lbl_arr, eigenvalues, eigenvectors = self.__func_PCA(lbl_arr, pca_postfix)
+        mean_lbl_arr = np.mean(lbl_arr, axis=0)
+        eigenvectors = eigenvectors.T
+
+        self.__save_obj(eigenvalues, dataset_name + self.eigenvalues_prefix + str(pca_postfix))
+        self.__save_obj(eigenvectors, dataset_name + self.eigenvectors_prefix + str(pca_postfix))
+        self.__save_obj(mean_lbl_arr, dataset_name + self.meanvector_prefix + str(pca_postfix))
+
+        '''calculate pose min max'''
+        p_1_arr = []
+        p_2_arr = []
+        p_3_arr = []
+
+        for p_item in pose_arr:
+            p_1_arr.append(p_item[0])
+            p_2_arr.append(p_item[1])
+            p_3_arr.append(p_item[2])
+
+        p_1_min = min(p_1_arr)
+        p_1_max = max(p_1_arr)
+
+        p_2_min = min(p_2_arr)
+        p_2_max = max(p_2_arr)
+
+        p_3_min = min(p_3_arr)
+        p_3_max = max(p_3_arr)
+
+        self.__save_obj(p_1_min, 'p_1_min')
+        self.__save_obj(p_1_max, 'p_1_max')
+
+        self.__save_obj(p_2_min, 'p_2_min')
+        self.__save_obj(p_2_max, 'p_2_max')
+
+        self.__save_obj(p_3_min, 'p_3_min')
+        self.__save_obj(p_3_max, 'p_3_max')
+
+        print('PCA-->done')
+
+    def __save_obj(self, obj, name):
+        with open('obj/' + name + '.pkl', 'wb') as f:
+            pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
+
+    def load_pose_obj(self):
+        with open('obj/p_1_min.pkl', 'rb') as f:
+            p_1_min = pickle.load(f)
+        with open('obj/p_1_max.pkl', 'rb') as f:
+            p_1_max = pickle.load(f)
+
+        with open('obj/p_2_min.pkl', 'rb') as f:
+            p_2_min = pickle.load(f)
+        with open('obj/p_2_max.pkl', 'rb') as f:
+            p_2_max = pickle.load(f)
+
+        with open('obj/p_3_min.pkl', 'rb') as f:
+            p_3_min = pickle.load(f)
+        with open('obj/p_3_max.pkl', 'rb') as f:
+            p_3_max = pickle.load(f)
+
+        return p_1_min, p_1_max, p_2_min, p_2_max, p_3_min, p_3_max
+
+
+    def load_pca_obj(self, dataset_name, pca_postfix=97):
+        with open('obj/' + dataset_name + self.eigenvalues_prefix + str(pca_postfix) + '.pkl', 'rb') as f:
+            eigenvalues = pickle.load(f)
+        with open('obj/' + dataset_name + self.eigenvectors_prefix + str(pca_postfix) + '.pkl', 'rb') as f:
+            eigenvectors = pickle.load(f)
+        with open('obj/' + dataset_name + self.meanvector_prefix + str(pca_postfix) + '.pkl', 'rb') as f:
+            meanvector = pickle.load(f)
+        return eigenvalues, eigenvectors, meanvector
+
+    def _func_PCA(self, input_data, pca_postfix):
+        input_data = np.array(input_data)
+        pca = PCA(n_components=pca_postfix/100)
+        # pca = PCA(n_components=0.98)
+        # pca = IncrementalPCA(n_components=50, batch_size=50)
+        pca.fit(input_data)
+        pca_input_data = pca.transform(input_data)
+        eigenvalues = pca.explained_variance_
+        eigenvectors = pca.components_
+        return pca_input_data, eigenvalues, eigenvectors
+
+    def __svd_func(self, input_data, pca_postfix):
+        svd = TruncatedSVD(n_components=50)
+        svd.fit(input_data)
+        pca_input_data = svd.transform(input_data)
+        eigenvalues = svd.explained_variance_
+        eigenvectors = svd.components_
+        return pca_input_data, eigenvalues, eigenvectors
+        # U, S, VT = svd(input_data)
+
+

requirements.txt

@@ -0,0 +1,23 @@
+# for cuda 9
+# pip install torch==1.2.0+cu92 torchvision==0.4.0+cu92 -f https://download.pytorch.org/whl/torch_stable.html
+#
+#torch
+#torchvision
+bottleneck
+numpy==1.19.2
+#tensorflow==1.14.0
+tensorflow==2.3.1
+#tensorflow-gpu==1.14
+# keras==2.2.4
+keras==2.4.3
+matplotlib
+opencv-python
+opencv-contrib-python
+scipy
+scikit-learn
+scikit-image
+Pillow
+tqdm
+efficientnet
+# tfkerassurgeon
+tensorboard

student_train.py

@@ -0,0 +1,292 @@
+from configuration import DatasetName, DatasetType, \
+    AffectnetConf, D300wConf, W300Conf, InputDataSize, LearningConfig, CofwConf, WflwConf
+from tf_record_utility import TFRecordUtility
+from clr_callback import CyclicLR
+from cnn_model import CNNModel
+from custom_Losses import Custom_losses
+from Data_custom_generator import CustomHeatmapGenerator
+from PW_Data_custom_generator import PWCustomHeatmapGenerator
+from image_utility import ImageUtility
+import img_printer as imgpr
+
+import tensorflow as tf
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+from datetime import datetime
+from sklearn.utils import shuffle
+from sklearn.model_selection import train_test_split
+from numpy import save, load, asarray
+import csv
+from skimage.io import imread
+import pickle
+
+
+# tf.compat.v1.enable_eager_execution()
+
+
+class StudentTrainer:
+
+    def __init__(self, dataset_name, use_augmneted):
+        self.dataset_name = dataset_name
+
+        if dataset_name == DatasetName.w300:
+            self.num_landmark = D300wConf.num_of_landmarks * 2
+            if use_augmneted:
+                self.img_path = D300wConf.augmented_train_image
+                self.annotation_path = D300wConf.augmented_train_annotation
+            else:
+                self.img_path = D300wConf.no_aug_train_image
+                self.annotation_path = D300wConf.no_aug_train_annotation
+
+        if dataset_name == DatasetName.cofw:
+            self.num_landmark = CofwConf.num_of_landmarks * 2
+            self.img_path = CofwConf.augmented_train_image
+            self.annotation_path = CofwConf.augmented_train_annotation
+
+        if dataset_name == DatasetName.wflw:
+            self.num_landmark = WflwConf.num_of_landmarks * 2
+            if use_augmneted:
+                self.img_path = WflwConf.augmented_train_image
+                self.annotation_path = WflwConf.augmented_train_annotation
+            else:
+                self.img_path = WflwConf.no_aug_train_image
+                self.annotation_path = WflwConf.no_aug_train_annotation
+
+    def train(self, arch_student, weight_path_student, loss_weight_student,
+              arch_tough_teacher, weight_path_tough_teacher, loss_weight_tough_teacher,
+              arch_tol_teacher, weight_path_tol_teacher, loss_weight_tol_teacher):
+        """"""
+        '''create loss'''
+        c_loss = Custom_losses()
+
+        '''create summary writer'''
+        summary_writer = tf.summary.create_file_writer(
+            "./train_logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S"))
+
+        '''making models'''
+        model_student = self.make_model(arch=arch_student, w_path=weight_path_student, is_old=False)
+        model_tough_teacher = self.make_model(arch=arch_tough_teacher, w_path=weight_path_tough_teacher)
+        model_tol_teacher = self.make_model(arch=arch_tol_teacher, w_path=weight_path_tol_teacher)
+
+        '''create optimizer'''
+        _lr = 1e-3
+        optimizer_student = self._get_optimizer(lr=_lr)
+
+        '''create sample generator'''
+        x_train_filenames, x_val_filenames, y_train_filenames, y_val_filenames = self._create_generators()
+        # x_train_filenames, y_train_filenames = self._create_generators()
+
+        '''create train configuration'''
+        step_per_epoch = len(x_train_filenames) // LearningConfig.batch_size
+
+        '''start train:'''
+        for epoch in range(LearningConfig.epochs):
+            x_train_filenames, y_train_filenames = self._shuffle_data(x_train_filenames, y_train_filenames)
+            for batch_index in range(step_per_epoch):
+                '''load annotation and images'''
+                images, annotation_gr, annotation_tough_teacher, annotation_tol_teacher, annotation_student = self._get_batch_sample(
+                    batch_index=batch_index, x_train_filenames=x_train_filenames,
+                    y_train_filenames=y_train_filenames, model_tough_t=model_tough_teacher,
+                    model_tol_t=model_tol_teacher, model_student=model_student)
+                '''convert to tensor'''
+                images = tf.cast(images, tf.float32)
+                annotation_gr = tf.cast(annotation_gr, tf.float32)
+                annotation_tough_teacher = tf.cast(annotation_tough_teacher, tf.float32)
+                annotation_tol_teacher = tf.cast(annotation_tol_teacher, tf.float32)
+
+                '''train step'''
+                self.train_step(epoch=epoch, step=batch_index, total_steps=step_per_epoch, images=images,
+                                model_student=model_student,
+                                annotation_gr=annotation_gr, annotation_tough_teacher=annotation_tough_teacher,
+                                annotation_tol_teacher=annotation_tol_teacher,
+                                l_w_stu=loss_weight_student, l_w_togh_t=loss_weight_tough_teacher,
+                                loss_w_tol_t=loss_weight_tol_teacher,
+                                optimizer=optimizer_student, summary_writer=summary_writer, c_loss=c_loss)
+            '''evaluating part'''
+            img_batch_eval, pn_batch_eval = self._create_evaluation_batch(x_val_filenames, y_val_filenames)
+            # loss_eval, loss_eval_tol_dif_stu, loss_eval_tol_dif_gt, loss_eval_tou_dif_stu, loss_eval_tou_dif_gt = \
+            loss_eval = self._eval_model(img_batch_eval, pn_batch_eval, model_student)
+            with summary_writer.as_default():
+                tf.summary.scalar('Eval-LOSS', loss_eval, step=epoch)
+                # tf.summary.scalar('Eval-loss_eval_tol_dif_stu', loss_eval_tol_dif_stu, step=epoch)
+                # tf.summary.scalar('Eval-loss_eval_tol_dif_gt', loss_eval_tol_dif_gt, step=epoch)
+                # tf.summary.scalar('Eval-loss_eval_tou_dif_stu', loss_eval_tou_dif_stu, step=epoch)
+                # tf.summary.scalar('Eval-loss_eval_tou_dif_gt', loss_eval_tou_dif_gt, step=epoch)
+            # model_student.save_weights('./models/stu_weight_' + '_' + str(epoch) + self.dataset_name + '_' + str(loss_eval) + '.h5')
+            '''save weights'''
+            model_student.save(
+                './models/stu_model_' + str(epoch) + '_' + self.dataset_name + '_' + str(loss_eval) + '.h5')
+            '''calculate Learning rate'''
+            # _lr = self._calc_learning_rate(iterations=epoch, step_size=20, base_lr=1e-5, max_lr=1e-1)
+            # optimizer_student = self._get_optimizer(lr=_lr)
+
+    def _calc_learning_rate(self, iterations, step_size, base_lr, max_lr):
+        cycle = np.floor(1 + iterations / (2 * step_size))
+        x = np.abs(iterations / step_size - 2 * cycle + 1)
+        lr = base_lr + (max_lr - base_lr) * np.maximum(0, (1 - x)) / float(2 ** (cycle - 1))
+        print('LR is: ' + str(lr))
+        return lr
+
+    # @tf.function
+    def train_step(self, epoch, step, total_steps, images, model_student, annotation_gr,
+                   annotation_tough_teacher, annotation_tol_teacher, annotation_student,
+                   l_w_stu, l_w_togh_t, loss_w_tol_t,
+                   optimizer, summary_writer, c_loss, train_dif):
+        with tf.GradientTape() as tape_student:
+            '''create annotation_predicted'''
+            # annotation_predicted, pr_tol, pr_tol_dif_gt, pr_tou, pr_tou_dif_gt = model_student(
+            annotation_predicted = model_student(
+                images, training=True)
+            '''calculate loss'''
+            loss_total, loss_main, loss_tough_assist, loss_tol_assist = c_loss.kd_loss(x_pr=annotation_predicted,
+                                                                                       x_gt=annotation_gr,
+                                                                                       x_tough=annotation_tough_teacher,
+                                                                                       x_tol=annotation_tol_teacher,
+                                                                                       alpha_tough=1.9,
+                                                                                       alpha_mi_tough=-0.45,
+                                                                                       alpha_tol=1.8,
+                                                                                       alpha_mi_tol=-0.4,
+                                                                                       main_loss_weight=l_w_stu,
+                                                                                       tough_loss_weight=l_w_togh_t,
+                                                                                       tol_loss_weight=loss_w_tol_t)
+            '''calculate gradient'''
+            gradients_of_student = tape_student.gradient(loss_total, model_student.trainable_variables)
+            '''apply Gradients:'''
+            optimizer.apply_gradients(zip(gradients_of_student, model_student.trainable_variables))
+            '''printing loss Values: '''
+            tf.print("->EPOCH: ", str(epoch), "->STEP: ", str(step) + '/' + str(total_steps),
+                     ' -> : LOSS: ', loss_total,
+                     ' -> : loss_main: ', loss_main,
+                     ' -> : loss_tough_assist: ', loss_tough_assist,
+                     ' -> : loss_tol_assist: ', loss_tol_assist)
+            with summary_writer.as_default():
+                tf.summary.scalar('LOSS', loss_total, step=epoch)
+                tf.summary.scalar('loss_main', loss_main, step=epoch)
+                tf.summary.scalar('loss_tough_assist', loss_tough_assist, step=epoch)
+                tf.summary.scalar('loss_tol_assist', loss_tol_assist, step=epoch)
+
+    def make_model(self, arch, w_path, is_old=False):
+        cnn = CNNModel()
+        model = cnn.get_model(arch=arch, output_len=self.num_landmark, input_tensor=None, weight_path=w_path,
+                              is_old=is_old)
+        if w_path is not None and arch != 'mobileNetV2_d' and not is_old:
+            model.load_weights(w_path)
+        # model.save('test_model'+arch+'.h5')
+        return model
+
+    def _eval_model(self, img_batch_eval, pn_batch_eval, model):
+        # annotation_predicted, pr_tol_dif_stu, pr_tol_dif_gt, pr_tou_dif_stu, pr_tou_dif_gt = model(img_batch_eval)
+        annotation_predicted = model(img_batch_eval)
+        loss_eval = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tol_dif_stu = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tol_dif_gt = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tou_dif_stu = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tou_dif_gt = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # return loss_eval, loss_eval_tol_dif_stu, loss_eval_tol_dif_gt, loss_eval_tou_dif_stu, loss_eval_tou_dif_gt
+        return loss_eval
+
+    def _get_optimizer(self, lr=1e-2, beta_1=0.9, beta_2=0.999, decay=1e-4):
+        return tf.keras.optimizers.Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, decay=decay)
+
+    def _shuffle_data(self, filenames, labels):
+        filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels)
+        return filenames_shuffled, y_labels_shuffled
+
+    def _create_generators(self):
+        fn_prefix = './file_names/' + self.dataset_name + '_'
+        # x_trains_path = fn_prefix + 'x_train_fns.npy'
+        # x_validations_path = fn_prefix + 'x_val_fns.npy'
+
+        tf_utils = TFRecordUtility(number_of_landmark=self.num_landmark)
+
+        filenames, labels = tf_utils.create_image_and_labels_name(img_path=self.img_path,
+                                                                  annotation_path=self.annotation_path)
+        filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels)
+        x_train_filenames, x_val_filenames, y_train, y_val = train_test_split(
+            filenames_shuffled, y_labels_shuffled, test_size=LearningConfig.batch_size, random_state=1)
+
+        # save(x_trains_path, filenames_shuffled)
+        # save(x_validations_path, y_labels_shuffled)
+
+        # save(x_trains_path, x_train_filenames)
+        # save(x_validations_path, x_val_filenames)
+        # save(y_trains_path, y_train)
+        # save(y_validations_path, y_val)
+
+        # return filenames_shuffled, y_labels_shuffled
+        return x_train_filenames, x_val_filenames, y_train, y_val
+
+    def _create_evaluation_batch(self, x_eval_filenames, y_eval_filenames):
+        img_path = self.img_path
+        pn_tr_path = self.annotation_path
+        '''create batch data and normalize images'''
+        batch_x = x_eval_filenames[0:LearningConfig.batch_size]
+        batch_y = y_eval_filenames[0:LearningConfig.batch_size]
+        '''create img and annotations'''
+        img_batch = np.array([imread(img_path + file_name) for file_name in batch_x]) / 255.0
+        if self.dataset_name == DatasetName.cofw:  # this ds is not normalized
+            pn_batch = np.array([load(pn_tr_path + file_name) for file_name in batch_y])
+        else:
+            pn_batch = np.array([self._load_and_normalize(pn_tr_path + file_name) for file_name in batch_y])
+        return img_batch, pn_batch
+
+    def _get_batch_sample(self, batch_index, x_train_filenames, y_train_filenames, model_tough_t, model_tol_t, model_student,
+                          train_dif):
+        img_path = self.img_path
+        pn_tr_path = self.annotation_path
+        '''create batch data and normalize images'''
+        batch_x = x_train_filenames[
+                  batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        batch_y = y_train_filenames[
+                  batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        img_batch = np.array([imread(img_path + file_name) for file_name in batch_x]) / 255.0
+        if self.dataset_name == DatasetName.cofw:  # this ds is not normalized
+            pn_batch = np.array([load(pn_tr_path + file_name) for file_name in batch_y])
+        else:
+            pn_batch = np.array([self._load_and_normalize(pn_tr_path + file_name) for file_name in batch_y])
+        '''prediction to create tough and tolerant batches'''
+        pn_batch_tough = model_tough_t.predict_on_batch(img_batch)
+        if not train_dif:
+            pn_batch_tol = model_tol_t.predict_on_batch(img_batch)
+            pn_batch_stu = None
+        else:
+            pn_batch_tol = None
+            pn_batch_stu = model_tol_t.predict_on_batch(img_batch)
+
+
+        # pn_batch_tough = 0
+        # pn_batch_tol = 0
+
+        '''test: print'''
+        # image_utility = ImageUtility()
+        # if self.dataset_name == DatasetName.cofw:  # this ds is not normalized
+        #     gr_s, gr_px_1, gr_Py_1 = image_utility.create_landmarks(pn_batch[0])
+        #     tou_s, tou_px_1, tou_Py_1 = image_utility.create_landmarks(pn_batch_tough[0])
+        #     tol_s, tol_px_1, tol_Py_1 = image_utility.create_landmarks(pn_batch_tol[0])
+        # else:
+        #     gr_s, gr_px_1, gr_Py_1 = image_utility.create_landmarks_from_normalized(pn_batch[0], 224, 224, 112, 112)
+        #     tou_s, tou_px_1, tou_Py_1 = image_utility.create_landmarks_from_normalized(pn_batch_tough[0], 224, 224, 112, 112)
+        #     tol_s, tol_px_1, tol_Py_1 = image_utility.create_landmarks_from_normalized(pn_batch_tol[0], 224, 224, 112, 112)
+        #
+        # imgpr.print_image_arr(str(batch_index)+'pts_gt', img_batch[0], gr_px_1, gr_Py_1)
+        # imgpr.print_image_arr(str(batch_index)+'pts_t100', img_batch[0], tou_px_1, tou_Py_1)
+        # imgpr.print_image_arr(str(batch_index)+'pts_t90', img_batch[0], tol_px_1, tol_Py_1)
+
+        return img_batch, pn_batch, pn_batch_tough, pn_batch_tol, pn_batch_stu
+
+    def _load_and_normalize(self, point_path):
+        annotation = load(point_path)
+
+        """for training we dont normalize COFW"""
+
+        '''normalize landmarks based on hyperface method'''
+        width = InputDataSize.image_input_size
+        height = InputDataSize.image_input_size
+        x_center = width / 2
+        y_center = height / 2
+        annotation_norm = []
+        for p in range(0, len(annotation), 2):
+            annotation_norm.append((x_center - annotation[p]) / width)
+            annotation_norm.append((y_center - annotation[p + 1]) / height)
+        return annotation_norm

teacher_trainer.py

@@ -0,0 +1,215 @@
+from configuration import DatasetName, DatasetType, \
+    AffectnetConf, D300wConf, W300Conf, InputDataSize, LearningConfig, CofwConf, WflwConf
+from tf_record_utility import TFRecordUtility
+from clr_callback import CyclicLR
+from cnn_model import CNNModel
+from custom_Losses import Custom_losses
+from Data_custom_generator import CustomHeatmapGenerator
+from PW_Data_custom_generator import PWCustomHeatmapGenerator
+from image_utility import ImageUtility
+import img_printer as imgpr
+
+import tensorflow as tf
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+from datetime import datetime
+from sklearn.utils import shuffle
+from sklearn.model_selection import train_test_split
+from numpy import save, load, asarray
+import csv
+from skimage.io import imread
+import pickle
+
+
+# tf.compat.v1.enable_eager_execution()
+
+
+class TeacherTrainer:
+
+    def __init__(self, dataset_name, use_augmneted):
+        self.dataset_name = dataset_name
+
+        if dataset_name == DatasetName.w300:
+            self.num_landmark = D300wConf.num_of_landmarks * 2
+            if use_augmneted:
+                self.img_path = D300wConf.augmented_train_image
+                self.annotation_path = D300wConf.augmented_train_annotation
+            else:
+                self.img_path = D300wConf.no_aug_train_image
+                self.annotation_path = D300wConf.no_aug_train_annotation
+
+        if dataset_name == DatasetName.cofw:
+            self.num_landmark = CofwConf.num_of_landmarks * 2
+            self.img_path = CofwConf.augmented_train_image
+            self.annotation_path = CofwConf.augmented_train_annotation
+
+        if dataset_name == DatasetName.wflw:
+            self.num_landmark = WflwConf.num_of_landmarks * 2
+            if use_augmneted:
+                self.img_path = WflwConf.augmented_train_image
+                self.annotation_path = WflwConf.augmented_train_annotation
+            else:
+                self.img_path = WflwConf.no_aug_train_image
+                self.annotation_path = WflwConf.no_aug_train_annotation
+
+    def train(self, arch, weight_path):
+        """"""
+        '''create loss'''
+        c_loss = Custom_losses()
+
+        '''create summary writer'''
+        summary_writer = tf.summary.create_file_writer(
+            "./train_logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S"))
+
+        '''making models'''
+        model = self.make_model(arch=arch, w_path=weight_path, is_old=False)
+
+        '''create optimizer'''
+        _lr = 1e-3
+        optimizer_student = self._get_optimizer(lr=_lr)
+
+        '''create sample generator'''
+        x_train_filenames, x_val_filenames, y_train_filenames, y_val_filenames = self._create_generators()
+
+        '''create train configuration'''
+        step_per_epoch = len(x_train_filenames) // LearningConfig.batch_size
+
+        '''start train:'''
+        for epoch in range(LearningConfig.epochs):
+            x_train_filenames, y_train_filenames = self._shuffle_data(x_train_filenames, y_train_filenames)
+            for batch_index in range(step_per_epoch):
+                '''load annotation and images'''
+                images, annotation_gr = self._get_batch_sample(
+                    batch_index=batch_index, x_train_filenames=x_train_filenames,
+                    y_train_filenames=y_train_filenames, model=model)
+                '''convert to tensor'''
+                images = tf.cast(images, tf.float32)
+                annotation_gr = tf.cast(annotation_gr, tf.float32)
+
+                '''train step'''
+                self.train_step(epoch=epoch, step=batch_index, total_steps=step_per_epoch, images=images,
+                                model=model,
+                                annotation_gr=annotation_gr,
+                                optimizer=optimizer_student,
+                                summary_writer=summary_writer, c_loss=c_loss)
+            '''evaluating part'''
+            img_batch_eval, pn_batch_eval = self._create_evaluation_batch(x_val_filenames, y_val_filenames)
+            # loss_eval, loss_eval_tol_dif_stu, loss_eval_tol_dif_gt, loss_eval_tou_dif_stu, loss_eval_tou_dif_gt = \
+            loss_eval = self._eval_model(img_batch_eval, pn_batch_eval, model)
+            with summary_writer.as_default():
+                tf.summary.scalar('Eval-LOSS', loss_eval, step=epoch)
+            '''save weights'''
+            model.save(
+                './models/teacher_model_' + str(epoch) + '_' + self.dataset_name + '_' + str(loss_eval) + '.h5')
+
+    # @tf.function
+    def train_step(self, epoch, step, total_steps, images,
+                   model, annotation_gr,
+                   optimizer, summary_writer, c_loss):
+        with tf.GradientTape() as tape:
+            '''create annotation_predicted'''
+            annotation_predicted = model(images, training=True)
+            '''calculate loss'''
+            loss = c_loss.MSE(x_pr=annotation_predicted, x_gt=annotation_gr)
+            '''calculate gradient'''
+            gradients = tape.gradient(loss, model.trainable_variables)
+            '''apply Gradients:'''
+            optimizer.apply_gradients(zip(gradients, model.trainable_variables))
+            '''printing loss Values: '''
+            tf.print("->EPOCH: ", str(epoch), "->STEP: ", str(step) + '/' + str(total_steps), ' -> : LOSS: ', loss)
+            with summary_writer.as_default():
+                tf.summary.scalar('LOSS', loss, step=epoch)
+
+    def make_model(self, arch, w_path, is_old=False):
+        cnn = CNNModel()
+        model = cnn.get_model(arch=arch, output_len=self.num_landmark, input_tensor=None, weight_path=w_path,
+                              is_old=is_old)
+        if w_path is not None and arch != 'mobileNetV2_d' and not is_old:
+            model.load_weights(w_path)
+        return model
+
+    def _eval_model(self, img_batch_eval, pn_batch_eval, model):
+        # annotation_predicted, pr_tol_dif_stu, pr_tol_dif_gt, pr_tou_dif_stu, pr_tou_dif_gt = model(img_batch_eval)
+        annotation_predicted = model(img_batch_eval)
+        loss_eval = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tol_dif_stu = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tol_dif_gt = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tou_dif_stu = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # loss_eval_tou_dif_gt = np.array(tf.reduce_mean(tf.abs(pn_batch_eval - annotation_predicted)))
+        # return loss_eval, loss_eval_tol_dif_stu, loss_eval_tol_dif_gt, loss_eval_tou_dif_stu, loss_eval_tou_dif_gt
+        return loss_eval
+
+    def _get_optimizer(self, lr=1e-2, beta_1=0.9, beta_2=0.999, decay=1e-4):
+        return tf.keras.optimizers.Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, decay=decay)
+
+    def _shuffle_data(self, filenames, labels):
+        filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels)
+        return filenames_shuffled, y_labels_shuffled
+
+    def _create_generators(self):
+        fn_prefix = './file_names/' + self.dataset_name + '_'
+        # x_trains_path = fn_prefix + 'x_train_fns.npy'
+        # x_validations_path = fn_prefix + 'x_val_fns.npy'
+
+        tf_utils = TFRecordUtility(number_of_landmark=self.num_landmark)
+
+        filenames, labels = tf_utils.create_image_and_labels_name(img_path=self.img_path,
+                                                                  annotation_path=self.annotation_path)
+        filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels)
+        x_train_filenames, x_val_filenames, y_train, y_val = train_test_split(
+            filenames_shuffled, y_labels_shuffled, test_size=LearningConfig.batch_size, random_state=1)
+
+        # save(x_trains_path, filenames_shuffled)
+        # save(x_validations_path, y_labels_shuffled)
+
+        # save(x_trains_path, x_train_filenames)
+        # save(x_validations_path, x_val_filenames)
+        # save(y_trains_path, y_train)
+        # save(y_validations_path, y_val)
+
+        # return filenames_shuffled, y_labels_shuffled
+        return x_train_filenames, x_val_filenames, y_train, y_val
+
+    def _create_evaluation_batch(self, x_eval_filenames, y_eval_filenames):
+        img_path = self.img_path
+        pn_tr_path = self.annotation_path
+        '''create batch data and normalize images'''
+        batch_x = x_eval_filenames[0:LearningConfig.batch_size]
+        batch_y = y_eval_filenames[0:LearningConfig.batch_size]
+        '''create img and annotations'''
+        img_batch = np.array([imread(img_path + file_name) for file_name in batch_x]) / 255.0
+        if self.dataset_name == DatasetName.cofw:  # this ds is not normalized
+            pn_batch = np.array([load(pn_tr_path + file_name) for file_name in batch_y])
+        else:
+            pn_batch = np.array([self._load_and_normalize(pn_tr_path + file_name) for file_name in batch_y])
+        return img_batch, pn_batch
+
+    def _get_batch_sample(self, batch_index, x_train_filenames, y_train_filenames):
+        img_path = self.img_path
+        pn_tr_path = self.annotation_path
+        '''create batch data and normalize images'''
+        batch_x = x_train_filenames[
+                  batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        batch_y = y_train_filenames[
+                  batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        img_batch = np.array([imread(img_path + file_name) for file_name in batch_x]) / 255.0
+        pn_batch = np.array([self._load_and_normalize(pn_tr_path + file_name) for file_name in batch_y])
+
+        return img_batch, pn_batch
+
+    def _load_and_normalize(self, point_path):
+        annotation = load(point_path)
+
+        """for training we dont normalize COFW"""
+
+        '''normalize landmarks based on hyperface method'''
+        width = InputDataSize.image_input_size
+        height = InputDataSize.image_input_size
+        x_center = width / 2
+        y_center = height / 2
+        annotation_norm = []
+        for p in range(0, len(annotation), 2):
+            annotation_norm.append((x_center - annotation[p]) / width)
+            annotation_norm.append((y_center - annotation[p + 1]) / height)
+        return annotation_norm

test.py

@@ -0,0 +1,40 @@
+from configuration import DatasetName, DatasetType, \
+    AffectnetConf, D300wConf, W300Conf, InputDataSize, LearningConfig, CofwConf, WflwConf
+from tf_record_utility import TFRecordUtility
+from image_utility import ImageUtility
+from skimage.transform import resize
+import numpy as np
+import math
+
+import cv2
+import os.path
+import scipy.io as sio
+from cnn_model import CNNModel
+import img_printer as imgpr
+from tqdm import tqdm
+
+
+class Test:
+    def __init__(self, dataset_name, arch, num_output_layers, weight_fname, has_pose=False):
+        self.dataset_name = dataset_name
+        self.has_pose = has_pose
+
+        if dataset_name == DatasetName.w300:
+            self.output_len = D300wConf.num_of_landmarks * 2
+        elif dataset_name == DatasetName.cofw:
+            self.output_len = CofwConf.num_of_landmarks * 2
+        elif dataset_name == DatasetName.wflw:
+            self.output_len = WflwConf.num_of_landmarks * 2
+
+        cnn = CNNModel()
+        model = cnn.get_model(arch=arch, input_tensor=None, output_len=self.output_len)
+
+        model.load_weights(weight_fname)
+
+        img = None # load a cropped image
+
+        image_utility = ImageUtility()
+        pose_predicted = []
+        image = np.expand_dims(img, axis=0)
+
+        pose_predicted = model.predict(image)[1][0]