fix bugs and optimizations

modules/hypernetworks/hypernetwork.py

@@ -36,14 +36,14 @@ class HypernetworkModule(torch.nn.Module):
             linears.append(torch.nn.Linear(int(dim * layer_structure[i]), int(dim * layer_structure[i+1])))
             # if skip_first_layer because first parameters potentially contain negative values
             # if i < 1: continue
+            if add_layer_norm:
+                linears.append(torch.nn.LayerNorm(int(dim * layer_structure[i+1])))
             if activation_func in HypernetworkModule.activation_dict:
                 linears.append(HypernetworkModule.activation_dict[activation_func]())
             else:
                 print("Invalid key {} encountered as activation function!".format(activation_func))
             # if use_dropout: 
             # linears.append(torch.nn.Dropout(p=0.3))
-            if add_layer_norm:
-                linears.append(torch.nn.LayerNorm(int(dim * layer_structure[i+1])))
 
         self.linear = torch.nn.Sequential(*linears)
 
@@ -115,11 +115,24 @@ class Hypernetwork:
 
         for k, layers in self.layers.items():
             for layer in layers:
-                layer.train()
                 res += layer.trainables()
 
         return res
 
+    def eval(self):
+        for k, layers in self.layers.items():
+            for layer in layers:
+                layer.eval()
+        for items in self.weights():
+            items.requires_grad = False
+
+    def train(self):
+        for k, layers in self.layers.items():
+            for layer in layers:
+                layer.train()
+        for items in self.weights():
+            items.requires_grad = True
+
     def save(self, filename):
         state_dict = {}
 
@@ -290,10 +303,6 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
         shared.sd_model.first_stage_model.to(devices.cpu)
 
     hypernetwork = shared.loaded_hypernetwork
-    weights = hypernetwork.weights()
-    for weight in weights:
-        weight.requires_grad = True
-
     losses = torch.zeros((32,))
 
     last_saved_file = "<none>"
@@ -304,10 +313,10 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
         return hypernetwork, filename
 
     scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)
-    # if optimizer == "AdamW": or else Adam / AdamW / SGD, etc...
-    optimizer = torch.optim.AdamW(weights, lr=scheduler.learn_rate)
+    optimizer = torch.optim.AdamW(hypernetwork.weights(), lr=scheduler.learn_rate)
 
     pbar = tqdm.tqdm(enumerate(ds), total=steps - ititial_step)
+    hypernetwork.train()
     for i, entries in pbar:
         hypernetwork.step = i + ititial_step
 
@@ -328,8 +337,9 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
 
             losses[hypernetwork.step % losses.shape[0]] = loss.item()
 
-            optimizer.zero_grad()
+            optimizer.zero_grad(set_to_none=True)
             loss.backward()
+            del loss
             optimizer.step()
         mean_loss = losses.mean()
         if torch.isnan(mean_loss):
@@ -346,9 +356,10 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
         })
 
         if hypernetwork.step > 0 and images_dir is not None and hypernetwork.step % create_image_every == 0:
+            torch.cuda.empty_cache()
             last_saved_image = os.path.join(images_dir, f'{hypernetwork_name}-{hypernetwork.step}.png')
-
-            optimizer.zero_grad()
+            with torch.no_grad():
+                hypernetwork.eval()
                 shared.sd_model.cond_stage_model.to(devices.device)
                 shared.sd_model.first_stage_model.to(devices.device)
 
@@ -385,6 +396,8 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
                     image.save(last_saved_image)
                     last_saved_image += f", prompt: {preview_text}"
 
+            hypernetwork.train()
+
         shared.state.job_no = hypernetwork.step
 
         shared.state.textinfo = f"""