diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py
index c809d39..9c9f6b8 100644
--- a/comfy/k_diffusion/sampling.py
+++ b/comfy/k_diffusion/sampling.py
@@ -8,6 +8,8 @@ import torchsde
 from tqdm.auto import trange, tqdm
 
 from . import utils
+from loguru import logger
+import threading
 
 
 def append_zero(x):
@@ -584,24 +586,36 @@ def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=N
 @torch.no_grad()
 def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=None):
     """DPM-Solver++(2M)."""
+    # logger.warning(f"Starting DPM-Solver++(2M) on model {id(model):x}")
     extra_args = {} if extra_args is None else extra_args
     s_in = x.new_ones([x.shape[0]])
     sigma_fn = lambda t: t.neg().exp()
     t_fn = lambda sigma: sigma.log().neg()
     old_denoised = None
 
+    # logger.warning(f"Starting iterations of DPM-Solver++(2M) on model {id(model):x}")
     for i in trange(len(sigmas) - 1, disable=disable):
+        # logger.warning(f"Starting iterations {i} on model {id(model):x}")
+        # logger.warning(f"denoise = {id(model):x}({id(x):x}, {id(sigmas):x} * {id(s_in)}, {id(extra_args)})")
         denoised = model(x, sigmas[i] * s_in, **extra_args)
+        # logger.warning(f"  denoised on model {id(model):x}")
         if callback is not None:
             callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
+        # logger.warning(f"  step 1 {id(model):x}")
         t, t_next = t_fn(sigmas[i]), t_fn(sigmas[i + 1])
+        # logger.warning(f"  step 2 {id(model):x}")
         h = t_next - t
+        # logger.warning(f"  step 3 {id(model):x}")
         if old_denoised is None or sigmas[i + 1] == 0:
+            # logger.warning(f"  step 3a {id(model):x}")
             x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised
         else:
+            # logger.warning(f"  step 3b {id(model):x}")
             h_last = t - t_fn(sigmas[i - 1])
             r = h_last / h
             denoised_d = (1 + 1 / (2 * r)) * denoised - (1 / (2 * r)) * old_denoised
             x = (sigma_fn(t_next) / sigma_fn(t)) * x - (-h).expm1() * denoised_d
+        # logger.warning(f"  step 4 end iteration {id(model):x}")
         old_denoised = denoised
+    # logger.warning(f"Done DPM-Solver++(2M) on model {id(model):x}")
     return x
diff --git a/comfy/model_management.py b/comfy/model_management.py
index c153232..92d1d14 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1,6 +1,9 @@
 import psutil
 from enum import Enum
-from comfy.cli_args import args
+from cli_args import args
+import threading
+from loguru import logger
+from contextlib import nullcontext
 
 class VRAMState(Enum):
     CPU = 0
@@ -16,6 +19,7 @@ set_vram_to = VRAMState.NORMAL_VRAM
 
 total_vram = 0
 total_vram_available_mb = -1
+enable_batch_optimisations = True
 
 accelerate_enabled = False
 xpu_available = False
@@ -45,13 +49,6 @@ try:
         except:
             total_vram = torch.cuda.mem_get_info(torch.cuda.current_device())[1] / (1024 * 1024)
     total_ram = psutil.virtual_memory().total / (1024 * 1024)
-    if not args.normalvram and not args.cpu:
-        if total_vram <= 4096:
-            print("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram")
-            set_vram_to = VRAMState.LOW_VRAM
-        elif total_vram > total_ram * 1.1 and total_vram > 14336:
-            print("Enabling highvram mode because your GPU has more vram than your computer has ram. If you don't want this use: --normalvram")
-            vram_state = VRAMState.HIGH_VRAM
 except:
     pass
 
@@ -74,10 +71,10 @@ else:
             print("xformers version:", XFORMERS_VERSION)
             if XFORMERS_VERSION.startswith("0.0.18"):
                 print()
-                print("WARNING: This version of xformers has a major bug where you will get black images when generating high resolution images.")
-                print("Please downgrade or upgrade xformers to a different version.")
-                print()
-                XFORMERS_ENABLED_VAE = False
+                # print("WARNING: This version of xformers has a major bug where you will get black images when generating high resolution images.")
+                # print("Please downgrade or upgrade xformers to a different version.")
+                # print()
+                # XFORMERS_ENABLED_VAE = False
         except:
             pass
     except:
@@ -125,7 +122,7 @@ except:
 if args.cpu:
     vram_state = VRAMState.CPU
 
-print(f"Set vram state to: {vram_state.name}")
+# print(f"Set vram state to: {vram_state.name}")
 
 def get_torch_device():
     global xpu_available
@@ -154,97 +151,177 @@ except:
     print("Could not pick default device.")
 
 
-current_loaded_model = None
-current_gpu_controlnets = []
-
-model_accelerated = False
-
-
-def unload_model():
-    global current_loaded_model
-    global model_accelerated
-    global current_gpu_controlnets
-    global vram_state
-
-    if current_loaded_model is not None:
-        if model_accelerated:
-            accelerate.hooks.remove_hook_from_submodules(current_loaded_model.model)
-            model_accelerated = False
-
-        #never unload models from GPU on high vram
-        if vram_state != VRAMState.HIGH_VRAM:
-            current_loaded_model.model.cpu()
-            current_loaded_model.model_patches_to("cpu")
-        current_loaded_model.unpatch_model()
-        current_loaded_model = None
-
-    if vram_state != VRAMState.HIGH_VRAM:
-        if len(current_gpu_controlnets) > 0:
-            for n in current_gpu_controlnets:
-                n.cpu()
-            current_gpu_controlnets = []
-
-
-def load_model_gpu(model):
-    global current_loaded_model
-    global vram_state
-    global model_accelerated
-
-    if model is current_loaded_model:
-        return
-    unload_model()
-    try:
-        real_model = model.patch_model()
-    except Exception as e:
+class ModelManager:
+    _instance = None
+    _initialised = False
+    _load_mutex = threading.RLock()
+    _property_mutex = threading.RLock()
+    sampler_mutex = threading.RLock()
+    vae_mutex = threading.RLock()
+
+    _property_mutex = nullcontext()
+
+    user_reserved_vram_mb = 0
+
+    # We are a singleton
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    # We initialise only ever once (in the lifetime of the singleton)
+    def __init__(self):
+        if not self._initialised:
+            self.models_in_use = []
+            self.current_loaded_models = []
+            self.current_gpu_controlnets = []
+            self.models_accelerated = []
+            self.__class__._initialised = True    
+
+    def set_user_reserved_vram(self, vram_mb):
+        with self._property_mutex:
+            self.user_reserved_vram_mb = vram_mb
+
+    def get_models_on_gpu(self):
+        with self._property_mutex:
+            return self.current_loaded_models[:]
+
+    def set_model_in_use(self, model):
+        with self._property_mutex:
+            self.models_in_use.append(model)
+
+    def is_model_in_use(self, model):
+        with self._property_mutex:
+            return model in self.models_in_use
+
+    def unload_model(self, model):
+        global vram_state
+        with self._property_mutex:
+            if model not in self.current_loaded_models:
+                logger.debug("Skip GPU unload as not on the GPU")
+                return
+
+            if model in self.models_in_use:
+                logger.debug("Not unloaded model as it is in use right now")
+                return
+
+            if model in self.models_accelerated:
+                accelerate.hooks.remove_hook_from_submodules(model.model)
+                self.models_accelerated.remove(model)
+
+            self.current_loaded_models.remove(model)
+
+        # Unload to RAM
+        model.model.cpu()
+        model.model_patches_to("cpu")
         model.unpatch_model()
-        raise e
-
-    model.model_patches_to(get_torch_device())
-    current_loaded_model = model
-    if vram_state == VRAMState.CPU:
-        pass
-    elif vram_state == VRAMState.MPS:
-        mps_device = torch.device("mps")
-        real_model.to(mps_device)
-        pass
-    elif vram_state == VRAMState.NORMAL_VRAM or vram_state == VRAMState.HIGH_VRAM:
-        model_accelerated = False
-        real_model.to(get_torch_device())
-    else:
-        if vram_state == VRAMState.NO_VRAM:
-            device_map = accelerate.infer_auto_device_map(real_model, max_memory={0: "256MiB", "cpu": "16GiB"})
-        elif vram_state == VRAMState.LOW_VRAM:
-            device_map = accelerate.infer_auto_device_map(real_model, max_memory={0: "{}MiB".format(total_vram_available_mb), "cpu": "16GiB"})
-
-        accelerate.dispatch_model(real_model, device_map=device_map, main_device=get_torch_device())
-        model_accelerated = True
-    return current_loaded_model
-
-def load_controlnet_gpu(control_models):
-    global current_gpu_controlnets
-    global vram_state
-    if vram_state == VRAMState.CPU:
-        return
-
-    if vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:
-        for m in control_models:
-            if hasattr(m, 'set_lowvram'):
-                m.set_lowvram(True)
-        #don't load controlnets like this if low vram because they will be loaded right before running and unloaded right after
-        return
-
-    models = []
-    for m in control_models:
-        models += m.get_models()
-
-    for m in current_gpu_controlnets:
-        if m not in models:
-            m.cpu()
+        return True
 
-    device = get_torch_device()
-    current_gpu_controlnets = []
-    for m in models:
-        current_gpu_controlnets.append(m.to(device))
+    def done_with_model(self, model):
+        with self._property_mutex:
+            if model in self.models_in_use:
+                self.models_in_use.remove(model)
+
+    def have_free_vram(self):
+        freemem = round(get_free_memory(get_torch_device()) / (1024 * 1024))
+        logger.debug(f"Free VRAM is: {freemem}MB ({len(self.current_loaded_models)} models loaded on GPU)")
+        return freemem > self.user_reserved_vram_mb
+
+    def load_model_gpu(self, model):
+        global vram_state
+        
+        with self._load_mutex:
+
+            # Don't run out of vram
+            if self.current_loaded_models:
+                if not self.have_free_vram():
+                    # Release the first least used model that we can
+                    for release_model in reversed(self.current_loaded_models):
+                        if self.unload_model(release_model):
+                            break
+                    freemem = round(get_free_memory(get_torch_device()) / (1024 * 1024))
+                    logger.debug(f"Unloaded a model, free VRAM is now: {freemem}MB ({len(self.current_loaded_models)} models loaded on GPU)")
+
+            if model in self.current_loaded_models:
+                # Move this model to the top of the list
+                self.current_loaded_models.insert(0, self.current_loaded_models.pop(self.current_loaded_models.index(model)))
+                return model
+            
+            try:
+                real_model = model.patch_model()
+            except Exception as e:
+                logger.error("Patching failed")
+                model.unpatch_model()
+                raise e
+            
+            self.current_loaded_models.insert(0, model)
+            model.model_patches_to(get_torch_device())
+
+            if vram_state == VRAMState.CPU:
+                pass
+            elif vram_state == VRAMState.MPS:
+                mps_device = torch.device("mps")
+                real_model.to(mps_device)
+            elif vram_state == VRAMState.NORMAL_VRAM or vram_state == VRAMState.HIGH_VRAM:
+                if model in self.models_accelerated:
+                    self.models_accelerated.remove(model)
+                real_model.to(get_torch_device())
+            else:
+                if vram_state == VRAMState.NO_VRAM:
+                    device_map = accelerate.infer_auto_device_map(real_model, max_memory={0: "256MiB", "cpu": "16GiB"})
+                elif vram_state == VRAMState.LOW_VRAM:
+                    device_map = accelerate.infer_auto_device_map(real_model, max_memory={0: "{}MiB".format(total_vram_available_mb), "cpu": "16GiB"})
+
+                accelerate.dispatch_model(real_model, device_map=device_map, main_device=get_torch_device())
+                self.models_accelerated.append(model)
+            return model
+
+    def load_controlnet_gpu(self, control_models):
+        with self._load_mutex:
+            global vram_state
+            if vram_state == VRAMState.CPU:
+                return
+
+            if vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:
+                #don't load controlnets like this if low vram because they will be loaded right before running and unloaded right after
+                return
+
+            models = []
+            for m in control_models:
+                models += m.get_models()
+
+            device = get_torch_device()
+            for m in models:
+                if m not in self.current_gpu_controlnets:
+                    self.current_gpu_controlnets.append(m.to(device))
+
+    def unload_controlnet_gpu(self, control_models):
+        with self._load_mutex:
+            global vram_state
+            if vram_state == VRAMState.CPU:
+                return
+
+            if vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:
+                #don't load controlnets like this if low vram because they will be loaded right before running and unloaded right after
+                return
+
+            models = []
+            for m in control_models:
+                models += m.get_models()
+
+            for m in models:
+                if m in self.current_gpu_controlnets:
+                    m.cpu()
+                    self.current_gpu_controlnets.remove(m)
+                    del m
+
+    def set_batch_optimisations(self, enable):
+        with self._property_mutex:
+            global enable_batch_optimisations
+            enable_batch_optimisations = enable
+
+
+model_manager = ModelManager()
 
 
 def load_if_low_vram(model):
@@ -326,6 +403,8 @@ def get_free_memory(dev=None, torch_free_too=False):
         return mem_free_total
 
 def maximum_batch_area():
+    if not enable_batch_optimisations:
+        return 0
     global vram_state
     if vram_state == VRAMState.NO_VRAM:
         return 0
diff --git a/comfy/sample.py b/comfy/sample.py
index 284efca..8f74a4a 100644
--- a/comfy/sample.py
+++ b/comfy/sample.py
@@ -57,7 +57,7 @@ def load_additional_models(positive, negative):
     gligen = get_models_from_cond(positive, "gligen") + get_models_from_cond(negative, "gligen")
     gligen = [x[1] for x in gligen]
     models = control_nets + gligen
-    comfy.model_management.load_controlnet_gpu(models)
+    #comfy.model_management.load_controlnet_gpu(models)
     return models
 
 def cleanup_additional_models(models):
@@ -72,7 +72,7 @@ def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative
         noise_mask = prepare_mask(noise_mask, noise.shape, device)
 
     real_model = None
-    comfy.model_management.load_model_gpu(model)
+    comfy.model_management.model_manager.load_model_gpu(model)
     real_model = model.model
 
     noise = noise.to(device)
@@ -81,12 +81,22 @@ def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative
     positive_copy = broadcast_cond(positive, noise.shape[0], device)
     negative_copy = broadcast_cond(negative, noise.shape[0], device)
 
-    models = load_additional_models(positive, negative)
+    try:
+        models = load_additional_models(positive, negative)
+        if models:
+            comfy.model_management.model_manager.load_controlnet_gpu(models)
+        comfy.model_management.model_manager.sampler_mutex.acquire()
 
-    sampler = comfy.samplers.KSampler(real_model, steps=steps, device=device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
+        sampler = comfy.samplers.KSampler(real_model, steps=steps, device=device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
 
-    samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar)
-    samples = samples.cpu()
+        samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar)
+        samples = samples.cpu()
+
+        if models:
+            comfy.model_management.model_manager.unload_controlnet_gpu(models)
+    finally:
+        comfy.model_management.model_manager.sampler_mutex.release()
+        model.unpatch_model()
+        cleanup_additional_models(models)
 
-    cleanup_additional_models(models)
     return samples
diff --git a/comfy/samplers.py b/comfy/samplers.py
index aa44fa8..7fde2fc 100644
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -251,8 +251,8 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con
                 output = model_function(input_x, timestep_, cond=c).chunk(batch_chunks)
                 del input_x
 
-                model_management.throw_exception_if_processing_interrupted()
-
+                # model_management.throw_exception_if_processing_interrupted()
+                
                 for o in range(batch_chunks):
                     if cond_or_uncond[o] == COND:
                         out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
@@ -262,6 +262,7 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con
                         out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
                 del mult
 
+
             out_cond /= out_count
             del out_count
             out_uncond /= out_uncond_count
@@ -269,7 +270,6 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con
 
             return out_cond, out_uncond
 
-
         max_total_area = model_management.maximum_batch_area()
         cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, cond_concat, model_options)
         if "sampler_cfg_function" in model_options:
diff --git a/comfy/sd.py b/comfy/sd.py
index c6be900..5774786 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -1,6 +1,7 @@
 import torch
 import contextlib
 import copy
+import threading
 
 from . import sd1_clip
 from . import sd2_clip
@@ -13,6 +14,8 @@ from .t2i_adapter import adapter
 
 from . import utils
 from . import clip_vision
+from loguru import logger
+import threading
 from . import gligen
 
 def load_model_weights(model, sd, verbose=False, load_state_dict_to=[]):
@@ -528,58 +531,78 @@ class VAE:
         return output
 
     def decode(self, samples_in):
-        model_management.unload_model()
-        self.first_stage_model = self.first_stage_model.to(self.device)
-        try:
-            free_memory = model_management.get_free_memory(self.device)
-            batch_number = int((free_memory * 0.7) / (2562 * samples_in.shape[2] * samples_in.shape[3] * 64))
-            batch_number = max(1, batch_number)
-
-            pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * 8), round(samples_in.shape[3] * 8)), device="cpu")
-            for x in range(0, samples_in.shape[0], batch_number):
-                samples = samples_in[x:x+batch_number].to(self.device)
-                pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(1. / self.scale_factor * samples) + 1.0) / 2.0, min=0.0, max=1.0).cpu()
-        except model_management.OOM_EXCEPTION as e:
-            print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
-            pixel_samples = self.decode_tiled_(samples_in)
-
-        self.first_stage_model = self.first_stage_model.cpu()
-        pixel_samples = pixel_samples.cpu().movedim(1,-1)
-        return pixel_samples
+        with model_management.model_manager.vae_mutex:
+            tid = threading.current_thread().ident
+            #logger.warning(f"VAE.decode({tid}) would unload model")
+            # model_management.unload_model()
+
+            #logger.warning(f"VAE.decode({tid}) first_stage_model to device {self.device}")
+            self.first_stage_model = self.first_stage_model.to(self.device)
+            try:
+                #logger.warning(f"VAE.decode({tid}) get free memory")
+                free_memory = model_management.get_free_memory(self.device)
+                batch_number = int((free_memory * 0.7) / (2562 * samples_in.shape[2] * samples_in.shape[3] * 64))
+                batch_number = max(1, batch_number)
+                #logger.warning(f"VAE.decode({tid}) {batch_number} batches")
+
+                #logger.warning(f"VAE.decode({tid}) torch.empty")
+                pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * 8), round(samples_in.shape[3] * 8)), device="cpu")
+                #logger.warning(f"VAE.decode({tid}) starting iterations")
+                for x in range(0, samples_in.shape[0], batch_number):
+                    samples = samples_in[x:x+batch_number].to(self.device)
+                    #logger.warning(f"VAE.decode({tid}) got samples")
+                    pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(1. / self.scale_factor * samples) + 1.0) / 2.0, min=0.0, max=1.0).cpu()
+                    #logger.warning(f"VAE.decode({tid}) clamped")
+            except model_management.OOM_EXCEPTION as e:
+                print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
+                pixel_samples = self.decode_tiled_(samples_in)
+
+            #logger.warning(f"VAE.decode({tid}) moving to cpu")
+            self.first_stage_model = self.first_stage_model.cpu()
+            #logger.warning(f"VAE.decode({tid}) calculating pixels")
+            pixel_samples = pixel_samples.cpu().movedim(1,-1)
+            #logger.warning(f"VAE.decode({tid}) returning")
+            return pixel_samples
 
     def decode_tiled(self, samples, tile_x=64, tile_y=64, overlap = 16):
-        model_management.unload_model()
-        self.first_stage_model = self.first_stage_model.to(self.device)
-        output = self.decode_tiled_(samples, tile_x, tile_y, overlap)
-        self.first_stage_model = self.first_stage_model.cpu()
-        return output.movedim(1,-1)
+        with model_management.model_manager.vae_mutex:
+            #logger.warning("VAE.decode_tiled() would unload model")
+            # model_management.unload_model()
+            self.first_stage_model = self.first_stage_model.to(self.device)
+            output = self.decode_tiled_(samples, tile_x, tile_y, overlap)
+            self.first_stage_model = self.first_stage_model.cpu()
+            return output.movedim(1,-1)
 
     def encode(self, pixel_samples):
-        model_management.unload_model()
-        self.first_stage_model = self.first_stage_model.to(self.device)
-        pixel_samples = pixel_samples.movedim(-1,1).to(self.device)
-        samples = self.first_stage_model.encode(2. * pixel_samples - 1.).sample() * self.scale_factor
-        self.first_stage_model = self.first_stage_model.cpu()
-        samples = samples.cpu()
-        return samples
+        #logger.warning("VAE.encode() would unload model")
+        # model_management.unload_model()
+        with model_management.model_manager.vae_mutex:
+            self.first_stage_model = self.first_stage_model.to(self.device)
+            pixel_samples = pixel_samples.movedim(-1,1).to(self.device)
+            samples = self.first_stage_model.encode(2. * pixel_samples - 1.).sample() * self.scale_factor
+            self.first_stage_model = self.first_stage_model.cpu()
+            samples = samples.cpu()
+            return samples
 
     def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
-        model_management.unload_model()
-        self.first_stage_model = self.first_stage_model.to(self.device)
-        pixel_samples = pixel_samples.movedim(-1,1).to(self.device)
-
-        steps = pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap)
-        steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap)
-        steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x * 2, tile_y // 2, overlap)
-        pbar = utils.ProgressBar(steps)
-
-        samples = utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
-        samples += utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
-        samples += utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
-        samples /= 3.0
-        self.first_stage_model = self.first_stage_model.cpu()
-        samples = samples.cpu()
-        return samples
+        #logger.warning("VAE.decode() would unload model")
+        # model_management.unload_model()
+        with model_management.model_manager.vae_mutex:
+            self.first_stage_model = self.first_stage_model.to(self.device)
+            pixel_samples = pixel_samples.movedim(-1,1).to(self.device)
+
+            steps = pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap)
+            steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap)
+            steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x * 2, tile_y // 2, overlap)
+            pbar = utils.ProgressBar(steps)
+
+            samples = utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
+            samples += utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
+            samples += utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
+            samples /= 3.0
+            self.first_stage_model = self.first_stage_model.cpu()
+            samples = samples.cpu()
+            return samples
 
 def broadcast_image_to(tensor, target_batch_size, batched_number):
     current_batch_size = tensor.shape[0]
@@ -686,8 +709,16 @@ class ControlNet:
         out.append(self.control_model)
         return out
 
+# Cache the controlnet diff models, about 4GB RAM
+_controlnet_models = {}
+
 def load_controlnet(ckpt_path, model=None):
-    controlnet_data = utils.load_torch_file(ckpt_path)
+    if ckpt_path in _controlnet_models:
+        controlnet_data = copy.deepcopy(_controlnet_models.get(ckpt_path))
+    else:
+        controlnet_data = utils.load_torch_file(ckpt_path)
+        _controlnet_models[ckpt_path] = copy.deepcopy(controlnet_data)
+
     pth_key = 'control_model.input_blocks.1.1.transformer_blocks.0.attn2.to_k.weight'
     pth = False
     sd2 = False
diff --git a/comfy_extras/nodes_upscale_model.py b/comfy_extras/nodes_upscale_model.py
index f9252ea..ee35ffb 100644
--- a/comfy_extras/nodes_upscale_model.py
+++ b/comfy_extras/nodes_upscale_model.py
@@ -4,6 +4,7 @@ from comfy import model_management
 import torch
 import comfy.utils
 import folder_paths
+import threading
 
 class UpscaleModelLoader:
     @classmethod
@@ -23,6 +24,7 @@ class UpscaleModelLoader:
 
 
 class ImageUpscaleWithModel:
+    _mutex = threading.Lock()
     @classmethod
     def INPUT_TYPES(s):
         return {"required": { "upscale_model": ("UPSCALE_MODEL",),
@@ -34,18 +36,19 @@ class ImageUpscaleWithModel:
     CATEGORY = "image/upscaling"
 
     def upscale(self, upscale_model, image):
-        device = model_management.get_torch_device()
-        upscale_model.to(device)
-        in_img = image.movedim(-1,-3).to(device)
-
-        tile = 128 + 64
-        overlap = 8
-        steps = in_img.shape[0] * comfy.utils.get_tiled_scale_steps(in_img.shape[3], in_img.shape[2], tile_x=tile, tile_y=tile, overlap=overlap)
-        pbar = comfy.utils.ProgressBar(steps)
-        s = comfy.utils.tiled_scale(in_img, lambda a: upscale_model(a), tile_x=tile, tile_y=tile, overlap=overlap, upscale_amount=upscale_model.scale, pbar=pbar)
-        upscale_model.cpu()
-        s = torch.clamp(s.movedim(-3,-1), min=0, max=1.0)
-        return (s,)
+        with ImageUpscaleWithModel._mutex:
+            device = model_management.get_torch_device()
+            upscale_model.to(device)
+            in_img = image.movedim(-1,-3).to(device)
+
+            tile = 128 + 64
+            overlap = 8
+            steps = in_img.shape[0] * comfy.utils.get_tiled_scale_steps(in_img.shape[3], in_img.shape[2], tile_x=tile, tile_y=tile, overlap=overlap)
+            pbar = comfy.utils.ProgressBar(steps)
+            s = comfy.utils.tiled_scale(in_img, lambda a: upscale_model(a), tile_x=tile, tile_y=tile, overlap=overlap, upscale_amount=upscale_model.scale, pbar=pbar)
+            upscale_model.cpu()
+            s = torch.clamp(s.movedim(-3,-1), min=0, max=1.0)
+            return (s,)
 
 NODE_CLASS_MAPPINGS = {
     "UpscaleModelLoader": UpscaleModelLoader,
diff --git a/execution.py b/execution.py
index dd88029..85d83ce 100644
--- a/execution.py
+++ b/execution.py
@@ -8,6 +8,7 @@ import traceback
 import gc
 import time
 
+from loguru import logger
 import torch
 import nodes
 
@@ -210,7 +211,6 @@ class PromptExecutor:
 
     def execute(self, prompt, prompt_id, extra_data={}, execute_outputs=[]):
         nodes.interrupt_processing(False)
-
         if "client_id" in extra_data:
             self.server.client_id = extra_data["client_id"]
         else:
@@ -249,6 +249,7 @@ class PromptExecutor:
 
                     recursive_execute(self.server, prompt, self.outputs, x, extra_data, executed, prompt_id)
             except Exception as e:
+                logger.error(traceback.format_exc())
                 if isinstance(e, comfy.model_management.InterruptProcessingException):
                     print("Processing interrupted")
                 else:
@@ -268,13 +269,14 @@ class PromptExecutor:
                     d = self.outputs.pop(o)
                     del d
             finally:
-                for x in executed:
-                    self.old_prompt[x] = copy.deepcopy(prompt[x])
+                # Don't do this, pointless waste of resources in horde
+                # for x in executed:
+                #     self.old_prompt[x] = copy.deepcopy(prompt[x])
                 self.server.last_node_id = None
                 if self.server.client_id is not None:
                     self.server.send_sync("executing", { "node": None, "prompt_id": prompt_id }, self.server.client_id)
 
-        print("Prompt executed in {:.2f} seconds".format(time.perf_counter() - execution_start_time))
+        # print("Prompt executed in {:.2f} seconds".format(time.perf_counter() - execution_start_time))
         gc.collect()
         comfy.model_management.soft_empty_cache()
 
@@ -341,7 +343,7 @@ def validate_inputs(prompt, item, validated):
     validated[unique_id] = ret
     return ret
 
-def validate_prompt(prompt):
+def validate_prompt(prompt, horde=False):
     outputs = set()
     for x in prompt:
         class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']]
@@ -349,6 +351,7 @@ def validate_prompt(prompt):
             outputs.add(x)
 
     if len(outputs) == 0:
+        logger.error("Prompt has no outputs")
         return (False, "Prompt has no outputs")
 
     good_outputs = set()
@@ -369,7 +372,7 @@ def validate_prompt(prompt):
         if valid == True:
             good_outputs.add(o)
         else:
-            print("Failed to validate prompt for output {} {}".format(o, reason))
+            logger.error("Failed to validate prompt for output {} {}".format(o, reason))
             print("output will be ignored")
             errors += [(o, reason)]
 
@@ -377,6 +380,9 @@ def validate_prompt(prompt):
         errors_list = "\n".join(set(map(lambda a: "{}".format(a[1]), errors)))
         return (False, "Prompt has no properly connected outputs\n {}".format(errors_list))
 
+    if not horde:
+        with open("../comfy-prompt.json", "wt", encoding="utf-8") as f:
+            f.write(json.dumps(prompt, indent=4))
     return (True, "", list(good_outputs))
 
 
diff --git a/main.py b/main.py
index 00cbf3c..1e1062a 100644
--- a/main.py
+++ b/main.py
@@ -26,7 +26,7 @@ import yaml
 import execution
 import folder_paths
 import server
-from nodes import init_custom_nodes
+from nodes import init_custom_nodes, load_custom_nodes
 
 
 def prompt_worker(q, server):
diff --git a/nodes.py b/nodes.py
index bc23e5c..440d2bc 100644
--- a/nodes.py
+++ b/nodes.py
@@ -29,6 +29,11 @@ import importlib
 
 import folder_paths
 
+import pickle
+import time
+from loguru import logger
+import threading
+
 def before_node_execution():
     comfy.model_management.throw_exception_if_processing_interrupted()
 
@@ -837,15 +842,19 @@ def common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive,
     if "noise_mask" in latent:
         noise_mask = latent["noise_mask"]
 
-    pbar = comfy.utils.ProgressBar(steps)
-    def callback(step, x0, x, total_steps):
-        pbar.update_absolute(step + 1, total_steps)
-
-    samples = comfy.sample.sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image,
-                                  denoise=denoise, disable_noise=disable_noise, start_step=start_step, last_step=last_step,
-                                  force_full_denoise=force_full_denoise, noise_mask=noise_mask, callback=callback)
-    out = latent.copy()
-    out["samples"] = samples
+    comfy.model_management.model_manager.set_model_in_use(model)
+    try:
+        pbar = comfy.utils.ProgressBar(steps)
+        def callback(step, x0, x, total_steps):
+            pbar.update_absolute(step + 1, total_steps)
+
+        samples = comfy.sample.sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image,
+                                    denoise=denoise, disable_noise=disable_noise, start_step=start_step, last_step=last_step,
+                                    force_full_denoise=force_full_denoise, noise_mask=noise_mask, callback=callback)
+        out = latent.copy()
+        out["samples"] = samples
+    finally:
+        comfy.model_management.model_manager.done_with_model(model)
     return (out, )
 
 class KSampler:
