diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 6d7a61c..9dc0979 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -176,7 +176,7 @@ class ModelPatcher:
             inplace_update = self.weight_inplace_update
 
             if key not in self.backup:
-                self.backup[key] = weight.to(device=device_to, copy=inplace_update)
+                self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)
 
             if device_to is not None:
                 temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)