diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
index ae34530b0..1f26aa60d 100644
--- a/src/stable-diffusion.cpp
+++ b/src/stable-diffusion.cpp
@@ -2990,7 +2990,7 @@ static std::optional<ImageGenerationLatents> prepare_image_generation_latents(sd
     latents.ref_images           = std::move(ref_images);
     latents.ref_latents          = std::move(ref_latents);
 
-    if (sd_version_is_inpaint(sd_ctx->sd->version)) {
+    if (!latents.init_latent.empty()) {
         latents.denoise_mask = std::move(latent_mask);
     }