diff --git a/src/qwaylandlayersurface.cpp b/src/qwaylandlayersurface.cpp
index 5270aaa..e5b2bff 100644
--- a/src/qwaylandlayersurface.cpp
+++ b/src/qwaylandlayersurface.cpp
@@ -44,7 +44,11 @@ QWaylandLayerSurface::QWaylandLayerSurface(QWaylandLayerShellIntegration *shell,
     setAnchor(m_interface->anchors());
     connect(m_interface, &Window::anchorsChanged, this, [this]() {
         setAnchor(m_interface->anchors());
-        setDesiredSize(m_window->windowContentGeometry().size());
+        if (m_explicitDesiredSize) {
+            setDesiredSize(m_interface->desiredSize());
+        } else {
+            setDesiredSize(m_window->windowContentGeometry().size());
+        }
     });
 
     setExclusiveZone(m_interface->exclusionZone());
@@ -62,9 +66,8 @@ QWaylandLayerSurface::QWaylandLayerSurface(QWaylandLayerShellIntegration *shell,
     });
 
     connect(m_interface, &Window::desiredSizeChanged, this, [this]() {
-        if (!m_settingDesiredSize) {
-            setDesiredSize(m_interface->desiredSize());
-        }
+        m_explicitDesiredSize = true;
+        setDesiredSize(m_interface->desiredSize());
     });
 
     setKeyboardInteractivity(m_interface->keyboardInteractivity());
@@ -72,7 +75,9 @@ QWaylandLayerSurface::QWaylandLayerSurface(QWaylandLayerShellIntegration *shell,
         setKeyboardInteractivity(m_interface->keyboardInteractivity());
     });
 
-    setDesiredSize(window->windowContentGeometry().size());
+    if (!m_explicitDesiredSize) {
+        setDesiredSize(window->windowContentGeometry().size());
+    }
 }
 
 QWaylandLayerSurface::~QWaylandLayerSurface()
@@ -127,12 +132,6 @@ void QWaylandLayerSurface::applyConfigure()
 
 void QWaylandLayerSurface::setDesiredSize(const QSize &size)
 {
-    // This guards setDesiredSize to avoid the connection between
-    // m_interface::desiredSizeChanged and this takes effect when
-    // we call m_interface->setDesiredSize from here
-    m_settingDesiredSize = true;
-    m_interface->setDesiredSize(size);
-
     const bool horizontallyConstrained = m_interface->anchors().testFlags({Window::AnchorLeft, Window::AnchorRight});
     const bool verticallyConstrained = m_interface->anchors().testFlags({Window::AnchorTop, Window::AnchorBottom});
 
@@ -144,7 +143,6 @@ void QWaylandLayerSurface::setDesiredSize(const QSize &size)
         effectiveSize.setHeight(0);
     }
     set_size(effectiveSize.width(), effectiveSize.height());
-    m_settingDesiredSize = false;
 }
 
 void QWaylandLayerSurface::setAnchor(uint anchor)
@@ -187,12 +185,16 @@ void QWaylandLayerSurface::setWindowGeometry(const QRect &geometry)
         return;
     }
 
-    setDesiredSize(geometry.size());
+    if (!m_explicitDesiredSize) {
+        setDesiredSize(geometry.size());
+    }
 }
 #else
 void QWaylandLayerSurface::setWindowSize(const QSize &size)
 {
-    setDesiredSize(size);
+    if (!m_explicitDesiredSize) {
+        setDesiredSize(size);
+    }
 }
 #endif
 
diff --git a/src/qwaylandlayersurface_p.h b/src/qwaylandlayersurface_p.h
index 5152146..076fa27 100644
--- a/src/qwaylandlayersurface_p.h
+++ b/src/qwaylandlayersurface_p.h
@@ -64,7 +64,7 @@ private:
     QSize m_pendingSize;
     QString m_activationToken;
 
-    bool m_settingDesiredSize = false;
+    bool m_explicitDesiredSize = false;
     bool m_configured = false;
 #if QT_VERSION < QT_VERSION_CHECK(6, 9, 0)
     bool m_configuring = false;