diff --git a/debian/control b/debian/control
index 0ecb1da9d..f838adace 100644
--- a/debian/control
+++ b/debian/control
@@ -20,6 +20,7 @@ Build-Depends: debhelper-compat (= 13),
                python3-sphinx,
                python3-sphinxcontrib.qthelp | python3-sphinx (<< 2.0),
                dh-sequence-sphinxdoc,
+Build-Depends-Indep:
                dh-elpa
 Rules-Requires-Root: no
 Standards-Version: 4.6.0
diff --git a/debian/rules b/debian/rules
index 5845c6856..80f014654 100755
--- a/debian/rules
+++ b/debian/rules
@@ -70,9 +70,15 @@ override_dh_sphinxdoc:
 	dh_sphinxdoc -pcmake-doc
 
 export ELPA_NAME = cmake-mode
-execute_before_dh_elpa: debian/cmake-mode-pkg.el
+
+# Instead of dh --with=elpa, we manually add dh_elpa to the sequence.  This
+# way, we can restrict the dh-elpa dependency (which transitively depends on
+# emacs) to indep builds and avoid it for the more fragile port architectures
+execute_after_dh_install-indep: debian/cmake-mode-pkg.el
+	dh_elpa
+
 debian/cmake-mode-pkg.el: debian/cmake-mode-pkg.el.in
 	sed -e 's/@VERSION@/$(DEB_VERSION_UPSTREAM)/' $< > $@
 
 %:
-	dh $@ --with elpa --builddirectory=Build
+	dh $@ --builddirectory=Build