diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7469392d..553d831f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -3,9 +3,9 @@ FROM golang:alpine AS builder
 WORKDIR /src
 COPY . .
 
-RUN apk add --no-cache curl git make
-RUN make V=1 download
+RUN apk add --no-cache curl git make libcap
 RUN make V=1 bin/step-ca
+RUN setcap CAP_NET_BIND_SERVICE=+eip bin/step-ca
 
 FROM smallstep/step-kms-plugin:cloud AS kms
 
@@ -14,8 +14,6 @@ FROM smallstep/step-cli:latest
 COPY --from=builder /src/bin/step-ca /usr/local/bin/step-ca
 COPY --from=kms /usr/local/bin/step-kms-plugin /usr/local/bin/step-kms-plugin
 
-USER root
-RUN apk add --no-cache libcap && setcap CAP_NET_BIND_SERVICE=+eip /usr/local/bin/step-ca
 USER step
 
 ENV CONFIGPATH="/home/step/config/ca.json"
diff --git a/docker/Dockerfile.hsm b/docker/Dockerfile.hsm
index 782b7431..61ea6694 100644
--- a/docker/Dockerfile.hsm
+++ b/docker/Dockerfile.hsm
@@ -6,8 +6,9 @@ COPY . .
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends \
         gcc pkgconf libpcsclite-dev
-RUN make V=1 download
-RUN make V=1 GOFLAGS="" build
+RUN make V=1 GOFLAGS="" bin/step-ca
+RUN apt-get install -y --no-install-recommends libcap2-bin && \
+    setcap CAP_NET_BIND_SERVICE=+eip bin/step-ca
 
 FROM smallstep/step-kms-plugin:bullseye AS kms
 
@@ -18,8 +19,6 @@ COPY --from=kms /usr/local/bin/step-kms-plugin /usr/local/bin/step-kms-plugin
 
 USER root
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends libcap2-bin && \
-    setcap CAP_NET_BIND_SERVICE=+eip /usr/local/bin/step-ca
 RUN apt-get install -y --no-install-recommends pcscd libpcsclite1
 RUN mkdir -p /run/pcscd
 RUN chown step:step /run/pcscd