diff --git a/authority/authority.go b/authority/authority.go
index a0eaf871..b5924061 100644
--- a/authority/authority.go
+++ b/authority/authority.go
@@ -575,6 +575,8 @@ func (a *Authority) CloseForReload() {
 
 // requiresDecrypter returns whether the Authority
 // requires a KMS that provides a crypto.Decrypter
+// Currently this is only required when SCEP is
+// enabled.
 func (a *Authority) requiresDecrypter() bool {
 	return a.requiresSCEPService()
 }
diff --git a/ca/ca.go b/ca/ca.go
index 6eb223eb..4998b45b 100644
--- a/ca/ca.go
+++ b/ca/ca.go
@@ -118,6 +118,7 @@ func (ca *CA) Init(config *config.Config) (*CA, error) {
 	if err != nil {
 		return nil, err
 	}
+	ca.auth = auth
 
 	tlsConfig, err := ca.getTLSConfig(auth)
 	if err != nil {
@@ -233,14 +234,15 @@ func (ca *CA) Init(config *config.Config) (*CA, error) {
 		handler = logger.Middleware(handler)
 	}
 
-	ca.auth = auth
 	ca.srv = server.New(config.Address, handler, tlsConfig)
 
-	// TODO: instead opt for having a single server.Server but two
-	// http.Servers handling the HTTP and HTTPS handler? The latter
-	// will probably introduce more complexity in terms of graceful
-	// reload.
-	if config.InsecureAddress != "" {
+	// only start the insecure server if the insecure address is configured
+	// and, currently, also only when it should serve SCEP endpoints.
+	if ca.shouldServeSCEPEndpoints() && config.InsecureAddress != "" {
+		// TODO: instead opt for having a single server.Server but two
+		// http.Servers handling the HTTP and HTTPS handler? The latter
+		// will probably introduce more complexity in terms of graceful
+		// reload.
 		ca.insecureSrv = server.New(config.InsecureAddress, insecureHandler, nil)
 	}
 
diff --git a/scep/scep.go b/scep/scep.go
index f56176d7..3323ac1d 100644
--- a/scep/scep.go
+++ b/scep/scep.go
@@ -33,7 +33,6 @@ var (
 	oidSCEPtransactionID  = asn1.ObjectIdentifier{2, 16, 840, 1, 113733, 1, 9, 7}
 	oidSCEPfailInfoText   = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 24}
 	//oidChallengePassword  = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 7}
-
 )
 
 // PKIMessage defines the possible SCEP message types