From b9d61bd0f67c1172a61ab4060e07e176202e822a Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sat, 3 Feb 2018 23:18:26 -0500 Subject: [PATCH] UPSTREAM: 59316: Exit if no client cert is available for 5m --- .../kubernetes/cmd/kubelet/app/server.go | 7 ++-- .../pkg/kubelet/certificate/transport.go | 41 +++++++++++++++---- .../pkg/kubelet/certificate/transport_test.go | 2 +- .../util/certificate/certificate_manager.go | 7 ++++ 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go b/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go index a3de3115eeab..06e09ea8cdb0 100644 --- a/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go +++ b/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go @@ -396,9 +396,10 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { if err != nil { return err } - // we set exitIfExpired to true because we use this client configuration to request new certs - if we are unable - // to request new certs, we will be unable to continue normal operation - if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, true); err != nil { + // we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable + // to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper + // or the bootstrapping credentials to potentially lay down new initial config. + if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, 5*time.Minute); err != nil { return err } } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport.go b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport.go index 1683ad09735a..b8d200efc293 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport.go @@ -43,15 +43,21 @@ import ( // connections, forcing the client to re-handshake with the server and use the // new certificate. // +// The exitAfter duration, if set, will terminate the current process if a certificate +// is not available from the store (because it has been deleted on disk or is corrupt) +// or if the certificate has expired and the server is responsive. This allows the +// process parent or the bootstrap credentials an opportunity to retrieve a new initial +// certificate. +// // stopCh should be used to indicate when the transport is unused and doesn't need // to continue checking the manager. -func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitIfExpired bool) error { - return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager, exitIfExpired) +func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitAfter time.Duration) error { + return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager, exitAfter) } // updateTransport is an internal method that exposes how often this method checks that the -// client cert has changed. Intended for testing. -func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitIfExpired bool) error { +// client cert has changed. +func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitAfter time.Duration) error { if clientConfig.Transport != nil { return fmt.Errorf("there is already a transport configured") } @@ -77,16 +83,35 @@ func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig conns: make(map[*closableConn]struct{}), } + lastCertAvailable := time.Now() lastCert := clientCertificateManager.Current() go wait.Until(func() { curr := clientCertificateManager.Current() - if exitIfExpired && curr != nil && time.Now().After(curr.Leaf.NotAfter) { - if clientCertificateManager.ServerHealthy() { - glog.Fatalf("The currently active client certificate has expired and the server is responsive, exiting.") + + if exitAfter > 0 { + now := time.Now() + if curr == nil { + // the certificate has been deleted from disk or is otherwise corrupt + if now.After(lastCertAvailable.Add(exitAfter)) { + if clientCertificateManager.ServerHealthy() { + glog.Fatalf("It has been %s since a valid client cert was found and the server is responsive, exiting.", exitAfter) + } else { + glog.Errorf("It has been %s since a valid client cert was found, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.", exitAfter) + } + } } else { - glog.Errorf("The currently active client certificate has expired, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.") + // the certificate is expired + if now.After(curr.Leaf.NotAfter) { + if clientCertificateManager.ServerHealthy() { + glog.Fatalf("The currently active client certificate has expired and the server is responsive, exiting.") + } else { + glog.Errorf("The currently active client certificate has expired, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.") + } + } + lastCertAvailable = now } } + if curr == nil || lastCert == curr { // Cert hasn't been rotated. return diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport_test.go index 6e476c3b8037..d05b28aa6186 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/certificate/transport_test.go @@ -187,7 +187,7 @@ func TestRotateShutsDownConnections(t *testing.T) { } // Check for a new cert every 10 milliseconds - if err := updateTransport(stop, 10*time.Millisecond, c, m, false); err != nil { + if err := updateTransport(stop, 10*time.Millisecond, c, m, 0); err != nil { t.Fatal(err) } diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go index 68d90aa7ea1f..8c3e99019724 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go @@ -268,6 +268,13 @@ func getCurrentCertificateOrBootstrap( return nil, false, fmt.Errorf("unable to parse certificate data: %v", err) } bootstrapCert.Leaf = certs[0] + + if _, err := store.Update(bootstrapCertificatePEM, bootstrapKeyPEM); err != nil { + utilruntime.HandleError(fmt.Errorf("Unable to set the cert/key pair to the bootstrap certificate: %v", err)) + } else { + glog.V(4).Infof("Updated the store to contain the initial bootstrap certificate") + } + return &bootstrapCert, true, nil }