From 3c18bf7139c77dc315d8f8691789475c50d39572 Mon Sep 17 00:00:00 2001 From: wu8685 Date: Sat, 6 Jul 2024 16:16:16 +0800 Subject: [PATCH 1/2] add util to self sign cert and update to WebhookConfiguration --- go.mod | 2 + go.sum | 7 + webhook/cert/cert.go | 134 +++++++++++++++ webhook/cert/cert_test.go | 55 ++++++ webhook/cert/error.go | 38 +++++ webhook/cert/error_test.go | 34 ++++ webhook/cert/fs.go | 226 +++++++++++++++++++++++++ webhook/cert/secret.go | 160 ++++++++++++++++++ webhook/cert/signer.go | 325 ++++++++++++++++++++++++++++++++++++ webhook/cert/signer_test.go | 209 +++++++++++++++++++++++ 10 files changed, 1190 insertions(+) create mode 100644 webhook/cert/cert.go create mode 100644 webhook/cert/cert_test.go create mode 100644 webhook/cert/error.go create mode 100644 webhook/cert/error_test.go create mode 100644 webhook/cert/fs.go create mode 100644 webhook/cert/secret.go create mode 100644 webhook/cert/signer.go create mode 100644 webhook/cert/signer_test.go diff --git a/go.mod b/go.mod index e4bc108..5411477 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,10 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.30.0 github.com/prometheus/client_golang v1.16.0 + github.com/spf13/afero v1.5.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 + github.com/zoumo/golib v0.2.0 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 k8s.io/api v0.28.4 k8s.io/apimachinery v0.28.4 diff --git a/go.sum b/go.sum index 963814f..3c7eeda 100644 --- a/go.sum +++ b/go.sum @@ -262,6 +262,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -332,6 +333,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -387,6 +389,8 @@ github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg= +github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= @@ -420,6 +424,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zoumo/golib v0.2.0 h1:K6W8WWrgnl2bXRvUaiXjAaiFKsCTHwnrBkBHZoFr8lE= +github.com/zoumo/golib v0.2.0/go.mod h1:gOMPRvDgn9m49tfHoKUb2RO0NqplNoe/qj5/ZrczjgQ= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -466,6 +472,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/webhook/cert/cert.go b/webhook/cert/cert.go new file mode 100644 index 0000000..4ed17f0 --- /dev/null +++ b/webhook/cert/cert.go @@ -0,0 +1,134 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "context" + "crypto/rsa" + "crypto/x509" + "fmt" + "net" + "time" + + "github.com/zoumo/golib/cert" + certutil "github.com/zoumo/golib/cert" +) + +type ( + Config = cert.Config + AltNames = cert.AltNames +) + +type ServingCerts struct { + Key []byte + Cert []byte + CAKey []byte + CACert []byte +} + +func (c *ServingCerts) Validate(host string) error { + if len(c.Key) == 0 { + return fmt.Errorf("private key is empty") + } + if len(c.Cert) == 0 { + return fmt.Errorf("cetificate is empty") + } + if len(c.CAKey) == 0 { + return fmt.Errorf("CA private key is empty") + } + if len(c.CACert) == 0 { + return fmt.Errorf("CA certificate is empty") + } + + tlsCert, err := cert.X509KeyPair(c.Cert, c.Key) + if err != nil { + return fmt.Errorf("invalid x509 keypair: %w", err) + } + + // verify cert with ca and host + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(c.CACert) { + return fmt.Errorf("no valid CA certificate found") + } + + options := x509.VerifyOptions{ + Roots: pool, + DNSName: host, + CurrentTime: time.Now(), + } + + _, err = tlsCert.X509Cert.Verify(options) + return err +} + +func GenerateSelfSignedCerts(cfg Config) (*ServingCerts, error) { + caKey, caCert, key, cert, err := generateSelfSignedCertKey(cfg) + if err != nil { + return nil, err + } + + keyPEM := certutil.MarshalRSAPrivateKeyToPEM(key) + cerPEM := certutil.MarshalCertToPEM(cert) + caKeyPEM := certutil.MarshalRSAPrivateKeyToPEM(caKey) + caCertPEM := certutil.MarshalCertToPEM(caCert) + + return &ServingCerts{ + CAKey: caKeyPEM.EncodeToMemory(), + CACert: caCertPEM.EncodeToMemory(), + Key: keyPEM.EncodeToMemory(), + Cert: cerPEM.EncodeToMemory(), + }, nil +} + +func GenerateSelfSignedCertKeyIfNotExist(path string, cfg cert.Config) error { + fscerts, err := NewFSProvider(path, FSOptions{}) + if err != nil { + return err + } + return fscerts.Ensure(context.Background(), cfg) +} + +func generateSelfSignedCertKey(cfg Config) (*rsa.PrivateKey, *x509.Certificate, *rsa.PrivateKey, *x509.Certificate, error) { + caKey, err := certutil.NewRSAPrivateKey() + if err != nil { + return nil, nil, nil, nil, err + } + + caCert, err := certutil.NewSelfSignedCACert(certutil.Config{ + CommonName: fmt.Sprintf("%s-ca@%d", cfg.CommonName, time.Now().Unix()), + }, caKey) + if err != nil { + return nil, nil, nil, nil, err + } + + key, err := certutil.NewRSAPrivateKey() + if err != nil { + return nil, nil, nil, nil, err + } + + if ip := net.ParseIP(cfg.CommonName); ip != nil { + cfg.AltNames.IPs = append(cfg.AltNames.IPs, ip) + } else { + cfg.AltNames.DNSNames = append(cfg.AltNames.DNSNames, cfg.CommonName) + } + + cert, err := certutil.NewSignedCert(cfg, key, caKey, caCert) + if err != nil { + return nil, nil, nil, nil, err + } + return caKey, caCert, key, cert, nil +} diff --git a/webhook/cert/cert_test.go b/webhook/cert/cert_test.go new file mode 100644 index 0000000..bbb6f5d --- /dev/null +++ b/webhook/cert/cert_test.go @@ -0,0 +1,55 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/zoumo/golib/cert" +) + +func TestServingCerts_Validate(t *testing.T) { + cfg := Config{ + CommonName: "foo.example.com", + AltNames: cert.AltNames{ + DNSNames: []string{"bar.example.com"}, + }, + } + certs, err := GenerateSelfSignedCerts(cfg) + assert.Nil(t, err) + assert.Nil(t, certs.Validate("foo.example.com")) + assert.Nil(t, certs.Validate("bar.example.com")) + assert.NotNil(t, certs.Validate("unknown.example.com")) +} + +func TestGenerateSelfSignedCerts(t *testing.T) { + cfg := Config{ + CommonName: "rollout.rollout-system.svc", + AltNames: cert.AltNames{ + DNSNames: []string{"rollout.rollout-system.svc", "foo.example.com"}, + }, + } + certs, err := GenerateSelfSignedCerts(cfg) + assert.Nil(t, err) + + err = certs.Validate("rollout.rollout-system.svc") + assert.Nil(t, err) + + err = certs.Validate("foo.example.com") + assert.Nil(t, err) +} diff --git a/webhook/cert/error.go b/webhook/cert/error.go new file mode 100644 index 0000000..bfacac6 --- /dev/null +++ b/webhook/cert/error.go @@ -0,0 +1,38 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "errors" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +var errNotFound = errors.New("not found") + +func newNotFound(name string, err error) error { + return fmt.Errorf("%s %w: %v", name, errNotFound, err) +} + +func IsNotFound(err error) bool { + return apierrors.IsNotFound(err) || errors.Is(err, errNotFound) +} + +func IsConflict(err error) bool { + return apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) +} diff --git a/webhook/cert/error_test.go b/webhook/cert/error_test.go new file mode 100644 index 0000000..b58a71f --- /dev/null +++ b/webhook/cert/error_test.go @@ -0,0 +1,34 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestIsNotFound(t *testing.T) { + err := newNotFound("testFile", errors.New("testError")) + assert.True(t, IsNotFound(err)) + + err = apierrors.NewNotFound(schema.GroupResource{}, "testResource") + assert.True(t, IsNotFound(err)) +} diff --git a/webhook/cert/fs.go b/webhook/cert/fs.go new file mode 100644 index 0000000..0d0c37b --- /dev/null +++ b/webhook/cert/fs.go @@ -0,0 +1,226 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "bytes" + "context" + "fmt" + "os" + "path" + + "github.com/spf13/afero" + "k8s.io/klog/v2" +) + +type FSProvider struct { + FSOptions + path string +} + +type FSOptions struct { + FS afero.Fs + CertName string + KeyName string + CAKeyName string + CACertName string +} + +func (o *FSOptions) setDefaults() { + if o.FS == nil { + o.FS = afero.NewOsFs() + } + if len(o.CertName) == 0 { + o.CertName = "tls.crt" + } + if len(o.KeyName) == 0 { + o.KeyName = "tls.key" + } + if len(o.CAKeyName) == 0 { + o.CAKeyName = "ca.key" + } + if len(o.CACertName) == 0 { + o.CACertName = "ca.crt" + } +} + +func NewFSProvider(path string, opts FSOptions) (*FSProvider, error) { + opts.setDefaults() + + if len(path) == 0 { + return nil, fmt.Errorf("cert path is required") + } + + return &FSProvider{ + path: path, + FSOptions: opts, + }, nil +} + +func (p *FSProvider) Ensure(_ context.Context, cfg Config) error { + certs, err := p.Load() + if err != nil && !IsNotFound(err) { + return err + } + + if IsNotFound(err) { + certs, err = GenerateSelfSignedCerts(cfg) + if err != nil { + return err + } + _, err = p.Overwrite(certs) + return err + } + + err = certs.Validate(cfg.CommonName) + if err != nil { + // re-generate if expired or invalid + klog.Info("certificates are invalid, regenerating...") + certs, err := GenerateSelfSignedCerts(cfg) + if err != nil { + return err + } + _, err = p.Overwrite(certs) + return err + } + return nil +} + +func (p *FSProvider) checkIfExist() error { + files := []string{ + path.Join(p.path, p.KeyName), + path.Join(p.path, p.CertName), + path.Join(p.path, p.CACertName), + path.Join(p.path, p.CAKeyName), + } + for _, file := range files { + _, err := p.FS.Stat(file) + if err == nil { + continue + } + + if os.IsNotExist(err) { + return newNotFound(file, err) + } + return err + } + return nil +} + +func (p *FSProvider) Load() (*ServingCerts, error) { + err := p.checkIfExist() + if err != nil { + return nil, err + } + + keyBytes, err := afero.ReadFile(p.FS, path.Join(p.path, p.KeyName)) + if err != nil { + return nil, err + } + certBytes, err := afero.ReadFile(p.FS, path.Join(p.path, p.CertName)) + if err != nil { + return nil, err + } + caBytes, err := afero.ReadFile(p.FS, path.Join(p.path, p.CACertName)) + if err != nil { + return nil, err + } + caKeyBytes, err := afero.ReadFile(p.FS, path.Join(p.path, p.CAKeyName)) + if err != nil { + return nil, err + } + + certs := &ServingCerts{ + Key: keyBytes, + Cert: certBytes, + CAKey: caKeyBytes, + CACert: caBytes, + } + + return certs, nil +} + +func (p *FSProvider) Overwrite(certs *ServingCerts) (bool, error) { + if certs == nil { + return false, fmt.Errorf("certs are required") + } + + stat, err := p.FS.Stat(p.path) + if err != nil && !os.IsNotExist(err) { + return false, err + } + if os.IsNotExist(err) { + err = os.MkdirAll(p.path, 0o755) + if err != nil { + return false, err + } + stat, _ = p.FS.Stat(p.path) + } + + if !stat.IsDir() { + return false, fmt.Errorf("the cert path %s must be a directory", p.path) + } + + keyPath := path.Join(p.path, p.KeyName) + var updated bool + changed, err := p.writeFile(keyPath, certs.Key) + if err != nil { + return false, fmt.Errorf("failed to write key to %s: %v", keyPath, err) + } + updated = changed || updated + + certPath := path.Join(p.path, p.CertName) + changed, err = p.writeFile(certPath, certs.Cert) + if err != nil { + return false, fmt.Errorf("failed to write cert to %s: %v", certPath, err) + } + updated = changed || updated + + caKeyPath := path.Join(p.path, p.CAKeyName) + changed, err = p.writeFile(caKeyPath, certs.CAKey) + if err != nil { + return false, fmt.Errorf("failed to write ca key to %s: %v", caKeyPath, err) + } + updated = changed || updated + + caCertPath := path.Join(p.path, p.CACertName) + changed, err = p.writeFile(caCertPath, certs.CACert) + if err != nil { + return false, fmt.Errorf("failed to write ca cert to %s: %v", caCertPath, err) + } + updated = changed || updated + return updated, nil +} + +func (p *FSProvider) writeFile(path string, data []byte) (bool, error) { + _, err := p.FS.Stat(path) + if err != nil && !os.IsNotExist(err) { + return false, err + } + if os.IsNotExist(err) { + // created + return true, afero.WriteFile(p.FS, path, data, 0o644) + } + // check data + inFile, _ := afero.ReadFile(p.FS, path) + + if bytes.Equal(data, inFile) { + return false, nil + } + // changed + return true, afero.WriteFile(p.FS, path, data, 0o644) +} diff --git a/webhook/cert/secret.go b/webhook/cert/secret.go new file mode 100644 index 0000000..65cb7b7 --- /dev/null +++ b/webhook/cert/secret.go @@ -0,0 +1,160 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + TLSPrivateKeyKey = corev1.TLSPrivateKeyKey + TLSCertKey = corev1.TLSCertKey + TLSCACertKey = "ca.crt" + TLSCAPrivateKeyKey = "ca.key" +) + +type SecretProvider struct { + client SecretClient + namespace string + name string +} + +type SecretClient interface { + Get(ctx context.Context, namespace string, name string) (*corev1.Secret, error) + Create(ctx context.Context, secret *corev1.Secret) error + Update(ctx context.Context, secret *corev1.Secret) error +} + +func NewSecretProvider(client SecretClient, namespace, name string) (*SecretProvider, error) { + if client == nil { + return nil, fmt.Errorf("secret client must not be nil") + } + return &SecretProvider{ + client: client, + namespace: namespace, + name: name, + }, nil +} + +func (p *SecretProvider) Ensure(ctx context.Context, cfg Config) (*ServingCerts, error) { + certs, err := p.Load(ctx) + if err != nil && !IsNotFound(err) { + return nil, err + } + + op := "" + if IsNotFound(err) { + op = "create" + } else if err := certs.Validate(cfg.CommonName); err != nil { + klog.ErrorS(err, "invalid certs in secret") + op = "overwrite" + } + + if len(op) > 0 { + certs, err = GenerateSelfSignedCerts(cfg) + if err != nil { + return nil, err + } + var opErr error + if op == "create" { + opErr = p.create(ctx, certs) + } else { + opErr = p.overwrite(ctx, certs) + } + if opErr != nil { + return nil, opErr + } + return certs, nil + } + + return certs, nil +} + +func (p *SecretProvider) Load(ctx context.Context) (*ServingCerts, error) { + secret, err := p.client.Get(ctx, p.namespace, p.name) + if err != nil { + return nil, err + } + + return convertSecretToCerts(secret), nil +} + +func (p *SecretProvider) create(ctx context.Context, certs *ServingCerts) error { + if certs == nil { + return fmt.Errorf("certs are required") + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: p.namespace, + Name: p.name, + }, + Type: corev1.SecretTypeTLS, + } + + writeCertsToSecret(secret, certs) + // create it + // If there is another controller racer, an AlreadyExistsError may be returned. + return p.client.Create(ctx, secret) +} + +func (p *SecretProvider) overwrite(ctx context.Context, certs *ServingCerts) error { + if certs == nil { + return fmt.Errorf("certs are required") + } + secret, err := p.client.Get(ctx, p.namespace, p.name) + if client.IgnoreNotFound(err) != nil { + // err != NotFound, return it + return err + } + + if apierrors.IsNotFound(err) { + // not found, create new one + return p.create(ctx, certs) + } + + // overwrite existing one + writeCertsToSecret(secret, certs) + + // If there is another controller racer, an Conflict may be returned. + return p.client.Update(ctx, secret) +} + +func convertSecretToCerts(secret *corev1.Secret) *ServingCerts { + return &ServingCerts{ + Key: secret.Data[TLSPrivateKeyKey], + Cert: secret.Data[TLSCertKey], + CAKey: secret.Data[TLSCAPrivateKeyKey], + CACert: secret.Data[TLSCACertKey], + } +} + +func writeCertsToSecret(secret *corev1.Secret, certs *ServingCerts) { + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + secret.Data[TLSPrivateKeyKey] = certs.Key + secret.Data[TLSCertKey] = certs.Cert + secret.Data[TLSCAPrivateKeyKey] = certs.CAKey + secret.Data[TLSCACertKey] = certs.CACert +} diff --git a/webhook/cert/signer.go b/webhook/cert/signer.go new file mode 100644 index 0000000..5955af0 --- /dev/null +++ b/webhook/cert/signer.go @@ -0,0 +1,325 @@ +/** + * Copyright 2024 The KusionStack Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cert + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + errutil "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "kusionstack.io/kube-utils/controller/mixin" + "kusionstack.io/kube-utils/multicluster" +) + +type WebhookCertSelfSigner struct { + *mixin.ReconcilerMixin + CertConfig + + fs *FSProvider + secret *SecretProvider +} + +type CertConfig struct { + Host string + AlternateHosts []string + + Namespace string + SecretName string + MutatingWebhookNames []string + ValidatingWebhookNames []string + + ContextWrapper func(context.Context) context.Context +} + +func New(mgr manager.Manager, cfg CertConfig) *WebhookCertSelfSigner { + return &WebhookCertSelfSigner{ + ReconcilerMixin: mixin.NewReconcilerMixin("webhook-cert-self-signer", mgr), + CertConfig: cfg, + } +} + +func (s *WebhookCertSelfSigner) SetupWithManager(mgr manager.Manager) error { + var err error + server := mgr.GetWebhookServer() + s.fs, err = NewFSProvider(server.CertDir, FSOptions{ + CertName: server.CertName, + KeyName: server.KeyName, + }) + if err != nil { + return err + } + s.secret, err = NewSecretProvider( + &secretClient{ + reader: s.APIReader, + writer: s.Client, + logger: s.Logger, + }, + s.Namespace, + s.SecretName, + ) + if err != nil { + return err + } + + // manually sync certs once + _, err = s.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: s.Namespace, + Name: s.SecretName, + }, + }) + if err != nil { + return err + } + ctrl, _ := controller.NewUnmanaged(s.GetControllerName(), mgr, controller.Options{ + Reconciler: s, + }) + + // add watches for secrets, webhook configs + types := []client.Object{ + &corev1.Secret{}, + &admissionregistrationv1.ValidatingWebhookConfiguration{}, + &admissionregistrationv1.MutatingWebhookConfiguration{}, + } + for i := range types { + t := types[i] + err = ctrl.Watch( + multicluster.FedKind(&source.Kind{Type: t}), + s.enqueueSecret(), + s.predictFunc(), + ) + if err != nil { + return err + } + } + // make controller run as non-leader election + return mgr.Add(&nonLeaderElectionController{Controller: ctrl}) +} + +func (s *WebhookCertSelfSigner) predictFunc() predicate.Funcs { + mutatingWebhookNameSet := sets.NewString(s.MutatingWebhookNames...) + validatingWebhookNameSet := sets.NewString(s.ValidatingWebhookNames...) + return predicate.NewPredicateFuncs(func(obj client.Object) bool { + if obj == nil { + return false + } + switch t := obj.(type) { + case *corev1.Secret: + return t.Namespace == s.Namespace && t.Name == s.SecretName + case *admissionregistrationv1.MutatingWebhookConfiguration: + return mutatingWebhookNameSet.Has(t.Name) + case *admissionregistrationv1.ValidatingWebhookConfiguration: + return validatingWebhookNameSet.Has(t.Name) + } + return false + }) +} + +func (s *WebhookCertSelfSigner) enqueueSecret() handler.EventHandler { + mapFunc := func(obj client.Object) []reconcile.Request { + return []reconcile.Request{ + { + NamespacedName: client.ObjectKey{ + Namespace: s.Namespace, + Name: s.SecretName, + }, + }, + } + } + return handler.EnqueueRequestsFromMapFunc(mapFunc) +} + +func (s *WebhookCertSelfSigner) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + if s.ContextWrapper != nil { + ctx = s.ContextWrapper(ctx) + } + cfg := Config{ + CommonName: s.Host, + AltNames: AltNames{ + DNSNames: s.AlternateHosts, + }, + } + servingCerts, err := s.secret.Ensure(ctx, cfg) + if err != nil { + if IsConflict(err) { + // create error on AlreadyExists + // update error on Conflict + // retry + return reconcile.Result{RequeueAfter: 1 * time.Second}, nil + } + return reconcile.Result{}, err + } + + if servingCerts == nil { + return reconcile.Result{}, fmt.Errorf("got empty serving certs from secret") + } + + // got valid serving certs in secret + // 1. write certs to fs + changed, err := s.fs.Overwrite(servingCerts) + if err != nil { + return reconcile.Result{}, err + } + if changed { + s.Logger.Info("write certs to files successfully") + } + + // 2. update caBundle in webhook configurations + err = s.ensureWebhookConfiguration(ctx, servingCerts.CACert) + if err != nil { + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (s *WebhookCertSelfSigner) ensureWebhookConfiguration(ctx context.Context, caBundle []byte) error { + var errList []error + mutatingCfg := &admissionregistrationv1.MutatingWebhookConfiguration{} + for _, name := range s.MutatingWebhookNames { + var changed bool + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + changed = false + if err := s.APIReader.Get(ctx, types.NamespacedName{Name: name}, mutatingCfg); err != nil { + return err + } + + for i := range mutatingCfg.Webhooks { + if string(mutatingCfg.Webhooks[i].ClientConfig.CABundle) != string(caBundle) { + changed = true + mutatingCfg.Webhooks[i].ClientConfig.CABundle = caBundle + } + } + + if !changed { + return nil + } + + return s.Client.Update(ctx, mutatingCfg) + }); err != nil { + s.Logger.Info("failed to update ca in mutating webhook", "name", name, "error", err.Error()) + if !errors.IsNotFound(err) { + errList = append(errList, fmt.Errorf("failed to update ca in mutating webhook %s: %s", name, err)) + } + continue + } + + if changed { + s.Logger.Info("ensure ca in mutating webhook", "name", name) + } + } + + validatingCfg := &admissionregistrationv1.ValidatingWebhookConfiguration{} + for _, name := range s.ValidatingWebhookNames { + var changed bool + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + changed = false + if err := s.APIReader.Get(ctx, types.NamespacedName{Name: name}, validatingCfg); err != nil { + return err + } + + for i := range validatingCfg.Webhooks { + if string(validatingCfg.Webhooks[i].ClientConfig.CABundle) != string(caBundle) { + changed = true + validatingCfg.Webhooks[i].ClientConfig.CABundle = caBundle + } + } + + if !changed { + return nil + } + + return s.Client.Update(ctx, validatingCfg) + }); err != nil { + s.Logger.Info("failed to update ca in validating webhook", "name", name, "error", err.Error()) + if !errors.IsNotFound(err) { + errList = append(errList, fmt.Errorf("failed to update ca in validating webhook %s: %s", name, err)) + } + continue + } + + if changed { + s.Logger.Info("ensure ca in validating webhook", "name", name) + } + } + + if len(errList) == 0 { + return nil + } + + return errutil.NewAggregate(errList) +} + +type nonLeaderElectionController struct { + controller.Controller +} + +func (c *nonLeaderElectionController) NeedLeaderElection() bool { + return false +} + +var _ SecretClient = &secretClient{} + +type secretClient struct { + reader client.Reader + writer client.Writer + logger logr.Logger +} + +// Create implements SecretClient. +func (s *secretClient) Create(ctx context.Context, secret *corev1.Secret) error { + err := s.writer.Create(ctx, secret) + if err == nil { + s.logger.Info("create secret successfully", "namespace", secret.Namespace, "name", secret.Name) + } + return err +} + +// Get implements SecretClient. +func (s *secretClient) Get(ctx context.Context, namespace string, name string) (*corev1.Secret, error) { + var secret corev1.Secret + err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &secret) + if err != nil { + return nil, err + } + return &secret, nil +} + +// Update implements SecretClient. +func (s *secretClient) Update(ctx context.Context, secret *corev1.Secret) error { + err := s.writer.Update(ctx, secret) + if err == nil { + s.logger.Info("update secret successfully", "namespace", secret.Namespace, "name", secret.Name) + } + return err +} diff --git a/webhook/cert/signer_test.go b/webhook/cert/signer_test.go new file mode 100644 index 0000000..69b426a --- /dev/null +++ b/webhook/cert/signer_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var ( + env *envtest.Environment + mgr manager.Manager + + ctx context.Context + cancel context.CancelFunc + c client.Client + + certPath string + host = "test.com" + alernativeHost = "test.alternative.com" + namespace = "test-ns" + secretName = "test-secret" + mutatingWebhookName = "test-mutating-webhook" + validatingWebhookName = "test-validating-webhook" +) + +var _ = Describe("Self signer", func() { + + It("sign cert for webhooks reconcile", func() { + none := admissionregistrationv1.SideEffectClassNone + mutatingWebhook := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: mutatingWebhookName, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: "hook-1.test.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: strPointer("https://test.io"), + }, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1beta1"}, + }, + { + Name: "hook-2.test.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: strPointer("https://test.io"), + }, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1beta1"}, + }, + }, + } + + validtingWebhook := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: validatingWebhookName, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: "hook-1.test.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: strPointer("https://test.io"), + }, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1beta1"}, + }, + { + Name: "hook-2.test.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: strPointer("https://test.io"), + }, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1beta1"}, + }, + }, + } + + Expect(c.Create(context.TODO(), mutatingWebhook)).Should(BeNil()) + Expect(c.Create(context.TODO(), validtingWebhook)).Should(BeNil()) + + Eventually(func() error { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: mutatingWebhookName}, mutatingWebhook); err != nil { + return err + } + for _, hook := range mutatingWebhook.Webhooks { + if len(hook.ClientConfig.CABundle) == 0 { + return fmt.Errorf("ca bundle is empty") + } + } + + return nil + }, 1*time.Second).Should(BeNil()) + + Eventually(func() error { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: validatingWebhookName}, validtingWebhook); err != nil { + return err + } + for _, hook := range validtingWebhook.Webhooks { + if len(hook.ClientConfig.CABundle) == 0 { + return fmt.Errorf("ca bundle is empty") + } + } + + return nil + }, 1*time.Second).Should(BeNil()) + }) +}) + +func TestResourceContextController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ResourceContext Test Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{} + + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + certPath, err = os.MkdirTemp(os.TempDir(), "cert-self-sign-test-*") + Expect(err).Should(BeNil()) + + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + Scheme: scheme.Scheme, + CertDir: certPath, + }) + Expect(err).NotTo(HaveOccurred()) + + c = mgr.GetClient() + Expect(createNamespace(c, namespace)).Should(BeNil()) + + signer := New(mgr, CertConfig{ + Host: host, + AlternateHosts: []string{alernativeHost}, + Namespace: namespace, + SecretName: secretName, + MutatingWebhookNames: []string{mutatingWebhookName}, + ValidatingWebhookNames: []string{validatingWebhookName}, + }) + Expect(signer.SetupWithManager(mgr)).Should(BeNil()) + // ignore PodDecoration SharedStrategyController + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + cancel() + err := os.RemoveAll(certPath) + Expect(err).Should(BeNil()) + + err = env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} + +func strPointer(str string) *string { + return &str +} From fd449b1de72f7c7d7e5c3a98c3f577ab4ef05429 Mon Sep 17 00:00:00 2001 From: wu8685 Date: Mon, 8 Jul 2024 20:30:47 +0800 Subject: [PATCH 2/2] fix comment issue --- webhook/cert/fs.go | 2 +- webhook/cert/signer.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/webhook/cert/fs.go b/webhook/cert/fs.go index 0d0c37b..509ee85 100644 --- a/webhook/cert/fs.go +++ b/webhook/cert/fs.go @@ -164,7 +164,7 @@ func (p *FSProvider) Overwrite(certs *ServingCerts) (bool, error) { return false, err } if os.IsNotExist(err) { - err = os.MkdirAll(p.path, 0o755) + err = p.FS.MkdirAll(p.path, 0o755) if err != nil { return false, err } diff --git a/webhook/cert/signer.go b/webhook/cert/signer.go index 5955af0..668a46b 100644 --- a/webhook/cert/signer.go +++ b/webhook/cert/signer.go @@ -38,7 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "kusionstack.io/kube-utils/controller/mixin" - "kusionstack.io/kube-utils/multicluster" ) type WebhookCertSelfSigner struct { @@ -114,7 +113,7 @@ func (s *WebhookCertSelfSigner) SetupWithManager(mgr manager.Manager) error { for i := range types { t := types[i] err = ctrl.Watch( - multicluster.FedKind(&source.Kind{Type: t}), + &source.Kind{Type: t}, s.enqueueSecret(), s.predictFunc(), )