温馨提示:
做本实验之前，建议大家为K8S集群所有节点拍快照。
- 服务端证书存储路径
[root@master231 ~]# ll /etc/kubernetes/pki/
total 72
drwxr-xr-x 3 root root 4096 Jun 3 09:27 ./
drwxr-xr-x 4 root root 4096 May 22 10:57 ../
-rw-r--r-- 1 root root 1285 May 22 10:57 apiserver.crt
-rw-r--r-- 1 root root 1155 May 22 10:57 apiserver-etcd-client.crt
-rw------- 1 root root 1679 May 22 10:57 apiserver-etcd-client.key
-rw------- 1 root root 1675 May 22 10:57 apiserver.key
-rw-r--r-- 1 root root 1164 May 22 10:57 apiserver-kubelet-client.crt
-rw------- 1 root root 1675 May 22 10:57 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1099 May 22 10:57 ca.crt
-rw------- 1 root root 1679 May 22 10:57 ca.key
drwxr-xr-x 2 root root 4096 May 22 10:57 etcd/
-rw-r--r-- 1 root root 1115 May 22 10:57 front-proxy-ca.crt
-rw------- 1 root root 1679 May 22 10:57 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 May 22 10:57 front-proxy-client.crt
-rw------- 1 root root 1675 May 22 10:57 front-proxy-client.key
-rw------- 1 root root 1679 May 22 10:57 sa.key
-rw------- 1 root root 451 May 22 10:57 sa.pub
-rw-r--r-- 1 root root 258 Jun 3 09:27 token.csv
[root@master231 ~]#
[root@master231 ~]# ll /etc/kubernetes/pki/etcd/
total 40
drwxr-xr-x 2 root root 4096 May 22 10:57 ./
drwxr-xr-x 3 root root 4096 Jun 3 09:27 ../
-rw-r--r-- 1 root root 1086 May 22 10:57 ca.crt
-rw------- 1 root root 1679 May 22 10:57 ca.key
-rw-r--r-- 1 root root 1159 May 22 10:57 healthcheck-client.crt
-rw------- 1 root root 1675 May 22 10:57 healthcheck-client.key
-rw-r--r-- 1 root root 1200 May 22 10:57 peer.crt
-rw------- 1 root root 1675 May 22 10:57 peer.key
-rw-r--r-- 1 root root 1200 May 22 10:57 server.crt
-rw------- 1 root root 1675 May 22 10:57 server.key
[root@master231 ~]#
- 客户端证书存储路径
[root@worker232 ~]# ll /var/lib/kubelet/pki/
total 20
drwxr-xr-x 2 root root 4096 Jul 29 16:48 ./
drwx------ 8 root root 4096 Jul 29 16:48 ../
-rw------- 1 root root 1114 Jul 29 16:48 kubelet-client-2024-07-29-08-48-04.pem
lrwxrwxrwx 1 root root 59 Jul 29 16:48 kubelet-client-current.pem -> /var/lib/kubelet/pki/kubelet-client-2024-07-29-08-48-04.pem
-rw-r--r-- 1 root root 2258 Jul 29 16:48 kubelet.crt
-rw------- 1 root root 1675 Jul 29 16:48 kubelet.key
[root@worker232 ~]#
- 检查 kubeadm 管理的本地 PKI 中证书的到期时间
[root@master231 ~]# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
W0613 11:26:52.024597 17246 utils.go:69] The recommended value for "resolvConf" in "KubeletConfiguration" is: /run/systemd/resolve/resolv.conf; the provided value is: /run/systemd/resolve/resolv.conf
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf May 22, 2026 02:57 UTC 342d ca no
apiserver May 22, 2026 02:57 UTC 342d ca no
apiserver-etcd-client May 22, 2026 02:57 UTC 342d etcd-ca no
apiserver-kubelet-client May 22, 2026 02:57 UTC 342d ca no
controller-manager.conf May 22, 2026 02:57 UTC 342d ca no
etcd-healthcheck-client May 22, 2026 02:57 UTC 342d etcd-ca no
etcd-peer May 22, 2026 02:57 UTC 342d etcd-ca no
etcd-server May 22, 2026 02:57 UTC 342d etcd-ca no
front-proxy-client May 22, 2026 02:57 UTC 342d front-proxy-ca no
scheduler.conf May 22, 2026 02:57 UTC 342d ca no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca May 20, 2035 02:57 UTC 9y no
etcd-ca May 20, 2035 02:57 UTC 9y no
front-proxy-ca May 20, 2035 02:57 UTC 9y no
[root@master231 ~]#
推荐阅读:
https://kubernetes.io/zh-cn/docs/reference/setup-tools/kubeadm/kubeadm-certs/
- 升级master节点的证书
[root@master231 ~]# kubeadm certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
W0613 11:30:03.679872 24568 utils.go:69] The recommended value for "resolvConf" in "KubeletConfiguration" is: /run/systemd/resolve/resolv.conf; the provided value is: /run/systemd/resolve/resolv.conf
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
Done renewing certificates. You must restart the kube-apiserver, kube-controller-manager, kube-scheduler and etcd, so that they can use the new certificates.
[root@master231 ~]#
[root@master231 ~]# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
W0613 11:30:19.059791 25126 utils.go:69] The recommended value for "resolvConf" in "KubeletConfiguration" is: /run/systemd/resolve/resolv.conf; the provided value is: /run/systemd/resolve/resolv.conf
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Jun 13, 2026 03:30 UTC 364d ca no
apiserver Jun 13, 2026 03:30 UTC 364d ca no
apiserver-etcd-client Jun 13, 2026 03:30 UTC 364d etcd-ca no
apiserver-kubelet-client Jun 13, 2026 03:30 UTC 364d ca no
controller-manager.conf Jun 13, 2026 03:30 UTC 364d ca no
etcd-healthcheck-client Jun 13, 2026 03:30 UTC 364d etcd-ca no
etcd-peer Jun 13, 2026 03:30 UTC 364d etcd-ca no
etcd-server Jun 13, 2026 03:30 UTC 364d etcd-ca no
front-proxy-client Jun 13, 2026 03:30 UTC 364d front-proxy-ca no
scheduler.conf Jun 13, 2026 03:30 UTC 364d ca no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca May 20, 2035 02:57 UTC 9y no
etcd-ca May 20, 2035 02:57 UTC 9y no
front-proxy-ca May 20, 2035 02:57 UTC 9y no
[root@master231 ~]#
[root@master231 ~]# ll /etc/kubernetes/pki/
total 72
drwxr-xr-x 3 root root 4096 Jun 3 09:27 ./
drwxr-xr-x 4 root root 4096 May 22 10:57 ../
-rw-r--r-- 1 root root 1285 Jun 13 11:30 apiserver.crt
-rw-r--r-- 1 root root 1155 Jun 13 11:30 apiserver-etcd-client.crt
-rw------- 1 root root 1679 Jun 13 11:30 apiserver-etcd-client.key
-rw------- 1 root root 1679 Jun 13 11:30 apiserver.key
-rw-r--r-- 1 root root 1164 Jun 13 11:30 apiserver-kubelet-client.crt
-rw------- 1 root root 1675 Jun 13 11:30 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1099 May 22 10:57 ca.crt
-rw------- 1 root root 1679 May 22 10:57 ca.key
drwxr-xr-x 2 root root 4096 May 22 10:57 etcd/
-rw-r--r-- 1 root root 1115 May 22 10:57 front-proxy-ca.crt
-rw------- 1 root root 1679 May 22 10:57 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 Jun 13 11:30 front-proxy-client.crt
-rw------- 1 root root 1675 Jun 13 11:30 front-proxy-client.key
-rw------- 1 root root 1679 May 22 10:57 sa.key
-rw------- 1 root root 451 May 22 10:57 sa.pub
-rw-r--r-- 1 root root 258 Jun 3 09:27 token.csv
[root@master231 ~]#
[root@master231 ~]# ll /etc/kubernetes/pki/etcd/
total 40
drwxr-xr-x 2 root root 4096 May 22 10:57 ./
drwxr-xr-x 3 root root 4096 Jun 3 09:27 ../
-rw-r--r-- 1 root root 1086 May 22 10:57 ca.crt
-rw------- 1 root root 1679 May 22 10:57 ca.key
-rw-r--r-- 1 root root 1159 Jun 13 11:30 healthcheck-client.crt
-rw------- 1 root root 1679 Jun 13 11:30 healthcheck-client.key
-rw-r--r-- 1 root root 1200 Jun 13 11:30 peer.crt
-rw------- 1 root root 1675 Jun 13 11:30 peer.key
-rw-r--r-- 1 root root 1200 Jun 13 11:30 server.crt
-rw------- 1 root root 1675 Jun 13 11:30 server.key
[root@master231 ~]#
- 升级worker节点的证书
温馨提示:
请确保实验前所有的worker节点是正常工作的。如果有NotReady节点，可能会导致该节点证书无法自动续期，因为该节点可能无法和ApiServer建立链接。
[root@master231 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master231 Ready control-plane,master 364d v1.23.17
worker232 Ready <none> 364d v1.23.17
worker233 Ready <none> 357d v1.23.17
[root@master231 ~]#
1.升级前查看
[root@worker232 ~]# openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -text -noout
Certificate:
Data:
...
Validity
Not Before: May 22 02:57:42 2025 GMT
Not After : May 22 02:57:44 2026 GMT
Subject: O = system:nodes, CN = system:node:master231
[root@worker233 ~]# openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -text -noout
Certificate:
Data:
...
Validity
Not Before: May 28 08:27:45 2025 GMT
Not After : May 28 08:27:45 2026 GMT
Subject: O = system:nodes, CN = system:node:worker233
2.使用kube-controller-manager进行续签证书:
参考链接:
https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-controller-manager/
3.修改静态Pod的kube-controller-manager资源清单
[root@master231 ~]# vim /etc/kubernetes/manifests/kube-controller-manager.yaml
...
spec:
containers:
- command:
- kube-controller-manager
...
# 所签名证书的有效期限。每个 CSR 可以通过设置 spec.expirationSeconds 来请求更短的证书。
- --cluster-signing-duration=87600h0m0s
# 启用controner manager自动签发CSR证书，可以不配置，默认就是启用的，但是建议配置上！害怕未来版本发生变化！
- --feature-gates=RotateKubeletServerCertificate=true
4.验证kube-controller-manager是否启动成功。
[root@master231 ~]# mv /etc/kubernetes/manifests/kube-controller-manager.yaml /opt/
[root@master231 ~]#
[root@master231 ~]# mv /opt/kube-controller-manager.yaml /etc/kubernetes/manifests/
[root@master231 ~]#
[root@master231 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true","reason":""}
[root@master231 ~]#
[root@master231 ~]# kubectl get pods -n kube-system -l component=kube-controller-manager -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-controller-manager-master231 1/1 Running 0 15s 10.0.0.231 master231 <none> <none>
[root@master231 ~]#
5.要求kubelet的配置文件中支持证书滚动，默认是启用的，无需配置。
[root@worker232 ~]# vim /var/lib/kubelet/config.yaml
...
rotateCertificates: true
6.客户端节点修改节点的时间
centos操作如下：【可跳过】
[root@worker232 ~]# date -s "2026-6-4"
[root@worker232 ~]#
[root@worker232 ~]# systemctl restart kubelet
ubuntu系统操作如下：
6.1 先关闭时间同步服务
[root@worker232 ~]# timedatectl set-ntp off
[root@worker232 ~]#
[root@worker233 ~]# timedatectl set-ntp off
[root@worker233 ~]#
6.2 修改即将过期的时间的前一天
[root@worker232 ~]# timedatectl set-time '2026-05-21 15:30:00' # 注意，这个时间点是你的客户端证书的到期的前一天！！！！
[root@worker232 ~]#
[root@worker232 ~]# date -R
Thu, 21 May 2026 15:30:02 +0800
[root@worker232 ~]#
[root@worker233 ~]# timedatectl set-time '2026-05-27 22:30:00'
[root@worker233 ~]#
[root@worker233 ~]# date -R
Wed, 27 May 2026 15:30:02 +0800
[root@worker233 ~]#
7.重启kubelet
[root@worker232 ~]# ll /var/lib/kubelet/pki/
total 20
drwxr-xr-x 2 root root 4096 May 22 2025 ./
drwx------ 8 root root 4096 May 22 2025 ../
-rw------- 1 root root 1114 May 22 2025 kubelet-client-2025-05-22-11-03-52.pem
lrwxrwxrwx 1 root root 59 May 22 2025 kubelet-client-current.pem -> /var/lib/kubelet/pki/kubelet-client-2025-05-22-11-03-52.pem
-rw-r--r-- 1 root root 2258 May 22 2025 kubelet.crt
-rw------- 1 root root 1675 May 22 2025 kubelet.key
[root@worker232 ~]#
[root@worker232 ~]# systemctl restart kubelet
[root@worker232 ~]#
[root@worker232 ~]# ll /var/lib/kubelet/pki/
total 24
drwxr-xr-x 2 root root 4096 May 21 15:30 ./
drwx------ 8 root root 4096 May 22 2025 ../
-rw------- 1 root root 1114 May 22 2025 kubelet-client-2025-05-22-11-03-52.pem
-rw------- 1 root root 1114 May 21 15:30 kubelet-client-2026-05-21-15-30-36.pem
lrwxrwxrwx 1 root root 59 May 21 15:30 kubelet-client-current.pem -> /var/lib/kubelet/pki/kubelet-client-2026-05-21-15-30-36.pem
-rw-r--r-- 1 root root 2258 May 22 2025 kubelet.crt
-rw------- 1 root root 1675 May 22 2025 kubelet.key
[root@worker232 ~]#
[root@worker233 ~]# systemctl restart kubelet.service
[root@worker233 ~]#
[root@worker233 ~]# ll /var/lib/kubelet/pki/
total 20
drwxr-xr-x 2 root root 4096 May 28 16:32 ./
drwxr-xr-x 8 root root 4096 May 28 16:32 ../
-rw------- 1 root root 1114 May 28 16:32 kubelet-client-2025-05-28-16-32-45.pem
lrwxrwxrwx 1 root root 59 May 28 16:32 kubelet-client-current.pem -> /var/lib/kubelet/pki/kubelet-client-2025-05-28-16-32-45.pem
-rw-r--r-- 1 root root 2258 May 28 16:32 kubelet.crt
-rw------- 1 root root 1675 May 28 16:32 kubelet.key
[root@worker233 ~]#
9.查看客户端的证书有效期
[root@worker232 ~]# openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -text -noout
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
24:2b:28:e3:6f:74:4a:0f:b0:25:c2:4d:29:e9:25:1a
Signature Algorithm: sha256WithRSAEncryption
Issuer: CN = kubernetes
Validity
Not Before: Jun 13 03:47:01 2025 GMT
Not After : May 20 02:57:42 2035 GMT # Duang~证书续期了10年！
...
10.验证能够正常工作（如果无法创建Pod，则需要删除一下calico的名称空间的Pod）
[root@master231 ~]# kubectl get pods -o wide -n calico-system
[root@master231 ~]# kubectl get pods -o wide -n calico-apiserver
[root@master231 ~]# kubectl -n calico-apiserver delete pods --all
[root@master231 ~]# kubectl get pods -o wide -n calico-apiserver
[root@master231 ~]#
[root@master231 ~]# kubectl apply -f oldboyedu-network-cni.yaml
pod/xiuxian-v1 created
pod/xiuxian-v2 created
[root@master231 ~]#
[root@master231 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
test-pods 1/1 Running 0 306d 10.100.203.136 worker232 <none> <none>
xiuxian-v1 0/1 ContainerCreating 0 1s <none> worker232 <none> <none>
xiuxian-v2 0/1 ContainerCreating 0 1s <none> worker233 <none> <none>
xiuxian-v3-fbbcf9474-bwmq2 1/1 Running 0 306d 10.100.160.133 master231 <none> <none>
[root@master231 ~]#
[root@master231 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
test-pods 1/1 Running 0 306d 10.100.203.136 worker232 <none> <none>
xiuxian-v1 1/1 Running 0 3s 10.100.203.140 worker232 <none> <none>
xiuxian-v2 1/1 Running 0 3s 10.100.140.76 worker233 <none> <none>
xiuxian-v3-fbbcf9474-bwmq2 1/1 Running 0 306d 10.100.160.133 master231 <none> <none>
[root@master231 ~]#
温馨提示:
生产环境中对于worker证书升级应该注意的事项：
– 对生产环境一定要有敬畏之心，不可随意;
– 对证书有效期有效期进行监控，很多开源组件都支持，比如zabbix，prometheus等。
– 在重启kubelet节点时，应该注意滚动更新，不要批量重启，避免Pod大面积无法访问的情况，从而造成业务的损失，甚至生产故障;
– 尽量在业务的低谷期做升级，或者证书续期，备份，恢复等集群操作，影响最小;