阿里云ACK的etcd证书过期手工升级操作-Toy模板网

这篇具有很好参考价值的文章主要介绍了阿里云ACK的etcd证书过期手工升级操作。希望对大家有所帮助。如果存在错误或未考虑完全的地方，请大家不吝赐教，您也可以点击"举报违法"按钮提交疑问。

一.问题现象

阿里云ACK的etcd证书过期，通过图形化界面升级提示升级失败，考虑通过脚本的方式升级ETCD相关的证书。由于在前期做类似的升级ETCD证书失败导致整个集群业务出现访问异常，所有在升级之前做好对应的备份操作是很有必要的

二.前期准备

1.由于升级操作是在master节点操作，涉及的证书文件变更都在master节点，所以针对阿里云ACK的3台master节点做快照。

2.针对阿里云ACK的3台master节点涉及的证书目录做备份，必要情况下备份到其他服务器上。

3.针对阿里云ACK的etcd数据库做备份操作

date;
CACERT="/var/lib/etcd/cert/ca.pem"
CERT="/var/lib/etcd/cert/etcd-server.pem"
EKY="/var/lib/etcd/cert/etcd-server-key.pem"
ENDPOINTS="172.16.0.87:2379"

#etcdctl snapshot save /data/etcd_backup_dir/etcd-snapshot-`date +%Y%m%d`.db

ETCDCTL_API=3 etcdctl \
--cacert="${CACERT}" --cert="${CERT}" --key="${EKY}" \
--endpoints=${ENDPOINTS} \
snapshot save /data/etcd_backup_dir/etcd-snapshot-`date +%Y%m%d`.db

备份ETCD的数据目录

cp -r /var/lib/etcd /var/lib/etcd-`date +%Y%m%d`.bak

4.针对阿里云ACK的所有部署资源的yaml文件做备份，涉及service deploy ingress configmap secret job cronjob daemonset statefulset pvc等，备份脚本如下：

#!/bin/bash
#define variable
BACKUP_PATH=/data/k8s-backup
BACKUP_PATH_DATA=$BACKUP_PATH/yaml/`date +%Y%m%d%H%M%S`
BACKUP_PATH_LOG=$BACKUP_PATH/log
BACKUP_LOG_FILE=$BACKUP_PATH_LOG/k8s-backup-`date +%Y%m%d%H%M%S`.log
# base function
function printlog(){
 echo "`date +'%Y-%m-%d %H:%M:%S'` $1"
 echo "`date +'%Y-%m-%d %H:%M:%S'` $1" >> $BACKUP_LOG_FILE 2>&1 
}
function printlogonly(){
 echo "`date +'%Y-%m-%d %H:%M:%S'` $1" >> $BACKUP_LOG_FILE 2>&1 
}
# set K8s type（此处可根据集群资源自行修改）
CONFIG_TYPE="service deploy ingress configmap secret job cronjob daemonset statefulset pvc"
# make dir
mkdir -p $BACKUP_PATH_DATA
mkdir -p $BACKUP_PATH_LOG
cd $BACKUP_PATH_DATA
# set namespace list
ns_list=`kubectl get ns | awk '{print $1}' | grep -v NAME`
if [ $# -ge 1 ]; then
ns_list="$@"
fi
# define counters
COUNT_NS=0
COUNT_ITEM_IN_NS=0
COUNT_ITEM_IN_TYPE=0
COUNT_ITEM_ALL=0
# print hint
printlog "Backup kubernetes config in namespaces: ${ns_list}"
printlog "Backup kubernetes config for [type: ${CONFIG_TYPE}]."
printlog "If you want to read the record of backup, please input command ' tail -100f ${BACKUP_LOG_FILE} '"
# ask and answer
message="This will backup resources of kubernetes cluster to yaml files."
printlog ${message}ls

# loop for namespaces
for ns in $ns_list;
do
COUNT_NS=`expr $COUNT_NS + 1`
printlog "Backup No.${COUNT_NS} namespace [namespace: ${ns}]."
COUNT_ITEM_IN_NS=0

## loop for types
for type in $CONFIG_TYPE; 
do
printlogonly "Backup type [namespace: ${ns}, type: ${type}]."
item_list=`kubectl -n $ns get $type | awk '{print $1}' | grep -v NAME | grep -v "No "`
COUNT_ITEM_IN_TYPE=0

## loop for items
for item in $item_list; 
do 
file_name=$BACKUP_PATH_DATA/${ns}_${type}_${item}.yaml
printlogonly "Backup kubernetes config yaml [namespace: ${ns}, type: ${type}, item: ${item}] to file: ${file_name}"
kubectl -n $ns get $type $item -o yaml > $file_name
COUNT_ITEM_IN_NS=`expr $COUNT_ITEM_IN_NS + 1`
COUNT_ITEM_IN_TYPE=`expr $COUNT_ITEM_IN_TYPE + 1`
COUNT_ITEM_ALL=`expr $COUNT_ITEM_ALL + 1`
printlogonly "Backup No.$COUNT_ITEM_ALL file done."
done;

done;
printlogonly "Backup $COUNT_ITEM_IN_TYPE files in [namespace: ${ns}, type: ${type}]."

printlog "Backup ${COUNT_ITEM_IN_NS} files done in [namespace: ${ns}]."
done;

# show stats
printlog "Backup ${COUNT_ITEM_ALL} yaml files in all."
printlog "kubernetes Backup completed, all done."
exit 0

三.升级证书操作步骤

1.确认集群Master节点之间配置了root用户的免密登录。

在Master上通过SSH方式登录其他任意Master节点，如果提示输入密码，请您按如下方式配置Master节点之间的免密登录。

1. ssh-keygen -t rsa  # 生成密钥。
2. ssh-copy-id -i ~/.ssh/id_rsa.pub $(internal-ip)     # 使用ssh-copy-id工具传输公钥到其他所有Master节点，$（internal-ip）为其他Master节点的内网IP。

2.分别复制以下脚本内容，保存并命名为restart-apiserver.sh和rotate-etcd.sh，然后将两者保存到同一个文件夹下

restart-apiserver.sh

 #! /bin/bash

function restart_apiserver() {
  apiserverID=$(/usr/bin/docker ps | grep kube-apiserver | grep -v NAME | awk '{print $1}')
  /usr/bin/docker stop $apiserverID
  rm -rf /root/kube-apiserver.yaml
  mv /etc/kubernetes/manifests/kube-apiserver.yaml /root/kube-apiserver.yaml
  while true; do
    NUM=$(docker ps | grep kube-apiserver | wc -l)
    if [[ $NUM == 0 ]]; then
      break
    fi
    sleep 1
  done
  /usr/bin/docker ps -a | grep kube-apiserver | awk '{print $1}' | xargs docker stop
  /usr/bin/docker ps -a | grep kube-apiserver | awk '{print $1}' | xargs docker rm

  mv /root/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml
  while true; do
    NUM=$(docker ps | grep kube-apiserver | grep -v pause | wc -l)
    if [[ $NUM == 1 ]]; then
      break
    fi
    sleep 1
  done

  k8s::wait_apiserver_ready
}

k8s::wait_apiserver_ready() {
	set -e
	for i in $(seq 180); do
		if kubectl get po &>/dev/null; then
			return 0
		else
			echo "wait apiserver to be ready, retry ${i}th after 1s"
			sleep 1
		fi
	done

	echo "failed to wait apiserver to be ready"
	return 1
}

function restart_container() {
  crictl pods | grep kube-apiserver | awk '{print $1}' | xargs -I '{}' crictl stopp {} || true
}

if [[ -f /usr/bin/docker ]]; then
  restart_apiserver
else
  restart_container
  k8s::wait_apiserver_ready
fi
echo "API Server restarted"

rotate-etcd.sh

#!/bin/bash

set -eo pipefail

dir=/tmp/etcdcert
KUBE_CERT_PATH=/etc/kubernetes/pki
ETCD_CERT_DIR=/var/lib/etcd/cert
ETCD_HOSTS=""
function get_etcdhosts() {
  name1=$(ls $ETCD_CERT_DIR | grep name | grep name-1.pem | sed 's/-name-1.pem//g')
  name2=$(ls $ETCD_CERT_DIR | grep name | grep name-2.pem | sed 's/-name-2.pem//g')
  name3=$(ls $ETCD_CERT_DIR | grep name | grep name-3.pem | sed 's/-name-3.pem//g')

  echo "hosts: " $name1 $name2 $name3
  #  ETCD_HOSTS=($name1 $name2 $name3)
  ETCD_HOSTS=$name1" "$name2" "$name3
}

function gencerts() {
  echo "generate ssl cert ..."
  rm -rf $dir
  mkdir -p "$dir"
  hosts=$(echo $ETCD_HOSTS | tr -s " " ",")

  echo "-----generate ca"
  echo '{"CN":"CA","key":{"algo":"rsa","size":2048}, "ca": {"expiry": "438000h"}}' |
    cfssl gencert -initca - | cfssljson -bare $dir/ca -
  echo '{"signing":{"default":{"expiry":"438000h","usages":["signing","key encipherment","server auth","client auth"]}}}' >$dir/ca-config.json

  echo "-----generate etcdserver"
  export ADDRESS=$hosts,ext1.example.com,coreos1.local,coreos1
  export NAME=etcd-server
  echo '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' |
    cfssl gencert -config=$dir/ca-config.json -ca=$dir/ca.pem -ca-key=$dir/ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $dir/$NAME
  export ADDRESS=
  export NAME=etcd-client
  echo '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' |
    cfssl gencert -config=$dir/ca-config.json -ca=$dir/ca.pem -ca-key=$dir/ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $dir/$NAME

  # gen peer-ca
  echo "-----generate peer certificates"
  echo '{"CN":"Peer-CA","key":{"algo":"rsa","size":2048}, "ca": {"expiry": "438000h"}}' | cfssl gencert -initca - | cfssljson -bare $dir/peer-ca -
  echo '{"signing":{"default":{"expiry":"438000h","usages":["signing","key encipherment","server auth","client auth"]}}}' >$dir/peer-ca-config.json
  i=0
  for host in $ETCD_HOSTS; do
    ((i = i + 1))
    export MEMBER=${host}-name-$i
    echo '{"CN":"'${MEMBER}'","hosts":[""],"key":{"algo":"rsa","size":2048}}' |
      cfssl gencert -ca=$dir/peer-ca.pem -ca-key=$dir/peer-ca-key.pem -config=$dir/peer-ca-config.json -profile=peer \
        -hostname="$hosts,${MEMBER}.local,${MEMBER}" - | cfssljson -bare $dir/${MEMBER}
    #-hostname="$host,${MEMBER}.local,${MEMBER}" - | cfssljson -bare $dir/${MEMBER}
  done

  ## backup
  TIMESTAMP=$(date "+%Y%m%d%H%M%S")
  \cp -r $ETCD_CERT_DIR $ETCD_CERT_DIR_$TIMESTAMP
  \cp -r $KUBE_CERT_PATH/etcd $KUBE_CERT_PATH/etcd_$TIMESTAMP

  # 制作bundle ca
  cat $KUBE_CERT_PATH/etcd/ca.pem >>$dir/bundle_ca.pem
  cat $ETCD_CERT_DIR/ca.pem >>$dir/bundle_ca.pem
  cat $dir/ca.pem >>$dir/bundle_ca.pem

  # 制作bundle peer-ca
  cat $ETCD_CERT_DIR/peer-ca.pem >$dir/bundle_peer-ca.pem
  cat $dir/peer-ca.pem >>$dir/bundle_peer-ca.pem

  # chown
  chown -R etcd:etcd $dir
  chmod 0644 $dir/*
}

function rotate_etcd_ca() {
  # Update certs on etcd nodes.
  for ADDR in $ETCD_HOSTS; do
    TIMESTAMP=$(date "+%Y%m%d%H%M%S")
    ssh -o StrictHostKeyChecking=no root@$ADDR cp -r $ETCD_CERT_DIR $ETCD_CERT_DIR_$TIMESTAMP
    echo "update etcd CA on node $ADDR"
    scp -o StrictHostKeyChecking=no $dir/bundle_ca.pem root@$ADDR:$ETCD_CERT_DIR/ca.pem
    scp -o StrictHostKeyChecking=no $dir/bundle_ca.pem root@$ADDR:$KUBE_CERT_PATH/etcd/ca.pem
    scp -o StrictHostKeyChecking=no $dir/etcd-client.pem root@$ADDR:$KUBE_CERT_PATH/etcd/etcd-client.pem
    scp -o StrictHostKeyChecking=no $dir/etcd-client-key.pem root@$ADDR:$KUBE_CERT_PATH/etcd/etcd-client-key.pem
    scp -o StrictHostKeyChecking=no $dir/bundle_peer-ca.pem root@$ADDR:$ETCD_CERT_DIR/peer-ca.pem
    ssh -o StrictHostKeyChecking=no root@$ADDR chown -R etcd:etcd $ETCD_CERT_DIR
    ssh -o StrictHostKeyChecking=no root@$ADDR chmod 0644 $ETCD_CERT_DIR/*
    echo "restart etcd on node $ADDR"
    ssh -o StrictHostKeyChecking=no root@$ADDR systemctl restart etcd
    echo "etcd on node $ADDR restarted"
    ssh -o StrictHostKeyChecking=no root@$ADDR /usr/bin/bash /tmp/restart-apiserver.sh
    echo "apiserver on node $ADDR restarted"
    sleep 10
  done
}

function rotate_etcd_certs() {
  for ADDR in $ETCD_HOSTS; do
    echo "update etcd peer certs on node $ADDR"
    scp -o StrictHostKeyChecking=no \
      $dir/{peer-ca-key.pem,etcd-server.pem,etcd-server-key.pem,etcd-client.pem,etcd-client-key.pem,ca-key.pem,*-name*.pem} root@$ADDR:$ETCD_CERT_DIR/

    ssh -o StrictHostKeyChecking=no root@$ADDR chown -R etcd:etcd $ETCD_CERT_DIR

    ssh -o StrictHostKeyChecking=no root@$ADDR \
       chmod 0400 $ETCD_CERT_DIR/{peer-ca-key.pem,etcd-server.pem,etcd-server-key.pem,etcd-client.pem,etcd-client-key.pem,ca-key.pem,*-name*.pem}

    echo "restart etcd on node $ADDR"
    ssh -o StrictHostKeyChecking=no root@$ADDR systemctl restart etcd
    echo "etcd on node $ADDR restarted"
    sleep 10
  done
}

function recover_etcd_ca() {
  # Update certs on etcd nodes.
  for ADDR in $ETCD_HOSTS; do
    echo "replace etcd CA on node $ADDR"
    scp -o StrictHostKeyChecking=no $dir/ca.pem root@$ADDR:$ETCD_CERT_DIR/ca.pem
    scp -o StrictHostKeyChecking=no $dir/ca.pem root@$ADDR:$KUBE_CERT_PATH/etcd/ca.pem
    scp -o StrictHostKeyChecking=no $dir/peer-ca.pem root@$ADDR:$ETCD_CERT_DIR/peer-ca.pem
    scp -o StrictHostKeyChecking=no $dir/ca.pem root@$ADDR:$KUBE_CERT_PATH/etcd/ca.pem
    ssh -o StrictHostKeyChecking=no root@$ADDR chown -R etcd:etcd $ETCD_CERT_DIR
    echo "restart apiserver on node $ADDR"
    ssh -o StrictHostKeyChecking=no root@$ADDR bash /tmp/restart-apiserver.sh
    echo "apiserver on node $ADDR restarted"
    echo "restart etcd on node $ADDR"
    sleep 5
    ssh -o StrictHostKeyChecking=no root@$ADDR systemctl restart etcd
    echo "etcd on node $ADDR restarted"
    sleep 5
  done
}

function renew_k8s_certs() {
  # 更新K8s证书，根据集群Region替换下面cn-hangzhou的默认镜像地域。
  for ADDR in $ETCD_HOSTS; do
    echo "renew k8s components cert on node $ADDR"
    #compatible containerd
    set +e
    ssh -o StrictHostKeyChecking=no root@$ADDR docker run --privileged=true  -v /:/alicoud-k8s-host --pid host --net host \
             registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0 /renew/upgrade-k8s.sh --role master

    ssh -o StrictHostKeyChecking=no root@$ADDR ctr image pull registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0
    ssh -o StrictHostKeyChecking=no root@$ADDR ctr run --privileged=true --mount type=bind,src=/,dst=/alicoud-k8s-host,options=rbind:rw \
            --net-host registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0 cert-rotate /renew/upgrade-k8s.sh --role master
    set -e
    echo "finished renew k8s components cert on $ADDR"
    sleep 5
  done
}

function generate_cm() {
    echo "generate status configmap"

    cat <<-"EOF" > /tmp/ack-rotate-etcd-ca-cm.yaml.tpl
apiVersion: v1
kind: ConfigMap
metadata:
  name: ack-rotate-etcd-status
  namespace: kube-system
data:
  status: "success"
  hosts: "$hosts"
EOF

    sed -e "s#\$hosts#$ETCD_HOSTS#" /tmp/ack-rotate-etcd-ca-cm.yaml.tpl | kubectl apply -f -
}

get_etcdhosts
echo "${ETCD_HOSTS[@]}"

echo "---renew k8s components certs---"
renew_k8s_certs
echo "---end to renew k8s components certs---"

# Update certs on etcd nodes.
for ADDR in $ETCD_HOSTS; do
  scp -o StrictHostKeyChecking=no restart-apiserver.sh root@$ADDR:/tmp/restart-apiserver.sh
  ssh -o StrictHostKeyChecking=no root@$ADDR chmod +x /tmp/restart-apiserver.sh
done

gencerts
echo "---rotate etcd ca and etcd client ca---"
rotate_etcd_ca
echo "---end to rotate etcd ca and etcd client ca---"
echo
echo "---rotate etcd peer and certs---"
rotate_etcd_certs
echo "---end to rotate etcd peer and certs---"

echo
echo "---replace etcd ca---"
recover_etcd_ca
echo "---end to replace etcd ca---"

generate_cm
echo "etcd CA and certs have succesfully rotated!"

提前下载镜像文件 registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0

镜像里面的/renew/upgrade-k8s.sh

#!/bin/sh
set -xe

if [ -d "/alicoud-k8s-host" ]; then
	rm -rf /alicoud-k8s-host/usr/local/k8s-upgrade
	mkdir -p /alicoud-k8s-host/usr/local/k8s-upgrade
	cp -r /renew/* /alicoud-k8s-host/usr/local/k8s-upgrade
	ls -l /alicoud-k8s-host/usr/local/k8s-upgrade
	chmod  -R +x /alicoud-k8s-host/usr/local/k8s-upgrade/
	chroot /alicoud-k8s-host /usr/local/k8s-upgrade/rotate.sh "$@"
fi

rotate.sh

#!/bin/sh
set -xe

if [ -d "/alicoud-k8s-host" ]; then
	rm -rf /alicoud-k8s-host/usr/local/k8s-upgrade
	mkdir -p /alicoud-k8s-host/usr/local/k8s-upgrade
	cp -r /renew/* /alicoud-k8s-host/usr/local/k8s-upgrade
	ls -l /alicoud-k8s-host/usr/local/k8s-upgrade
	chmod  -R +x /alicoud-k8s-host/usr/local/k8s-upgrade/
	chroot /alicoud-k8s-host /usr/local/k8s-upgrade/rotate.sh "$@"
fi





[root@master01 ~]# docker run -it registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0 cat /renew/rotate.sh
#!/usr/bin/env bash

set -e -x

public::common::log() {
  echo $(date +"[%Y%m%d %H:%M:%S]: ") $1
}

function retry() {
  local n=0
  local try=$1
  local cmd="${@:2}"
  [[ $# -le 1 ]] && {
    echo "Usage $0 <retry_number> <Command>"
  }
  set +e
  until
    [[ $n -ge $try ]]
  do
    $cmd && break || {
      echo "Command Fail.."
      ((n++))
      echo "retry $n :: [$cmd]"
      sleep 2
    }
  done
  set -e
}

public::upgrade::backupmaster() {
  local backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)
  if [ ! -f $backup_dir/kubelet.conf ]; then
    mkdir -p $backup_dir $backup_dir/kubelet $backup_dir/etcd
    cp -rf /etc/kubernetes/ $backup_dir/
    cp /etc/kubeadm/kubeadm.cfg $backup_dir/
    cp /etc/systemd/system/kubelet.service.d/10-kubeadm.conf $backup_dir
    cp /etc/kubernetes/kubelet.conf $backup_dir
    cp -rf /var/lib/kubelet/pki/* $backup_dir/kubelet
    cp -rf /var/lib/etcd/cert/* $backup_dir/etcd
  else
    public::common::log "master configuration is already backup, skip."
  fi
}

public::upgrade::backupnode() {
  public::common::log "Begin the node backup working."

  local backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)
  if [ ! -f $backup_dir/10-kubeadm.conf ]; then
    mkdir -p $backup_dir $backup_dir/kubelet
    cp -rf /etc/kubernetes/ $backup_dir/
    cp /etc/kubernetes/kubelet.conf $backup_dir
    cp /etc/systemd/system/kubelet.service.d/10-kubeadm.conf $backup_dir
    cp -rf /var/lib/kubelet/pki/* $backup_dir/kubelet
  else
    public::common::log "node configuration is already backup, skip."
  fi
}

public::main::master-rotate() {
  ls -l /usr/local
  pwd
  local backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)
  echo "mode is $MODE"
  if ! grep "rotate-certificates" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf; then
    ./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=master -nodeip=$NODE_IP -auto-rotate=false >$backup_dir/renew.log
  else
    ./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=master -nodeip=$NODE_IP >$backup_dir/renew.log
  fi
  if [[ "$MODE" = "etcd" ]]; then
    public::common::log "Successful update cert on $(hostname)"
    exit 0
  fi

  sleep 1
  #renew the dashboard certs
  if [ -d /etc/kubernetes/pki/dashboard ]; then
    cp -rf /etc/kubernetes/pki/apiserver.crt /etc/kubernetes/pki/dashboard/dashboard.crt
    cp -rf /etc/kubernetes/pki/apiserver.key /etc/kubernetes/pki/dashboard/dashboard.key
    cp -rf /etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/dashboard/dashboard-ca.crt
    cat /etc/kubernetes/pki/client-ca.crt >>/etc/kubernetes/pki/dashboard/dashboard-ca.crt
    if [ -f /etc/kubernetes/pki/user-ca.crt ]; then
      cat /etc/kubernetes/pki/user-ca.crt >>/etc/kubernetes/pki/dashboard/dashboard-ca.crt
    fi
    cp -rf /etc/kubernetes/pki/dashboard/dashboard-ca.crt /etc/kubernetes/pki/apiserver-ca.crt
  else
    cp -rf /etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/apiserver-ca.crt
    cat /etc/kubernetes/pki/client-ca.crt >>/etc/kubernetes/pki/apiserver-ca.crt
    if [ -f /etc/kubernetes/pki/user-ca.crt ]; then
      cat /etc/kubernetes/pki/user-ca.crt >>/etc/kubernetes/pki/apiserver-ca.crt
    fi
  fi

  # /etc/kubernetes/manifests pod can not be pull up automatically. use this to workaround.
  set +e
  docker ps | grep kube-controller-manager | awk '{print $1}' | xargs -I '{}' docker restart {} || true
  crictl pods | grep kube-controller-manager | awk '{print $1}' | xargs -I '{}' crictl stopp {} || true
  set -e

  sleep 1
  #restart kubelet
  service kubelet restart
  sleep 1

  # /etc/kubernetes/manifests pod can not be pull up automatically. use this to workaround.
  set +e
  docker ps | grep kube-apiserver | awk '{print $1}' | xargs -I '{}' docker restart {} || true
  crictl pods | grep kube-apiserver | awk '{print $1}' | xargs -I '{}' crictl stopp {} || true
  sleep 1
  docker ps | grep kube-scheduler | awk '{print $1}' | xargs -I '{}' docker restart {} || true
  crictl pods | grep kube-scheduler | awk '{print $1}' | xargs -I '{}' crictl stopp {} || true
  set -e

  public::common::log "Successful update cert on $(hostname)"
}

public::main::node-rotate() {
  if [ -f /etc/kubernetes/pki/apiserver.crt ]; then
    public::common::log "Skip node rotate on master node"
    exit 0
  fi
  local backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)

  if ! grep "rotate-certificates" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf; then
    ./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=worker -auto-rotate=false -key=$KEY >$backup_dir/renew.log
  else
    ./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=worker -key=$KEY >$backup_dir/renew.log
  fi
  sleep 1
  #restart kubelet
  service kubelet restart
  sleep 1

  public::common::log "Successful update cert on $(hostname)"
}

public::main::master() {
  public::upgrade::backupmaster
  public::main::master-rotate
}

public::main::node() {
  public::upgrade::backupnode
  public::main::node-rotate
}

main() {
  #use renew mode in default
  export MODE=renew

  while
    [[ $# -gt 0 ]]
  do
    key="$1"

    case $key in
    --role)
      export ROLE=$2
      shift
      ;;
    --mode)
      export MODE=$2
      shift
      ;;
    --nodeip)
      export NODE_IP=$2
      shift
      ;;
    --rootkey)
      export KEY=$2
      shift
      ;;
    *)
      public::common::log "unkonw option [$key]"
      exit 1
      ;;
    esac
    shift
  done

  mkdir -p /etc/kubeadm/backup-rotate-$(date +%F)
  #public::upgrade::backup

  ######################################################
  case $ROLE in

  "source")
    public::common::log "source scripts"
    ;;
  "master")
    public::main::master
    ;;
  "node")
    public::main::node
    ;;
  *)
    echo "
        Usage:
            $0 --role master|node --mode renew|rotate
            ./rotate.sh
        "
    ;;
  esac
}

main "$@"

3.在任意Master节点上运行bash rotate-etcd.sh。

当看到命令行输出etcd CA and certs have succesfully rotated!时，表示所有Master节点上的证书和K8s证书已经轮转完成。

脚本执行过程中，会将etcd相关的服务端和客户端证书备份在如下目录中：