使用K8S Statefulset控制器部署Etcd数据库集群（四十九）
[root@k8s-master1 etcd]# vi
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: etcd
name: etcd
namespace: etcd
spec:
replicas: 3
selector:
matchLabels:
app: etcd
serviceName: etcd
template:
metadata:
labels:
app: etcd
spec:
containers:
- name: etcd
image: cnych/etcd:v3.4.13
imagePullPolicy: IfNotPresent
ports:
- containerPort: 2380
name: peer
protocol: TCP
- containerPort: 2379
name: client
protocol: TCP
env:
- name: INITIAL_CLUSTER_SIZE
value: "3"
- name: MY_NAMESPACE #pod所在命名空间的变量
valueFrom:
fieldRef:
fieldPath:
- name: POD_IP #pod ip变量
valueFrom:
fieldRef:
fieldPath:
- name: SET_NAME
value: "etcd"
command: #整个command脚本的意思就是，首先判断有几个节点，然后循环展示出每个节点的地址+端口，然后删除所有的集群信息，然后在一个一个的重新加入集群，最后启动集群
- /bin/sh
- -ec
- |
HOSTNAME=$(hostname)
ETCDCTL_API=3
eps() { #获取各个节点的2379地址
EPS=""
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}.${MY_NAMESPACE}.:2379"
done
echo ${EPS}
}
member_hash() {
etcdctl member list | grep -w "$HOSTNAME" | awk '{ print $1}' | awk -F "," '{ print $1}'
}
initial_peers() { #获取各个节点的2380通信地址
PEERS=""
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}.${MY_NAMESPACE}.:2380"
done
echo ${PEERS}
}
# etcd-SET_ID
SET_ID=${HOSTNAME##*-} #编号
# 如果当前编号大于等于集群数量，则说明新增节点（多加一个pod）
if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then
# export ETCDCTL_ENDPOINTS=$(eps)
# member already added?
#如果当前节点存在ectd列表中，则将其先删除
MEMBER_HASH=$(member_hash)
if [ -n "${MEMBER_HASH}" ]; then
# the member hash exists but for some reason etcd failed
# as the datadir has not be created, we can remove the member
# and retrieve new hash
echo "Remove member ${MEMBER_HASH}"
etcdctl --endpoints=$(eps) member remove ${MEMBER_HASH}
fi
#删掉旧信息后，开始新增节点
echo "Adding new member"
#新增一个节点
echo "etcdctl --endpoints=$(eps) member add ${HOSTNAME} --peer-urls=http://${HOSTNAME}.${SET_NAME}.${MY_NAMESPACE}.:2380"
etcdctl member --endpoints=$(eps) add ${HOSTNAME} --peer-urls=http://${HOSTNAME}.${SET_NAME}.${MY_NAMESPACE}.:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs
#如果新增节点的命令执行不成功则删除new_member_envs这个文件并退出脚本
if [ $? -ne 0 ]; then
echo "member add ${HOSTNAME} error."
rm -f /var/run/etcd/new_member_envs
exit 1
fi
#将/var/run/etcd/new_member_envs里面的变量全部设置成环境变量
echo "==> Loading env vars of existing cluster..."
sed -ie "s/^/export /" /var/run/etcd/new_member_envs
cat /var/run/etcd/new_member_envs
. /var/run/etcd/new_member_envs
#启动当前节点，每个pod都会来一遍，因此这里是启动当前节点
echo "etcd --name ${HOSTNAME} --initial-advertise-peer-urls ${ETCD_INITIAL_ADVERTISE_PEER_URLS} --listen-peer-urls http://${POD_IP}:2380 --listen-client-urls http://${POD_IP}:2379,http://127.0.0.1:2379 --advertise-client-urls http://${HOSTNAME}.${SET_NAME}.${MY_NAMESPACE}.:2379 --data-dir /var/run/etcd/ --initial-cluster ${ETCD_INITIAL_CLUSTER} --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE}"
exec etcd --listen-peer-urls http://${POD_IP}:2380 \
--listen-client-urls http://${POD_IP}:2379,http://127.0.0.1:2379 \
--advertise-client-urls http://${HOSTNAME}.${SET_NAME}.${MY_NAMESPACE}.:2379 \
--data-dir /var/run/etcd/
fi
#测试所有节点是否都已经成功启动
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
while true; do
echo "Waiting for ${SET_NAME}-${i}.${SET_NAME}.${MY_NAMESPACE}. to come up"
ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME}.${MY_NAMESPACE}. > /dev/null && break
sleep 1s
done
done
#当所有节点准备就绪后，启动集群
echo "join member ${HOSTNAME}"
# join member
exec etcd --name ${HOSTNAME} \
--initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}.${MY_NAMESPACE}.:2380 \
--listen-peer-urls http://${POD_IP}:2380 \
--listen-client-urls http://${POD_IP}:2379,http://127.0.0.1:2379 \
--advertise-client-urls http://${HOSTNAME}.${SET_NAME}.${MY_NAMESPACE}.:2379 \
--initial-cluster-token etcd-cluster-1 \
--data-dir /var/run/etcd/ \
--initial-cluster $(initial_peers) \
--initial-cluster-state new
lifecycle:
preStop:
exec:
command: #钩子函数，当pod启动后，将环境变量文件删除
- /bin/sh
- -ec
- |
HOSTNAME=$(hostname)
member_hash() {
etcdctl member list | grep -w "$HOSTNAME" | awk '{ print $1}' | awk -F "," '{ print $1}'
}
eps() {
EPS=""
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}.${MY_NAMESPACE}.:2379"
done
echo ${EPS}
}
export ETCDCTL_ENDPOINTS=$(eps)
SET_ID=${HOSTNAME##*-}
# Removing member from cluster
if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then
echo "Removing ${HOSTNAME} from etcd cluster"
etcdctl member remove $(member_hash)
if [ $? -eq 0 ]; then
# Remove everything otherwise the cluster will no longer scale-up
rm -rf /var/run/etcd/*
fi
fi
volumeMounts:
- mountPath: /var/run/etcd
name: etcd-data
volumeClaimTemplates:
- metadata:
name: etcd-data
spec:
storageClassName: etcd-storageclass
accessModes:
- "ReadWriteOnce"
resources:
requests:
# upstream recommended max is 700M
storage: 1Gi