「KUBERNETES」- 部署测试集群（kubeadm、单节点、集群、1.22、测试）

# 关闭防火墙
systemctl stop firewalld && systemctl disable firewalld

# 设置 SELINUX 关闭
# setenforce 0
yes | cp /etc/selinux/config /etc/selinux/config.backup
sed -i 's%SELINUX=enforcing%SELINUX=disabled%g' /etc/selinux/config

# 关闭 Swap 分区
# swapoff -a && sysctl -w vm.swappiness=0
yes | cp /etc/fstab /etc/fstab.backup
sed -i -E 's/(.+\s+swap\s+.+)/# \1/g' /etc/fstab
swapoff -a

# 加载内核模块
cat > /etc/modules-load.d/kubernetes.conf <<EOF
br_netfilter
EOF

cat > /etc/sysctl.d/kubernets.conf <<EOF
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-arptables = 1
net.ipv4.ip_forward = 1
EOF
sysctl --system

mkdir /etc/docker
cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

systemctl enable docker
systemctl daemon-reload
systemctl restart docker

# 针对官方源，网络通常不通（除非使用网络加速，YUM 支持）
# https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64

# 使用阿里镜像站
cat > /etc/yum.repos.d/kubernetes-ali.repo <<EOF
[kubernetes-ali]
name=Kubernetes ALi
# for x86
# baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
# fro aarch64
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-aarch64/
enabled=1
gpgcheck=0
EOF

# 最后，更新 YUM 缓存
yum makecache

# --------------------------------------------------------- # 安装 socat / conntrack-tools 依赖

# Q：针对 uosEuler release 20 系统，其仓库未包含 socat conntrack-tools 软件？
# A：我们下载 uos-server-20-1060e-arm64 系统，并从中提取如上两个软件包；

yum install -y socat conntrack-tools

# --------------------------------------------------------- # 安装 Kubernetes 组件

yum install -y kubeadm-1.22.3 kubelet-1.22.3-0 kubectl-1.22.3

yum install kubernetes-cni.aarch64                          # for ARM 64
yum install kubernetes-cni.x86_64                           # for x86_64

# 如果没有 enable 服务，则在 kubeadm init 时会有警告；
# 但是不要 start 服务，这时候还没有初始化完成，缺少启动服务的某些配置文件（比如 /var/lib/kubelet/config.yaml 文件）；
# 这得感谢群里朋友的反馈 :-)
systemctl enable kubelet

# --------------------------------------------------------- # 在初始化前

# 如果必要，则执行 kubeadm config images list 命令，并将相关镜像保存到私有仓库中；

kubeadm config images list

# --------------------------------------------------------- # 开始初始化

# 如果直接使用官方镜像，则初始化可能失败。因为它会去 k8s.gcr.io 拉取镜像，而国内网络无法访问；
# 所以，我们使用 kubeadm init --image-repository 选项指定阿里云的镜像来初始化，我们没有私用私有镜像仓库；

kubeadm init                             \
    --pod-network-cidr=10.244.0.0/16     \
    --image-repository registry.aliyuncs.com/google_containers

# --------------------------------------------------------- # 等待初始化结束

# 在执行 kubeadm init 结束后，留意下面的输出：

...
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 172.19.100.12:6443 --token gfryox.ytmse7e3r0scurcy \
        --discovery-token-ca-cert-hash sha256:d2fcd2bc71c258f5af9cc444094c4add43705e52adad17c78b6f35bf54689ea3

# 上述内容：
# 1. 提示你初始化成功
# 2. 然后，执行下面的三条命令
# 3. 告诉你应该向集群中部署一个 Pod 网络，这一点参考官方中列出的网络选择
# 4. 在工作节点上执行命令可以加入集群中；

# 创建网络
kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml

# 然后，执行如下命令查看状态
kubectl get nodes
kubectl get pods --all-namespaces

# nodes 要处于 Ready 状态，pod 要处于 running 状态
# 当显示 ContainerCreating 时，表示正在创建，稍等即可

kubectl taint nodes --all node-role.kubernetes.io/master-

kubeadm join 172.19.100.12:6443 --token gfryox.ytmse7e3r0scurcy \
        --discovery-token-ca-cert-hash sha256:d2fcd2bc71c258f5af9cc444094c4add43705e52adad17c78b6f35bf54689ea3

# ！！！ error execution phase preflight: unable to fetch the kubeadm-config ConfigMap: failed to get config map: Unauthorized
# ！！！ 遇到这错误是因为--token 选项的 TOKEN 时效了（部署 MASTER 和 NODE 我间隔了很久），口令是有生存时间（TTL）的；
# ！！！ 使用 kubeadm token create 命令重新创建 token 口令（在 Master 上执行）；
# ！！！ https://github.com/kubernetes/kubeadm/issues/1310
# ！！！ 或者执行 kubeadm token create --print-join-command 命令，重新生成 JOIN 命令

# 重新生成 token 值
kubeadm token create

# 如果 token 过期，创建永不过期的 token 值（不建议）
kubeadm token create --ttl 0

kubectl get nodes

kubectl drain "<node-name>" --ignore-daemonsets --delete-local-data

kubectl delete node "<node-name>"

# kubectl get nodes
NAME         STATUS   ROLES    AGE     VERSION
k8s-master   Ready    master   25h     v1.14.0
k8s-node01   Ready    <none>   7m19s   v1.14.0

# journalctl -f -u kubelet.service
...
5 月 07 21:10:34 foo-k8s-x86-cp120 kubelet[7393]: I0507 21:10:34.453087    7393 cni.go:204] "Error validating CNI config list" configList="{\n  \"name\": \"cbr0\",\n  \"cniVersion\": \"0.3.1\",\n  \"plugins\": [\n    {\n      \"type\": \"flannel\",\n      \"delegate\": {\n        \"hairpinMode\": true,\n        \"isDefaultGateway\": true\n      }\n    },\n    {\n      \"type\": \"portmap\",\n      \"capabilities\": {\n        \"portMappings\": true\n      }\n    }\n  ]\n}\n" err="[failed to find plugin \"portmap\" in path [/opt/cni/bin]]"
5 月 07 21:10:34 foo-k8s-x86-cp120 kubelet[7393]: I0507 21:10:34.453654    7393 cni.go:239] "Unable to update cni config" err="no valid networks found in /etc/cni/net.d"
5 月 07 21:10:34 foo-k8s-x86-cp120 kubelet[7393]: E0507 21:10:34.551170    7393 kubelet.go:2337] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
...

kubectl describe pod -n kube-system coredns-f9fd979d6-bgggv
...
OCI message: "process_linux.go:264: applying cgroup configuration for process caused \"No such device or address\""
...