k8s1.30.0高可用集群部署

负载均衡

nginx负载均衡

两台nginx负载均衡
vim /etc/nginx/nginx.conf

1. stream {
2.     upstream kube-apiserver {
3.         server 192.168.0.11:6443     max_fails=3 fail_timeout=30s;
4.         #server 192.168.0.12:6443     max_fails=3 fail_timeout=30s;
5.         #server 192.168.0.13:6443     max_fails=3 fail_timeout=30s;
6.     }
7.     server {
8.         listen 6443;
9.         proxy_connect_timeout 2s;
10.         proxy_timeout 900s;
11.         proxy_pass kube-apiserver;
12.     }
13. }

复制代码

keepalived

nginx检测脚本
vim /data/shell/check_nginx_status.sh

1. #!/bin/bash
2. nginx_status=$(ps -ef | grep nginx | grep -v grep | grep -v check | wc -l)
4. if [ $nginx_status -eq 0 ];then
5.   systemctl stop keepalived.service
6. fi

复制代码

master节点keepalived设置（不抢占机制）
vim /etc/keepalived/keepalived.conf

1. global_defs {
2.     router_id real-server1
3. }
5. vrrp_script chk_nginx {
6.     script "/data/shell/check_nginx_status.sh"
7.     interval 2
8. }
10. vrrp_instance VI_1 {
11.      state BACKUP
12.      interface ens32
13.      virtual_router_id 151
14.      priority 100
15.      nopreempt
16.      advert_int 5
17.      authentication {
18.          auth_type  PASS
19.          auth_pass  1111
20.      }
21.      virtual_ipaddress {
22.        192.168.0.10/24
23.      }
24.    
25.       track_script {                                                                                 
26.        chk_nginx
27.     }
28. }

复制代码

backup节点keepalived设置
vim /etc/keepalived/keepalived.conf

1. global_defs {
2.     router_id real-server2
3. }
5. vrrp_script chk_nginx {
6.     script "/data/shell/check_nginx_status.sh"
7.     interval 2
8. }
10. vrrp_instance VI_1 {
11.      state BACKUP
12.      interface ens32
13.      virtual_router_id 151
14.      priority 50
15.      nopreempt
16.      advert_int 5
17.      authentication {
18.          auth_type  PASS
19.          auth_pass  1111
20.      }
21.      virtual_ipaddress {
22.        192.168.0.10/24
23.      }
24.    
25.       track_script {                                                                                 
26.        chk_nginx
27.     }
28. }

复制代码

k8s节点体系设置

vim /etc/modules-load.d/containerd.conf

1. overlay
2. br_netfilter

复制代码

1. modprobe overlay
2. modprobe br_netfilter

复制代码

vim /etc/sysctl.d/k8s.conf

1. net.bridge.bridge-nf-call-iptables=1
2. net.bridge.bridge-nf-call-ip6tables=1
3. net.ipv4.ip_forward=1
4. vm.swappiness=0
5. vm.overcommit_memory=1
6. vm.panic_on_oom=0
7. fs.inotify.max_user_instances=8192
8. fs.inotify.max_user_watches=1048576
9. fs.file-max=52706963
10. fs.nr_open=52706963
11. net.ipv6.conf.all.disable_ipv6=1
12. net.netfilter.nf_conntrack_max=2310720

复制代码

1. sysctl --system

复制代码

ipvsadm

1. yum install ipset ipvsadm

复制代码

1. modprobe br_netfilter
2. modprobe overlay
3. modprobe ip_conntrack
4. modprobe  ip_vs
5. modprobe  ip_vs_rr
6. modprobe  ip_vs_wrr
7. modprobe  ip_vs_sh
8. modprobe  nf_conntrack
9. lsmod | grep conntrack
10. lsmod | grep br_netfilt
11. lsmod | grep overlay
12. lsmod |egrep  "ip_vs|nf_conntrack"

复制代码

1. cat > /etc/modules-load.d/kubernetes.conf << EOF
2. # /etc/modules-load.d/kubernetes.conf
3. br_netfilter
4. ip_vs
5. ip_vs_rr
6. ip_vs_wrr
7. ip_vs_sh
8. nf_conntrack_ipv4
9. ip_tables
10. EOF

复制代码

1. chmod a+x /etc/modules-load.d/kubernetes.conf

复制代码

containerd

1. wget https://github.com/containerd/containerd/releases/download/v1.7.23/cri-containerd-cni-1.7.23-linux-amd64.tar.gz
3. tar xvf cri-containerd-cni-1.7.23-linux-amd64.tar.gz -C /
5. wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
6. cp runc.amd64 /usr/local/sbin/runc

复制代码

1. mkdir -p /etc/containerd
2. containerd config default > /etc/containerd/config.toml

复制代码

vim /etc/containerd/config.toml

1. ...
2. SystemdCgroup = true
3. ...
4. sandbox_image = "registry.cn-beijing.aliyuncs.com/wuxingge123/pause:3.9"
5. ...
6.       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
7.         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
8.           endpoint = ["https://p4oudlho.mirror.aliyuncs.com"]

复制代码

设置私有堆栈

1. [plugins."io.containerd.grpc.v1.cri".registry]
2.   [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
3.     [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
4.       endpoint = ["https://bqr1dr1n.mirror.aliyuncs.com"]
5.   [plugins."io.containerd.grpc.v1.cri".registry.configs]
6.     [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".tls]
7.       insecure_skip_verify = true
8.     [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".auth]
9.       username = "admin"
10.       password = "Harbor12345"

复制代码

启动containerd

1. systemctl start containerd.service
2. systemctl enable containerd.service

复制代码

k8s

安装

https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/x86_64/
vim /etc/yum.repos.d/k8s.repo

1. [kubernetes]
2. name=kubernetes
3. baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
4. enabled=1
5. gpgcheck=0

复制代码

在线安装

1. yum install kubelet-1.30.0 kubectl-1.30.0 kubeadm-1.30.0

复制代码

离线安装

1. yum localinstall kubernetes-cni-1.4.0-150500.1.1.x86_64.rpm
2. yum localinstall cri-tools-1.30.0-150500.1.1.x86_64.rpm
3. yum localinstall kubeadm-1.30.0-150500.1.1.x86_64.rpm kubectl-1.30.0-150500.1.1.x86_64.rpm kubelet-1.30.0-150500.1.1.x86_64.rpm

复制代码

设置kubelet

vim /etc/sysconfig/kubelet

1. KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

复制代码

1. systemctl enable kubelet

复制代码

下载镜像

1. kubeadm config images list --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123
3. kubeadm config images pull --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123

复制代码

calico准备

1. wget https://docs.projectcalico.org/v3.25/manifests/calico.yaml --no-check-certificate

复制代码

vim calico.yaml

1.             - name: CALICO_IPV4POOL_CIDR
2.               value: "10.224.0.0/16"

复制代码

下载calico镜像

1. ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/cni:v3.25.0
3. ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/node:v3.25.0
5. ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/kube-controllers:v3.25.0

复制代码

k8s init

生成初始化设置文件

1. kubeadm config print init-defaults > kubeadm-config.yaml

复制代码

vim kubeadm-config.yaml

1. apiVersion: kubeadm.k8s.io/v1beta3
2. bootstrapTokens:
3. - groups:
4.   - system:bootstrappers:kubeadm:default-node-token
5.   token: abcdef.0123456789abcdef
6.   ttl: 24h0m0s
7.   usages:
8.   - signing
9.   - authentication
10. kind: InitConfiguration
11. localAPIEndpoint:
12.   advertiseAddress: 192.168.0.11
13.   bindPort: 6443
14. nodeRegistration:
15.   criSocket: unix:///var/run/containerd/containerd.sock
16.   imagePullPolicy: IfNotPresent
17.   taints: null
18. ---
19. apiServer:
20.   timeoutForControlPlane: 4m0s
21. apiVersion: kubeadm.k8s.io/v1beta3
22. certificatesDir: /etc/kubernetes/pki
23. clusterName: kubernetes
24. controlPlaneEndpoint: 192.168.0.10:6443
25. controllerManager: {}
26. dns: {}
27. etcd:
28.   local:
29.     dataDir: /var/lib/etcd
30. imageRepository: swr.cn-jl-1.manageone.cloud.cnpc/gsms-project
31. kind: ClusterConfiguration
32. kubernetesVersion: 1.30.0
33. networking:
34.   dnsDomain: cluster.local
35.   podSubnet: 10.224.0.0/16
36.   serviceSubnet: 10.96.0.0/12
37. scheduler: {}

复制代码

初始化集群

1. kubeadm init --config kubeadm-config.yaml --upload-certs

复制代码

初始化成功显示以下内容

1. Your Kubernetes control-plane has initialized successfully!
3. To start using your cluster, you need to run the following as a regular user:
5.   mkdir -p $HOME/.kube
6.   sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
7.   sudo chown $(id -u):$(id -g) $HOME/.kube/config
9. Alternatively, if you are the root user, you can run:
11.   export KUBECONFIG=/etc/kubernetes/admin.conf
13. You should now deploy a pod network to the cluster.
14. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
15.   https://kubernetes.io/docs/concepts/cluster-administration/addons/
17. You can now join any number of the control-plane node running the following command on each as root:
19.   kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
20.         --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \
21.         --control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826
23. Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
24. As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
25. "kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
27. Then you can join any number of worker nodes by running the following on each as root:
29. kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
30.         --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464

复制代码

设置kubectl客户端

1. mkdir -p $HOME/.kube
2. sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
3. sudo chown $(id -u):$(id -g) $HOME/.kube/config

复制代码

设置kubectl命令补全

1. echo "source <(kubectl completion bash)" >> /etc/profile

复制代码

部署calico

1. kubectl apply -f calico.yaml

复制代码

添加master节点

1.   kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
2.         --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \
3.         --control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826

复制代码

添加node节点

1. kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
2.         --discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464

复制代码

ingress-nginx v1.11.2

1. kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.2/deploy/static/provider/cloud/deploy.yaml

复制代码

metrics-server

1. wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

复制代码

免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。