安装1.22版k8s,依赖cri-dockerd、flannel

安装k8s

一、软件依赖

docker:docker-20.10.21.tgz
cri-dockerd:cri-dockerd-0.3.1-3.el7.x86_64.rpm
flannel: flannel-v0.22.0-linux-amd64.tar.gz

二、安装

修改主机名解析
修改内核参数
安装docker
安装cri-dockerd
安装kubeadm kubelet kubectl
修改kubelet启动参数
初始化集群
安装网络插件
添加节点

2.1添加host,同步至所有节点

10.0.0.11    k8s-master01
10.0.0.13    k8s-node01
10.0.0.14    k8s-node02

2.2 修改内核参数

function set_kernel_parameters(){
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipvsadm -y
}

2.3 安装docker

function install(){
yum -y install bridge-utils
tar xf /usr/local/src/docker-${DOCKER_VERSION}.tgz  -C  /usr/src/
cp /usr/src/docker/* /usr/bin
mkdir -p /etc/docker
cat > /usr/lib/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target  firewalld.service 
Wants=network-online.target


[Service]
Type=notify

ExecStart=/usr/bin/dockerd -H unix://var/run/docker.sock
ExecReload=/bin/kill -s HUP \$MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

StartLimitInterval=60s

LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity


Delegate=yes


KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
EOF

tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://ze8nwkh8.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
  "max-size": "200m"
  },  
  "storage-driver": "overlay2" 
}
EOF
systemctl daemon-reload
systemctl enable --now docker.service
systemctl restart docker
groupadd docker
useradd -M -g docker -s /bin/nologin docker
}

2.4 安装cri

function installcri(){
    groupadd docker
useradd -M -g docker -s /bin/nologin docker
yum install -y yum-utils
yum-config-manager     --add-repo     https://download.docker.com/linux/centos/docker-ce.repo
cd /usr/local/src
yum -y install /usr/local/src/cri-dockerd-0.3.1-3.el7.x86_64.rpm
sed -Ei.bak "s@^(ExecStart).*@\1=/usr/bin/cri-dockerd  --container-runtime-endpoint fd://  --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.7@g" /usr/lib/systemd/system/cri-docker.service
systemctl daemon-reload
systemctl enable --now cri-docker

}

2.5 安装kubeadm kubelet kubectl

function installk8s(){
    local K8S_VERSION="1.22.0"
cat << EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.cloud.tencent.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
EOF

yum install -y --nogpgcheck  kubelet-${K8S_VERSION} kubeadm-${K8S_VERSION} kubectl-${K8S_VERSION}

}

2.6 修改kubelet参数

cat << EOF > /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS=" --container-runtime-endpoint=/run/cri-dockerd.sock"
EOF
systemctl enable kubelet

2.7 初始化集群

kubeadm config print init-defaults > kubeadm.yaml #初始化配置文件
---bootstrapTokens
//修改apiserverip
advertiseAddress
//修改运行时socket
criSocket
//修改控制面名称
name
---apiServer
//增加控制平面访问域名
controlPlaneEndpoint
//修改了k8s版本
kubernetesVersion
//修改pod网段
podSubnet
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

//初始化
kubeadm init --config=kubeadm.yaml --v=5 --upload-certs

2.7.1 验证节点是否初始化成功

//copy apiserver 鉴权文件
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
kubectl get pods -A
注意：coredns 在初始化完成后不能马上运行，因为有node污点机制，不能在master调度，需要在增加一个机器后才能运行

2.8 安装flannel

mkdir -p /opt/bin && tar xf  /usr/local/src/flannel-v0.22.0-linux-amd64.tar.gz -C  /opt/bin
kubectl apply -f /usr/local/src/kube-flannel.yml

2.9 添加节点

//用token加入集群
kubeadm join k8s.master01:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:655542544a694a4a7b1101da7aa245f893d46caef90e3903236b244b3ae1b3a5

2.9.1设置永不过期token

kubeadm token  create --ttl=0

2.9.2 使用配置文件添加节点

kubeadm config print join-defaults> join.yaml

kubeadm.k8s.io/v1beta3

caCertPath: /etc/kubernetes/pki/ca.crt
discovery:
  bootstrapToken:
    apiServerEndpoint: k8s.master01:6443 //控制平面地址
    token: mmdmu4.jn29xptk5f3xk81j //鉴权token 
    unsafeSkipCAVerification: true
    caCertHashes:
    - sha256:655542544a694a4a7b1101da7aa245f893d46caef90e3903236b244b3ae1b3a5  //验证ca
  timeout: 5m0s
  tlsBootstrapToken: mmdmu4.jn29xptk5f3xk81j
kind: JoinConfiguration
nodeRegistration:
  criSocket: /run/cri-dockerd.sock //默认的运行时
  imagePullPolicy: IfNotPresent
  name: k8s-node02 //当前节点名称
  taints: //自定义自污类型
  - key: nodetype
    value: app 
    effect: NoSchedule

三、检测

kubectl get nodes
kubectl get pods -A
//如果coredns 是Blocked 状态，删掉重建
kubectl delete pods pod名称 -n kube-system

四、重置

kubeadm reset --cri-socket unix:///run/cri-dockerd.sock && rm -rf /etc/kubernetes/ /var/lib/kubelet /var/lib/dockershim /var/run/kubernetes /var/lib/cni /etc/cni/net.d

五、生成token和密钥

kubeadm token  create --ttl=0
kubeadm token list #查看所有token
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' #获取密钥

六、节点 NotReady 故障排除

#检查节点kubelet 信息
journalctl -u kubelet --since today |less
#本次原因是因为部署ingress 绑定了固定IP，固定IP应该绑定当前节点的外网IP，必须不能和k8s集群的网段相同，否则固定IP节点的IP加入到ipvs中，不能和控制面板通信。

七、ingress 故障排除

# 测试环境不能 LoadBalancer，所有绑定固定IP，外部流量策略采用 cluster
Service 注释掉  #externalTrafficPolicy: Cluster
增加 externalIPs
#测试
while true;do curl http://test.mfcqc.com/api/ping && echo -e '\n' && sleep 3s; done;

八、生成证书

#文档位置：
https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/certificates/

#生成key:
(umask 077;openssl genrsa -out ca.key 2048)
#直接生成crt
openssl req -x509 -new -nodes -key ca.key -subj "/C=CN/ST=Beijing/L=Beijing/O=test/CN=test.mfcqc.com" -days 10000 -out ca.crt

#脚本文件
kubectl create secret tls  ingress-tls --cert=ca.crt  --key=ca.key --dry-run=client -o yaml>ingress-tls.yaml

九、客户端用户认证
9.1 令牌认证

1. 生成token
echo "$(openssl rand -hex 3).$(openssl rand -hex 8)"
2. 创建账户 token,user,id,groups
ff3c9a.295e304155eced9e,test,100000,kubeusers
3. 修改 kube-apiserver.yaml // 注意：不要直接修改，copy到其他目录修改完毕后，再copy到/etc/kubernetes/manifests
4. 验证
curl -k -H "Authorization: Bearer ff3c9a.295e304155eced9e" https://10.0.0.11:6443/api/v1/namespaces/default/pods

9.2 数字证书认证

1. 生成key
(umask 077;openssl genrsa -out test.key 2048)
2. 生成csr
openssl req -new -key ./test.key -out ./test.csr -subj "/C=CN/ST=Beijing/L=Beijing/O=kubeusers/CN=test"
3. 签署证书
openssl x509 -req -days 365 -CAkey ../ca.key -CA ../ca.crt -CAcreateserial -in ./test.csr -out ./test.crt
4. 同步到客户端
  /etc/kubernetes/pki
  test.crt //用户证书
  test.key //用户key
  ca.crt //ca证书
5. 客户端验证
kubectl -s https://k8s-master01:6443 --client-certificate=/etc/kubernetes/pki/test.crt --client-key=/etc/kubernetes/pki/test.key --insecure-skip-tls-verify=false   get pods -A
6. http验证
curl --cert ./test.crt --key ./test.key --cacert ./ca.crt https://k8s-master01:6443/api/v1/namespaces/default/pods

十、kubectl config 定义用户

kubectl config view #查看当前配置

10.1 定义静态令牌用户

1.定义集群
kubectl config set-cluster k8stest --embed-certs=true --certificate-authority=/etc/kubernetes/pki/ca.crt --server=https://k8s-master01:6443 --kubeconfig=/etc/kubernetes/conf/test.conf
2.定义用户
kubectl config set-credentials test --token=ff3c9a.295e304155eced9e --kubeconfig=/etc/kubernetes/conf/test.conf
3.定义用户和集群上下文
kubectl config set-context   test@k8stest  --cluster=k8stest --user=test   --kubeconfig=/etc/kubernetes/conf/test.conf
4.使用当前用户
kubectl config use-context test@k8stest --kubeconfig=/etc/kubernetes/conf/test.conf
5.用户当前用户配置查看信息
kubectl get pods -A --kubeconfig=/etc/kubernetes/conf/test.conf

10.2定义数字证书用户

1.定义用户
kubectl config  set-credentials test2 --client-certificate=/etc/kubernetes/pki/client/test.crt  --client-key=/etc/kubernetes/pki/client/test.key  --kubeconfig=/etc/kubernetes/conf/test.conf
2.定义用户上下文
kubectl config set-context   test2@k8stest  --cluster=k8stest --user=test2   --kubeconfig=/etc/kubernetes/conf/test.conf
3.检测用户
kubectl get pods -A --context='test2@k8stest' --kubeconfig=/etc/kubernetes/conf/test.conf

十一、鉴权

11.1 创建role
kubectl create role  myrole  --verb=get,list,watch,create,delete    --resource=service,ingress --dry-run=client -o yaml>myrole.yaml
11.2  创建rolebinding
kubectl create rolebinding  myrolebinding   --role=myrole --user=test  --dry-run=client -o yaml > myrolebinding.yaml
11.3 测试获取资源
kubectl get svc -n mfcqc --kubeconfig=/etc/kubernetes/conf/test.conf
11.4 内置的role
cluster-admin //集群管理员
admin  //命名空间内的管理员
edit  //命名空间可写权限，不允许查看修改role，rolebinding
view  //命名空间内以只读方式查看资源对象，不包括role,rolebinding 和 secret