1. <dd id="erndk"></dd>
                1. Kubernetes CKA認證運維工程師筆記-Kubernetes故障排查

                  互聯網 2021/12/25 23:39:34

                  Kubernetes CKA認證運維工程師筆記-Kubernetes故障排查 1. 應用程序故障排查2. 管理節點與工作節點異常排查3. Service訪問異常排查1. 應用程序故障排查 # 置頂type類型,是Pod還是deployment等,再接名稱 kubectl describe TYPE/NAME # 查看日志,一個Pod有兩個容器的話,…

                  Kubernetes CKA認證運維工程師筆記-Kubernetes故障排查

                  • 1. 應用程序故障排查
                  • 2. 管理節點與工作節點異常排查
                  • 3. Service訪問異常排查

                  1. 應用程序故障排查

                  # 置頂type類型,是Pod還是deployment等,再接名稱
                  kubectl describe TYPE/NAME
                  # 查看日志,一個Pod有兩個容器的話,要用-c指定容器名
                  kubectl logs TYPE/NAME [-c CONTAINER]
                  # 進入容器中進行查看
                  kubectl exec POD [-c CONTAINER] --COMMAND [args...]
                  
                  [root@k8s-master ~]# kubectl get pods
                  NAME                                     READY   STATUS             RESTARTS   AGE
                  client1                                  1/1     Running            5          2d23h
                  client2                                  0/1     ImagePullBackOff   4          2d23h
                  configmap-demo-pod                       0/1     ImagePullBackOff   3          4d10h
                  my-pod2                                  1/1     Running            11         4d17h
                  nfs-client-provisioner-58d675cd5-dx7n4   0/1     ImagePullBackOff   6          4d11h
                  pod-taint                                1/1     Running            9          10d
                  secret-demo-pod                          1/1     Running            4          4d9h
                  sh                                       1/1     Running            6          4d10h
                  test-76846b5956-gftn9                    1/1     Running            2          4d10h
                  test-76846b5956-r7s9k                    1/1     Running            2          4d10h
                  test-76846b5956-trpbn                    1/1     Running            2          4d10h
                  test2-78c4694588-87b9r                   1/1     Running            5          4d12h
                  web-0                                    1/1     Running            4          4d11h
                  web-1                                    0/1     ImagePullBackOff   3          4d11h
                  web-2                                    0/1     ImagePullBackOff   3          4d11h
                  web-96d5df5c8-vc9kf                      1/1     Running            3          3d
                  [root@k8s-master ~]# kubectl get pods
                  NAME                                     READY   STATUS             RESTARTS   AGE
                  client1                                  1/1     Running            5          2d23h
                  client2                                  0/1     ImagePullBackOff   4          2d23h
                  configmap-demo-pod                       0/1     ImagePullBackOff   3          4d10h
                  my-pod2                                  1/1     Running            11         4d17h
                  nfs-client-provisioner-58d675cd5-dx7n4   0/1     ImagePullBackOff   6          4d11h
                  pod-taint                                1/1     Running            9          10d
                  secret-demo-pod                          1/1     Running            4          4d9h
                  sh                                       1/1     Running            6          4d10h
                  test-76846b5956-gftn9                    1/1     Running            2          4d10h
                  test-76846b5956-r7s9k                    1/1     Running            2          4d10h
                  test-76846b5956-trpbn                    1/1     Running            2          4d10h
                  test2-78c4694588-87b9r                   1/1     Running            5          4d12h
                  web-0                                    1/1     Running            4          4d11h
                  web-1                                    0/1     ImagePullBackOff   3          4d11h
                  web-2                                    0/1     ImagePullBackOff   3          4d11h
                  web-96d5df5c8-vc9kf                      1/1     Running            3          3d
                  [root@k8s-master ~]# kubectl describe pod web-96d5df5c8-vc9kf
                  Name:         web-96d5df5c8-vc9kf
                  Namespace:    default
                  Priority:     0
                  Node:         k8s-node2/10.0.0.63
                  Start Time:   Wed, 22 Dec 2021 22:11:51 +0800
                  Labels:       app=web
                                pod-template-hash=96d5df5c8
                  Annotations:  cni.projectcalico.org/podIP: 10.244.169.158/32
                                cni.projectcalico.org/podIPs: 10.244.169.158/32
                  Status:       Running
                  IP:           10.244.169.158
                  IPs:
                    IP:           10.244.169.158
                  Controlled By:  ReplicaSet/web-96d5df5c8
                  Containers:
                    nginx:
                      Container ID:   docker://f3243ba267e377896e3c5de8a2909d9dd12ed3b2a3fbd80b0094711e5a3f8c81
                      Image:          nginx
                      Image ID:       docker-pullable://nginx@sha256:366e9f1ddebdb844044c2fafd13b75271a9f620819370f8971220c2b330a9254
                      Port:           <none>
                      Host Port:      <none>
                      State:          Running
                        Started:      Sat, 25 Dec 2021 22:09:45 +0800
                      Last State:     Terminated
                        Reason:       Error
                        Exit Code:    255
                        Started:      Fri, 24 Dec 2021 15:08:58 +0800
                        Finished:     Sat, 25 Dec 2021 22:02:39 +0800
                      Ready:          True
                      Restart Count:  3
                      Environment:    <none>
                      Mounts:
                        /var/run/secrets/kubernetes.io/serviceaccount from default-token-8grtj (ro)
                  Conditions:
                    Type              Status
                    Initialized       True 
                    Ready             True 
                    ContainersReady   True 
                    PodScheduled      True 
                  Volumes:
                    default-token-8grtj:
                      Type:        Secret (a volume populated by a Secret)
                      SecretName:  default-token-8grtj
                      Optional:    false
                  QoS Class:       BestEffort
                  Node-Selectors:  <none>
                  Tolerations:     node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                                   node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
                  Events:
                    Type    Reason          Age    From     Message
                    ----    ------          ----   ----     -------
                    Normal  SandboxChanged  10m    kubelet  Pod sandbox changed, it will be killed and re-created.
                    Normal  Pulling         9m38s  kubelet  Pulling image "nginx"
                    Normal  SandboxChanged  4m5s   kubelet  Pod sandbox changed, it will be killed and re-created.
                    Normal  Pulling         3m53s  kubelet  Pulling image "nginx"
                    Normal  Pulled          3m37s  kubelet  Successfully pulled image "nginx" in 16.296403014s
                    Normal  Created         3m36s  kubelet  Created container nginx
                    Normal  Started         3m36s  kubelet  Started container nginx
                  [root@k8s-master ~]# kubectl logs web-96d5df5c8-vc9kf
                  /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
                  /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
                  /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
                  10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
                  10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
                  /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
                  /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
                  /docker-entrypoint.sh: Configuration complete; ready for start up
                  2021/12/25 14:09:46 [notice] 1#1: using the "epoll" event method
                  2021/12/25 14:09:46 [notice] 1#1: nginx/1.21.4
                  2021/12/25 14:09:46 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6) 
                  2021/12/25 14:09:46 [notice] 1#1: OS: Linux 3.10.0-1160.45.1.el7.x86_64
                  2021/12/25 14:09:46 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
                  2021/12/25 14:09:46 [notice] 1#1: start worker processes
                  2021/12/25 14:09:46 [notice] 1#1: start worker process 31
                  2021/12/25 14:09:46 [notice] 1#1: start worker process 32
                  [root@k8s-master ~]# kubectl exec -it web-96d5df5c8-vc9kf bash
                  kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
                  root@web-96d5df5c8-vc9kf:/# exit
                  exit
                  [root@k8s-master ~]# kubectl exec -it web-96d5df5c8-vc9kf -- bash
                  root@web-96d5df5c8-vc9kf:/# 
                  

                  2. 管理節點與工作節點異常排查

                  管理節點組件:

                  • kube-apiserver
                  • kube-controller-manager
                  • kube-scheduler

                  工作節點組件:

                  • kubelet
                  • kube-proxy

                  Kubernetes集群架構圖
                  在這里插入圖片描述
                  需要先區分部署方式:
                  1、kubeadm
                  除kubelet外,其他組件均采用靜態Pod啟動

                  [root@k8s-master ~]# ls /etc/kubernetes/manifests/
                  etcd.yaml            kube-controller-manager.yaml
                  kube-apiserver.yaml  kube-scheduler.yaml
                  [root@k8s-master ~]# cat /var/lib/kubelet/config.yaml 
                  apiVersion: kubelet.config.k8s.io/v1beta1
                  authentication:
                    anonymous:
                      enabled: false
                    webhook:
                      cacheTTL: 0s
                      enabled: true
                    x509:
                      clientCAFile: /etc/kubernetes/pki/ca.crt
                  authorization:
                    mode: Webhook
                    webhook:
                      cacheAuthorizedTTL: 0s
                      cacheUnauthorizedTTL: 0s
                  cgroupDriver: cgroupfs
                  clusterDNS:
                  - 10.96.0.10
                  clusterDomain: cluster.local
                  cpuManagerReconcilePeriod: 0s
                  evictionPressureTransitionPeriod: 0s
                  fileCheckFrequency: 0s
                  healthzBindAddress: 127.0.0.1
                  healthzPort: 10248
                  httpCheckFrequency: 0s
                  imageMinimumGCAge: 0s
                  kind: KubeletConfiguration
                  logging: {}
                  nodeStatusReportFrequency: 0s
                  nodeStatusUpdateFrequency: 0s
                  rotateCertificates: true
                  runtimeRequestTimeout: 0s
                  staticPodPath: /etc/kubernetes/manifests
                  streamingConnectionIdleTimeout: 0s
                  syncFrequency: 0s
                  volumeStatsAggPeriod: 0s
                  
                  [root@k8s-master ~]# kubectl get pods -n kube-system
                  NAME                                      READY   STATUS    RESTARTS   AGE
                  calico-kube-controllers-97769f7c7-c4cg5   1/1     Running   3          30h
                  calico-node-4pwdc                         1/1     Running   16         33d
                  calico-node-9r6zd                         1/1     Running   16         33d
                  calico-node-vqzdj                         1/1     Running   17         33d
                  client1                                   1/1     Running   5          2d23h
                  coredns-6d56c8448f-gcgrh                  1/1     Running   16         33d
                  coredns-6d56c8448f-mdl7c                  1/1     Running   2          30h
                  etcd-k8s-master                           1/1     Running   3          30h
                  filebeat-5pwh7                            1/1     Running   11         10d
                  filebeat-pt848                            1/1     Running   11         10d
                  kube-apiserver-k8s-master                 1/1     Running   3          30h
                  kube-controller-manager-k8s-master        1/1     Running   3          30h
                  kube-proxy-87lbj                          1/1     Running   3          30h
                  kube-proxy-mcdnv                          1/1     Running   2          30h
                  kube-proxy-mchc9                          1/1     Running   2          30h
                  kube-scheduler-k8s-master                 1/1     Running   3          30h
                  metrics-server-84f9866fdf-rz676           1/1     Running   15         4d15h
                  

                  2、二進制
                  所有組件均采用systemd管理

                  常見問題:

                  • 網絡不通
                    • ping 節點
                    • telnet 節點端口
                  • 啟動失敗,一般配置文件或者依賴服務
                  • 平臺不兼容
                  # kubeadm部署的,直接查看kube-apiserver-k8s-master的Pod日志
                  [root@k8s-master ~]# kubectl logs kube-apiserver-k8s-master -n kube-system
                  Flag --insecure-port has been deprecated, This flag will be removed in a future version.
                  I1225 14:12:18.558307       1 server.go:625] external host was not specified, using 10.0.0.61
                  I1225 14:12:18.558513       1 server.go:163] Version: v1.19.3
                  I1225 14:12:19.123232       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
                  I1225 14:12:19.123294       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
                  I1225 14:12:19.124128       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
                  I1225 14:12:19.124167       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
                  I1225 14:12:19.126549       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.126601       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.139669       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.139693       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.146921       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.146944       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.147383       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:12:19.147669       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:12:19.147718       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:12:19.194115       1 master.go:271] Using reconciler: lease
                  I1225 14:12:19.194533       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.194550       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.221352       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.221377       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.230469       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.230511       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.240139       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.240181       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.255518       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.255555       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.265105       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.265191       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.275038       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.275076       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.285281       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.285336       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.302076       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.302102       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.314415       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.314679       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.327616       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.327671       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.338580       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.338901       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.354401       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.354487       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.363624       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.363651       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.376090       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.376133       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.386480       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.386534       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.394978       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.395030       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.404842       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.404888       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.559645       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.559692       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.576723       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.576767       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.588265       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.588284       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.596125       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.596145       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.608161       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.608212       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.619144       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.619196       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.626852       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.626895       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.644521       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.644550       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.658031       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.658090       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.669971       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.670265       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.692800       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.692836       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.708784       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.708826       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.734898       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.735032       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.755957       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.755982       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.772847       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.772872       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.788862       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.788886       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.803723       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.803754       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.818516       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.818551       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.826818       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.826857       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.837298       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.837339       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.844194       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.844217       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.857209       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.857597       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.867066       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.867181       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.877262       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.877302       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.889062       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.889099       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.896457       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.902303       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.910393       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.910423       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.927814       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.927861       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.940076       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.940098       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.952012       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.952115       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.961099       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.961123       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.975537       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.975585       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.988067       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.988145       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:19.995939       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:19.995965       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.018436       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.018502       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.109379       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.109398       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.121750       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.121777       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.138751       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.138786       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.148112       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.151713       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.161554       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.161578       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.175335       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.175359       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.193425       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.194080       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.262691       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.262740       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.277204       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.277249       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.299607       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.299713       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.315284       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.315481       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.328823       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.328848       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.345828       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.345871       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.361304       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.361328       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  W1225 14:12:20.640827       1 genericapiserver.go:412] Skipping API batch/v2alpha1 because it has no resources.
                  W1225 14:12:20.659984       1 genericapiserver.go:412] Skipping API discovery.k8s.io/v1alpha1 because it has no resources.
                  W1225 14:12:20.685600       1 genericapiserver.go:412] Skipping API node.k8s.io/v1alpha1 because it has no resources.
                  W1225 14:12:20.717635       1 genericapiserver.go:412] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
                  W1225 14:12:20.722620       1 genericapiserver.go:412] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
                  W1225 14:12:20.746581       1 genericapiserver.go:412] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
                  W1225 14:12:20.774071       1 genericapiserver.go:412] Skipping API apps/v1beta2 because it has no resources.
                  W1225 14:12:20.774104       1 genericapiserver.go:412] Skipping API apps/v1beta1 because it has no resources.
                  I1225 14:12:20.794493       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
                  I1225 14:12:20.794524       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
                  I1225 14:12:20.801886       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.801939       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:20.810029       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:20.810055       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:23.548796       1 dynamic_cafile_content.go:167] Starting request-header::/etc/kubernetes/pki/front-proxy-ca.crt
                  I1225 14:12:23.548865       1 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
                  I1225 14:12:23.549360       1 dynamic_serving_content.go:130] Starting serving-cert::/etc/kubernetes/pki/apiserver.crt::/etc/kubernetes/pki/apiserver.key
                  I1225 14:12:23.549780       1 secure_serving.go:197] Serving securely on [::]:6443
                  I1225 14:12:23.549835       1 dynamic_serving_content.go:130] Starting aggregator-proxy-cert::/etc/kubernetes/pki/front-proxy-client.crt::/etc/kubernetes/pki/front-proxy-client.key
                  I1225 14:12:23.549858       1 tlsconfig.go:240] Starting DynamicServingCertificateController
                  I1225 14:12:23.552336       1 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
                  I1225 14:12:23.552372       1 shared_informer.go:240] Waiting for caches to sync for cluster_authentication_trust_controller
                  I1225 14:12:23.553014       1 apiservice_controller.go:97] Starting APIServiceRegistrationController
                  I1225 14:12:23.553087       1 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
                  I1225 14:12:23.553110       1 controller.go:83] Starting OpenAPI AggregationController
                  I1225 14:12:23.553250       1 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
                  I1225 14:12:23.553295       1 dynamic_cafile_content.go:167] Starting request-header::/etc/kubernetes/pki/front-proxy-ca.crt
                  I1225 14:12:23.561604       1 available_controller.go:404] Starting AvailableConditionController
                  I1225 14:12:23.561627       1 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
                  I1225 14:12:23.561671       1 autoregister_controller.go:141] Starting autoregister controller
                  I1225 14:12:23.561678       1 cache.go:32] Waiting for caches to sync for autoregister controller
                  I1225 14:12:23.561791       1 customresource_discovery_controller.go:209] Starting DiscoveryController
                  E1225 14:12:23.666220       1 controller.go:152] Unable to remove old endpoints from kubernetes service: StorageError: key not found, Code: 1, Key: /registry/masterleases/10.0.0.61, ResourceVersion: 0, AdditionalErrorMsg: 
                  I1225 14:12:23.954656       1 shared_informer.go:247] Caches are synced for cluster_authentication_trust_controller 
                  I1225 14:12:23.995142       1 crdregistration_controller.go:111] Starting crd-autoregister controller
                  I1225 14:12:23.995162       1 shared_informer.go:240] Waiting for caches to sync for crd-autoregister
                  I1225 14:12:23.995170       1 shared_informer.go:247] Caches are synced for crd-autoregister 
                  I1225 14:12:23.995261       1 controller.go:86] Starting OpenAPI controller
                  I1225 14:12:24.019280       1 naming_controller.go:291] Starting NamingConditionController
                  I1225 14:12:24.019448       1 establishing_controller.go:76] Starting EstablishingController
                  I1225 14:12:24.019750       1 nonstructuralschema_controller.go:186] Starting NonStructuralSchemaConditionController
                  I1225 14:12:24.021435       1 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
                  I1225 14:12:24.021505       1 crd_finalizer.go:266] Starting CRDFinalizer
                  I1225 14:12:24.084663       1 cache.go:39] Caches are synced for AvailableConditionController controller
                  I1225 14:12:24.089038       1 cache.go:39] Caches are synced for autoregister controller
                  I1225 14:12:24.155442       1 cache.go:39] Caches are synced for APIServiceRegistrationController controller
                  I1225 14:12:24.299909       1 trace.go:205] Trace[1198217794]: "Get" url:/api/v1/namespaces/ingress-nginx/secrets/nginx-ingress-serviceaccount-token-vh69r,user-agent:kube-apiserver/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:::1 (25-Dec-2021 14:12:23.797) (total time: 502ms):
                  Trace[1198217794]: ---"About to write a response" 502ms (14:12:00.299)
                  Trace[1198217794]: [502.112729ms] [502.112729ms] END
                  I1225 14:12:24.318639       1 controller.go:606] quota admission added evaluator for: leases.coordination.k8s.io
                  I1225 14:12:24.356155       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:24.356216       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:24.389194       1 trace.go:205] Trace[414373803]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-node2,user-agent:kubelet/v1.19.0 (linux/amd64) kubernetes/e199641,client:10.0.0.63 (25-Dec-2021 14:12:23.849) (total time: 539ms):
                  Trace[414373803]: ---"About to write a response" 539ms (14:12:00.389)
                  Trace[414373803]: [539.865826ms] [539.865826ms] END
                  I1225 14:12:24.389582       1 trace.go:205] Trace[346194256]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-node1,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.62 (25-Dec-2021 14:12:23.761) (total time: 627ms):
                  Trace[346194256]: ---"About to write a response" 627ms (14:12:00.389)
                  Trace[346194256]: [627.763742ms] [627.763742ms] END
                  I1225 14:12:24.393405       1 trace.go:205] Trace[538299640]: "Get" url:/api/v1/namespaces/kube-system/pods/kube-apiserver-k8s-master,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.61 (25-Dec-2021 14:12:23.845) (total time: 547ms):
                  Trace[538299640]: ---"About to write a response" 510ms (14:12:00.356)
                  Trace[538299640]: [547.414287ms] [547.414287ms] END
                  I1225 14:12:24.512082       1 trace.go:205] Trace[82502510]: "Create" url:/api/v1/namespaces/default/events,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.61 (25-Dec-2021 14:12:23.846) (total time: 665ms):
                  Trace[82502510]: ---"Object stored in database" 665ms (14:12:00.511)
                  Trace[82502510]: [665.364934ms] [665.364934ms] END
                  I1225 14:12:24.516643       1 trace.go:205] Trace[1819760371]: "GuaranteedUpdate etcd3" type:*core.Event (25-Dec-2021 14:12:23.818) (total time: 698ms):
                  Trace[1819760371]: ---"Transaction prepared" 459ms (14:12:00.277)
                  Trace[1819760371]: ---"Transaction committed" 238ms (14:12:00.516)
                  Trace[1819760371]: [698.586941ms] [698.586941ms] END
                  I1225 14:12:24.523401       1 trace.go:205] Trace[1243567460]: "Patch" url:/api/v1/namespaces/default/events/configmap-demo-pod.16c404be8eee341b,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.62 (25-Dec-2021 14:12:23.795) (total time: 721ms):
                  Trace[1243567460]: ---"Object stored in database" 696ms (14:12:00.516)
                  Trace[1243567460]: [721.328661ms] [721.328661ms] END
                  I1225 14:12:24.629195       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:24.635269       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:24.851524       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:24.851565       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:24.925875       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:24.925902       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:24.941178       1 storage_scheduling.go:143] all system priority classes are created successfully or already exist.
                  I1225 14:12:25.002392       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.002477       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:25.093867       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.093912       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  E1225 14:12:25.245098       1 customresource_handler.go:668] error building openapi models for hostendpoints.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.HostEndpoint.properties.spec.properties.ports.items.<array>.properties.protocol has invalid property: anyOf
                  I1225 14:12:25.245607       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.245627       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:25.277321       1 controller.go:132] OpenAPI AggregationController: action for item : Nothing (removed from the queue).
                  E1225 14:12:25.283228       1 customresource_handler.go:668] error building openapi models for felixconfigurations.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.FelixConfiguration.properties.spec.properties.kubeNodePortRanges.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.FelixConfiguration.properties.spec.properties.natPortRange has invalid property: anyOf
                  I1225 14:12:25.284239       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.284261       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  E1225 14:12:25.329108       1 customresource_handler.go:668] error building openapi models for globalnetworkpolicies.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.notProtocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.protocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.notProtocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.protocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
                  I1225 14:12:25.330596       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.330710       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:25.357189       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.357217       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:25.392966       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.392992       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  E1225 14:12:25.438707       1 customresource_handler.go:668] error building openapi models for networkpolicies.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.notProtocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.protocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.notProtocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.protocol has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
                  ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
                  I1225 14:12:25.439540       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.439593       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:25.448117       1 trace.go:205] Trace[1794566532]: "Get" url:/apis/rbac.authorization.k8s.io/v1/clusterroles/system:aggregate-to-edit,user-agent:kube-apiserver/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:::1 (25-Dec-2021 14:12:24.878) (total time: 569ms):
                  Trace[1794566532]: ---"About to write a response" 569ms (14:12:00.447)
                  Trace[1794566532]: [569.28003ms] [569.28003ms] END
                  I1225 14:12:25.654884       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:25.654910       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:26.441116       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:26.441157       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:12:26.579163       1 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
                  E1225 14:12:29.106613       1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
                  E1225 14:12:34.125834       1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": context deadline exceeded
                  E1225 14:12:39.126932       1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
                  I1225 14:12:41.508946       1 controller.go:606] quota admission added evaluator for: endpoints
                  I1225 14:12:54.169433       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:12:54.169468       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:12:54.169476       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:12:56.191213       1 controller.go:606] quota admission added evaluator for: endpointslices.discovery.k8s.io
                  I1225 14:12:56.631645       1 client.go:360] parsed scheme: "endpoint"
                  I1225 14:12:56.631729       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
                  I1225 14:13:36.699461       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:13:36.699504       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:13:36.699512       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:14:12.928690       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:14:12.928831       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:14:12.928859       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:14:51.609220       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:14:51.609377       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:14:51.609409       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:15:30.414981       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:15:30.415048       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:15:30.415057       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:16:13.416069       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:16:13.416140       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:16:13.416158       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:16:53.202182       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:16:53.202277       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:16:53.202288       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:17:27.709485       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:17:27.709530       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:17:27.709542       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:18:10.159300       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:18:10.159338       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:18:10.159345       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:18:40.716569       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:18:40.716701       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:18:40.716722       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:19:24.247113       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:19:24.247185       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:19:24.247219       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:20:08.987275       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:20:08.987543       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:20:08.987583       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:20:44.858512       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:20:44.858557       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:20:44.858569       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:21:21.613762       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:21:21.613892       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:21:21.614077       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:21:52.143822       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:21:52.143911       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:21:52.143929       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:22:27.359651       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:22:27.359762       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:22:27.359787       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:23:11.063713       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:23:11.063746       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:23:11.063754       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:23:42.744602       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:23:42.744670       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:23:42.744688       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:24:15.053047       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:24:15.053141       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:24:15.053167       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:24:57.281040       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:24:57.286666       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:24:57.286712       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:25:38.863844       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:25:38.863903       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:25:38.863912       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:26:18.572451       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:26:18.572482       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:26:18.572489       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:26:53.678319       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:26:53.678531       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:26:53.678573       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:27:36.433874       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:27:36.434093       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:27:36.434125       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:28:18.084057       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:28:18.084239       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:28:18.084255       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:28:50.563060       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:28:50.563113       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:28:50.563124       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:29:21.855603       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:29:21.855751       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:29:21.856461       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  I1225 14:29:52.347034       1 client.go:360] parsed scheme: "passthrough"
                  I1225 14:29:52.347112       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
                  I1225 14:29:52.347130       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  
                  
                  # 二進制部署的,用journalctl -u kube-apiserver
                  [root@k8s-master ~]# journalctl -u kube-apiserver
                  -- No entries --
                  [root@k8s-master ~]# journalctl -u kubelet
                  -- Logs begin at Sat 2021-12-25 22:12:00 CST, end at Sat 2021-12-25 22:40:02 CST. --
                  Dec 25 22:12:07 k8s-master systemd[1]: Started kubelet: The Kubernetes Node Agent.
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729022    1419 server.go:411] Version: v1.19.3
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729772    1419 server.go:831] Client rotation is on, will bootstrap in background
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.739425    1419 certificate_store.go:130] Loading cert/key pair from "/var/lib/kubelet/pki/kubelet-client-current.pem".
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.745546    1419 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
                  Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.157376    1419 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.208464    1419 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209254    1419 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209278    1419 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName:
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212786    1419 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212810    1419 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212815    1419 container_manager_linux.go:316] Creating device plugin manager: true
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222255    1419 client.go:77] Connecting to docker on unix:///var/run/docker.sock
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222336    1419 client.go:94] Start docker client with request timeout=2m0s
                  Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.261344    1419 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to 
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.261377    1419 docker_service.go:241] Hairpin mode set to "hairpin-veth"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.560569    1419 docker_service.go:256] Docker cri networking managed by cni
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591354    1419 docker_service.go:261] Docker Info: &{ID:LZWZ:7SPV:BJT7:3OAX:HPZJ:2U5R:3D3E:SXVB:A5PX:PJX3:3IHG:OEDN Contain
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591426    1419 docker_service.go:274] Setting cgroupDriver to cgroupfs
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634518    1419 remote_runtime.go:59] parsed scheme: ""
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634540    1419 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635527    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635572    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635629    1419 remote_image.go:50] parsed scheme: ""
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635637    1419 remote_image.go:50] scheme "" not registered, fallback to default scheme
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635651    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635657    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635731    1419 kubelet.go:261] Adding pod path: /etc/kubernetes/manifests
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635762    1419 kubelet.go:273] Watching apiserver
                  Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678016    1419 reflector.go:127] k8s.io/kubernetes/pkg/kubelet/config/apiserver.go:46: Failed to watch *v1.Pod: failed to l
                  Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678147    1419 reflector.go:127] k8s.io/kubernetes/pkg/kubelet/kubelet.go:438: Failed to watch *v1.Node: failed to list *v1
                  Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678221    1419 reflector.go:127] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Service: failed to list *v1
                  Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.977478    1419 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Dep
                  Dec 25 22:12:10 k8s-master kubelet[1419]: For verbose messaging see aws.Config.CredentialsChainVerboseErrors
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.040815    1419 kuberuntime_manager.go:214] Container runtime docker initialized, version: 20.10.11, apiVersion: 1.41.0
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.042898    1419 server.go:1147] Started kubelet
                  Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.044515    1419 kubelet.go:1218] Image garbage collection failed once. Stats initialization may not have completed yet: fail
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.053448    1419 fs_resource_analyzer.go:64] Starting FS ResourceAnalyzer
                  Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.056752    1419 event.go:273] Unable to write event: 'Post "https://10.0.0.61:6443/api/v1/namespaces/default/events": dial t
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.062250    1419 volume_manager.go:265] Starting Kubelet Volume Manager
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.062322    1419 server.go:152] Starting to listen on 0.0.0.0:10250
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.071183    1419 server.go:424] Adding debug handlers to kubelet server.
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.079514    1419 desired_state_of_world_populator.go:139] Desired state populator starts to run
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.088034    1419 topology_manager.go:219] [topologymanager] RemoveContainer - Container ID: db716200328937af6f50e1cd3c23d1391
                  Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.102900    1419 controller.go:136] failed to ensure node lease exists, will retry in 200ms, error: Get "https://10.0.0.61:64
                  Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.103298    1419 reflector.go:127] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.CSIDriver: failed to list *
                  Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.179554    1419 kubelet.go:2183] node "k8s-master" not found
                  Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.186533    1419 client.go:87] parsed scheme: "unix"
                  
                  [root@k8s-master ~]# journalctl -u kubelet > a.txt
                  [root@k8s-master ~]# more a.txt 
                  -- Logs begin at Sat 2021-12-25 22:12:00 CST, end at Sat 2021-12-25 22:40:02 CST. --
                  Dec 25 22:12:07 k8s-master systemd[1]: Started kubelet: The Kubernetes Node Agent.
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729022    1419 server.go:411] Version: v1.19.3
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729772    1419 server.go:831] Client rotation is on, will bootstrap in background
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.739425    1419 certificate_store.go:130] Loading cert/key pair from "/var/lib/kubelet/pki/kubelet-client-current.pem".
                  Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.745546    1419 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
                  Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.157376    1419 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.208464    1419 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209254    1419 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209278    1419 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName:
                   SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false No
                  deAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionT
                  hresholds:[{Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.inodesFree Operator:LessThan Value:{Quan
                  tity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>} {Si
                  gnal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUMa
                  nagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212786    1419 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212810    1419 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212815    1419 container_manager_linux.go:316] Creating device plugin manager: true
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222255    1419 client.go:77] Connecting to docker on unix:///var/run/docker.sock
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222336    1419 client.go:94] Start docker client with request timeout=2m0s
                  Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.261344    1419 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to 
                  "hairpin-veth"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.261377    1419 docker_service.go:241] Hairpin mode set to "hairpin-veth"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.560569    1419 docker_service.go:256] Docker cri networking managed by cni
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591354    1419 docker_service.go:261] Docker Info: &{ID:LZWZ:7SPV:BJT7:3OAX:HPZJ:2U5R:3D3E:SXVB:A5PX:PJX3:3IHG:OEDN Contain
                  ers:27 ContainersRunning:0 ContainersPaused:0 ContainersStopped:27 Images:16 Driver:overlay2 DriverStatus:[[Backing Filesystem xfs] [Supports d_type true] [Native Overlay Diff true
                  ] [userxattr false]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file
                   local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:tru
                  e IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:25 OomKillDisable:true NGoroutines:34 SystemTime:2021-12-25T22:12:10.561900318+08:00 LoggingDrive
                  r:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:3.10.0-1160.45.1.el7.x86_64 OperatingSystem:CentOS Linux 7 (Core) OSType:linux Architecture:x86_64 IndexServerAddr
                  ess:https://index.docker.io/v1/ RegistryConfig:0xc00067cc40 NCPU:2 MemTotal:1907732480 GenericResources:[] DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:k8s-ma
                  ster Labels:[] ExperimentalBuild:false ServerVersion:20.10.11 ClusterStore: ClusterAdvertise: Runtimes:map[io.containerd.runc.v2:{Path:runc Args:[]} io.containerd.runtime.v1.linux:
                  {Path:runc Args:[]} runc:{Path:runc Args:[]}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:
                  0 Cluster:<nil> Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:7b11cfaabd73bb80907dd23182b9347b4245eb5d Expected:7b11cfaabd73bb80907dd
                  23182b9347b4245eb5d} RuncCommit:{ID:v1.0.2-0-g52b36a2 Expected:v1.0.2-0-g52b36a2} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=seccomp,profile=default] ProductLic
                  ense: Warnings:[]}
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591426    1419 docker_service.go:274] Setting cgroupDriver to cgroupfs
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634518    1419 remote_runtime.go:59] parsed scheme: ""
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634540    1419 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635527    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
                  l> <nil>}
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635572    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635629    1419 remote_image.go:50] parsed scheme: ""
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635637    1419 remote_image.go:50] scheme "" not registered, fallback to default scheme
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635651    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
                  l> <nil>}
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635657    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
                  Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635731    1419 kubelet.go:261] Adding pod path: /etc/kubernetes/manifests
                  
                  [root@k8s-master ~]# cat /usr/lib/systemd/system/kubelet.service
                  [Unit]
                  Description=kubelet: The Kubernetes Node Agent
                  Documentation=https://kubernetes.io/docs/
                  Wants=network-online.target
                  After=network-online.target
                  
                  [Service]
                  ExecStart=/usr/bin/kubelet
                  Restart=always
                  StartLimitInterval=0
                  RestartSec=10
                  
                  [Install]
                  WantedBy=multi-user.target
                  [root@k8s-master ~]# /usr/bin/kubelet
                  I1225 22:44:25.980513   28854 server.go:411] Version: v1.19.3
                  W1225 22:44:25.980849   28854 server.go:553] standalone mode, no API client
                  W1225 22:44:25.980983   28854 container_manager_linux.go:951] CPUAccounting not enabled for pid: 28854
                  W1225 22:44:25.980991   28854 container_manager_linux.go:954] MemoryAccounting not enabled for pid: 28854
                  W1225 22:44:26.082976   28854 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
                  W1225 22:44:26.127990   28854 server.go:468] No api server defined - no events will be sent to API server.
                  I1225 22:44:26.128031   28854 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /
                  I1225 22:44:26.128393   28854 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
                  I1225 22:44:26.128413   28854 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:nodefs.inodesFree Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>} {Signal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
                  I1225 22:44:26.128818   28854 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
                  I1225 22:44:26.128827   28854 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
                  I1225 22:44:26.128833   28854 container_manager_linux.go:316] Creating device plugin manager: true
                  I1225 22:44:26.129807   28854 client.go:77] Connecting to docker on unix:///var/run/docker.sock
                  I1225 22:44:26.129830   28854 client.go:94] Start docker client with request timeout=2m0s
                  W1225 22:44:26.183754   28854 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to "hairpin-veth"
                  I1225 22:44:26.183783   28854 docker_service.go:241] Hairpin mode set to "hairpin-veth"
                  I1225 22:44:26.247433   28854 docker_service.go:256] Docker cri networking managed by kubernetes.io/no-op
                  ......
                  
                  

                  3. Service訪問異常排查

                  Service工作流程圖
                  在這里插入圖片描述
                  Service一般是訪問不通,有以下可能性:

                  1. Service是否關聯Pod?
                  # 查看標簽是否正確,已關聯到Pod;是否已創建Pod
                  [root@k8s-master ~]# kubectl get ep
                  NAME             ENDPOINTS                                           AGE
                  fuseim.pri-ifs   <none>                                              4d12h
                  kubernetes       10.0.0.61:6443                                      33d
                  my-dep           <none>                                              30d
                  my-service       10.244.36.119:80,10.244.36.122:80,10.244.36.98:80   24d
                  nginx            10.244.36.119:80,10.244.36.122:80,10.244.36.98:80   4d11h
                  [root@k8s-master ~]# kubectl get pods
                  NAME                                     READY   STATUS    RESTARTS   AGE
                  client1                                  1/1     Running   5          3d
                  client2                                  1/1     Running   5          3d
                  configmap-demo-pod                       1/1     Running   4          4d10h
                  my-pod2                                  1/1     Running   11         4d17h
                  nfs-client-provisioner-58d675cd5-dx7n4   1/1     Running   7          4d12h
                  pod-taint                                1/1     Running   9          10d
                  secret-demo-pod                          1/1     Running   4          4d9h
                  sh                                       1/1     Running   6          4d11h
                  test-76846b5956-gftn9                    1/1     Running   2          4d11h
                  test-76846b5956-r7s9k                    1/1     Running   2          4d11h
                  test-76846b5956-trpbn                    1/1     Running   2          4d11h
                  test2-78c4694588-87b9r                   1/1     Running   5          4d12h
                  web-0                                    1/1     Running   4          4d11h
                  web-1                                    1/1     Running   4          4d11h
                  web-2                                    1/1     Running   4          4d11h
                  web-96d5df5c8-vc9kf                      1/1     Running   3          3d
                  
                  1. Service指定target-port端口是否正常?
                  # 查看service中的target-port是否是指定的端口
                  [root@k8s-master ~]# kubectl edit svc nginx
                  Edit cancelled, no changes made.
                  # Please edit the object below. Lines beginning with a '#' will be ignored,
                  # and an empty file will abort the edit. If an error occurs while saving this file will be
                  # reopened with the relevant failures.
                  #
                  apiVersion: v1
                  kind: Service
                  metadata:
                    annotations:
                      kubectl.kubernetes.io/last-applied-configuration: |
                        {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"nginx"},"name":"nginx","namespace":"default"},"spec":{"clusterIP":"None","ports":[{"name":"web","port":80}],"selector":{"app":"nginx"}}}
                    creationTimestamp: "2021-12-21T02:56:44Z"
                    labels:
                      app: nginx
                    name: nginx
                    namespace: default
                    resourceVersion: "2334070"
                    selfLink: /api/v1/namespaces/default/services/nginx
                    uid: 5f07839a-04e4-4214-bbbe-d69357de10d4
                  spec:
                    clusterIP: None
                    ports:
                    - name: web
                      port: 80
                      protocol: TCP
                      targetPort: 80
                    selector:
                      app: nginx
                    sessionAffinity: None
                    type: ClusterIP
                  status:
                    loadBalancer: {}
                  
                  1. Pod正常工作嗎?
                  # 查看Pod的IP地址,然后用curl命令去看是否返回正常的內容
                  [root@k8s-master ~]# kubectl get pods -o wide
                  NAME                                     READY   STATUS    RESTARTS   AGE     IP               NODE        NOMINATED NODE   READINESS GATES
                  client1                                  1/1     Running   5          3d      10.244.36.99     k8s-node1   <none>           <none>
                  client2                                  1/1     Running   5          3d      10.244.36.92     k8s-node1   <none>           <none>
                  configmap-demo-pod                       1/1     Running   4          4d10h   10.244.36.101    k8s-node1   <none>           <none>
                  my-pod2                                  1/1     Running   11         4d17h   10.244.169.130   k8s-node2   <none>           <none>
                  nfs-client-provisioner-58d675cd5-dx7n4   1/1     Running   7          4d12h   10.244.36.116    k8s-node1   <none>           <none>
                  pod-taint                                1/1     Running   9          10d     10.244.169.132   k8s-node2   <none>           <none>
                  secret-demo-pod                          1/1     Running   4          4d9h    10.244.36.118    k8s-node1   <none>           <none>
                  sh                                       1/1     Running   6          4d11h   10.244.36.114    k8s-node1   <none>           <none>
                  test-76846b5956-gftn9                    1/1     Running   2          4d11h   10.244.36.111    k8s-node1   <none>           <none>
                  test-76846b5956-r7s9k                    1/1     Running   2          4d11h   10.244.36.100    k8s-node1   <none>           <none>
                  test-76846b5956-trpbn                    1/1     Running   2          4d11h   10.244.169.185   k8s-node2   <none>           <none>
                  test2-78c4694588-87b9r                   1/1     Running   5          4d13h   10.244.36.123    k8s-node1   <none>           <none>
                  web-0                                    1/1     Running   4          4d12h   10.244.36.122    k8s-node1   <none>           <none>
                  web-1                                    1/1     Running   4          4d12h   10.244.36.119    k8s-node1   <none>           <none>
                  web-2                                    1/1     Running   4          4d12h   10.244.36.98     k8s-node1   <none>           <none>
                  web-96d5df5c8-vc9kf                      1/1     Running   3          3d      10.244.169.158   k8s-node2   <none>           <none>
                  [root@k8s-master ~]# curl 10.244.169.158
                  
                  1. Service是否通過DNS工作?
                  # 查看coredns組件是否正常工作
                  [root@k8s-master ~]# kubectl get pods -n kube-system
                  NAME                                      READY   STATUS    RESTARTS   AGE
                  calico-kube-controllers-97769f7c7-c4cg5   1/1     Running   3          31h
                  calico-node-4pwdc                         1/1     Running   16         33d
                  calico-node-9r6zd                         1/1     Running   16         33d
                  calico-node-vqzdj                         1/1     Running   17         33d
                  client1                                   1/1     Running   5          3d
                  coredns-6d56c8448f-gcgrh                  1/1     Running   16         33d
                  coredns-6d56c8448f-mdl7c                  1/1     Running   2          31h
                  etcd-k8s-master                           1/1     Running   3          31h
                  filebeat-5pwh7                            1/1     Running   11         10d
                  filebeat-pt848                            1/1     Running   11         10d
                  kube-apiserver-k8s-master                 1/1     Running   3          31h
                  kube-controller-manager-k8s-master        1/1     Running   3          31h
                  kube-proxy-87lbj                          1/1     Running   3          31h
                  kube-proxy-mcdnv                          1/1     Running   2          31h
                  kube-proxy-mchc9                          1/1     Running   2          31h
                  kube-scheduler-k8s-master                 1/1     Running   3          31h
                  metrics-server-84f9866fdf-rz676           1/1     Running   15         4d16h
                  
                  1. kube-proxy正常工作嗎?
                  # kubeadm部署的查看kube-proxy看是否工作正常
                  [root@k8s-master ~]# kubectl get pods -n kube-system
                  NAME                                      READY   STATUS    RESTARTS   AGE
                  calico-kube-controllers-97769f7c7-c4cg5   1/1     Running   3          31h
                  calico-node-4pwdc                         1/1     Running   16         33d
                  calico-node-9r6zd                         1/1     Running   16         33d
                  calico-node-vqzdj                         1/1     Running   17         33d
                  client1                                   1/1     Running   5          3d
                  coredns-6d56c8448f-gcgrh                  1/1     Running   16         33d
                  coredns-6d56c8448f-mdl7c                  1/1     Running   2          31h
                  etcd-k8s-master                           1/1     Running   3          31h
                  filebeat-5pwh7                            1/1     Running   11         10d
                  filebeat-pt848                            1/1     Running   11         10d
                  kube-apiserver-k8s-master                 1/1     Running   3          31h
                  kube-controller-manager-k8s-master        1/1     Running   3          31h
                  kube-proxy-87lbj                          1/1     Running   3          31h
                  kube-proxy-mcdnv                          1/1     Running   2          31h
                  kube-proxy-mchc9                          1/1     Running   2          31h
                  kube-scheduler-k8s-master                 1/1     Running   3          31h
                  metrics-server-84f9866fdf-rz676           1/1     Running   15         4d16h
                  
                  # 二進制查看systemd
                  
                  
                  1. kube-proxy是否正常寫iptables規則?
                  # 使用iptables-save |grep service名稱來查看,對應規則是否被創建
                  [root@k8s-master ~]# iptables -L -n
                  Chain INPUT (policy ACCEPT)
                  target     prot opt source               destination         
                  cali-INPUT  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Cz_u1IQiXIMmKD4c */
                  KUBE-FIREWALL  all  --  0.0.0.0/0            0.0.0.0/0           
                  
                  Chain FORWARD (policy DROP)
                  target     prot opt source               destination         
                  cali-FORWARD  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:wUHhoiAYhphO9Mso */
                  KUBE-FORWARD  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding rules */
                  DOCKER-USER  all  --  0.0.0.0/0            0.0.0.0/0           
                  DOCKER-ISOLATION-STAGE-1  all  --  0.0.0.0/0            0.0.0.0/0           
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            ctstate RELATED,ESTABLISHED
                  DOCKER     all  --  0.0.0.0/0            0.0.0.0/0           
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0           
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0           
                  
                  Chain OUTPUT (policy ACCEPT)
                  target     prot opt source               destination         
                  cali-OUTPUT  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:tVnHkvAo15HuiPy0 */
                  KUBE-FIREWALL  all  --  0.0.0.0/0            0.0.0.0/0           
                  
                  Chain DOCKER (1 references)
                  target     prot opt source               destination         
                  
                  Chain DOCKER-ISOLATION-STAGE-1 (1 references)
                  target     prot opt source               destination         
                  DOCKER-ISOLATION-STAGE-2  all  --  0.0.0.0/0            0.0.0.0/0           
                  RETURN     all  --  0.0.0.0/0            0.0.0.0/0           
                  
                  Chain DOCKER-ISOLATION-STAGE-2 (1 references)
                  target     prot opt source               destination         
                  DROP       all  --  0.0.0.0/0            0.0.0.0/0           
                  RETURN     all  --  0.0.0.0/0            0.0.0.0/0           
                  
                  Chain DOCKER-USER (1 references)
                  target     prot opt source               destination         
                  RETURN     all  --  0.0.0.0/0            0.0.0.0/0           
                  
                  Chain KUBE-FIREWALL (2 references)
                  target     prot opt source               destination         
                  DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
                  DROP       all  -- !127.0.0.0/8          127.0.0.0/8          /* block incoming localnet connections */ ! ctstate RELATED,ESTABLISHED,DNAT
                  
                  Chain KUBE-FORWARD (1 references)
                  target     prot opt source               destination         
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding rules */ mark match 0x4000/0x4000
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding conntrack pod source rule */ ctstate RELATED,ESTABLISHED
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding conntrack pod destination rule */ ctstate RELATED,ESTABLISHED
                  
                  Chain KUBE-KUBELET-CANARY (0 references)
                  target     prot opt source               destination         
                  
                  Chain cali-FORWARD (1 references)
                  target     prot opt source               destination         
                  MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:vjrMJCRpqwy5oRoX */ MARK and 0xfff1ffff
                  cali-from-hep-forward  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:A_sPAO0mcxbT9mOV */ mark match 0x0/0x10000
                  cali-from-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:8ZoYfO5HKXWbB3pk */
                  cali-to-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:jdEuaPBe14V2hutn */
                  cali-to-hep-forward  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:12bc6HljsMKsmfr- */
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:MH9kMp5aNICL-Olv */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000
                  
                  Chain cali-INPUT (1 references)
                  target     prot opt source               destination         
                  ACCEPT     4    --  0.0.0.0/0            0.0.0.0/0            /* cali:PajejrV4aFdkZojI */ /* Allow IPIP packets from Calico hosts */ match-set cali40all-hosts-net src ADDRTYPE match dst-type LOCAL
                  DROP       4    --  0.0.0.0/0            0.0.0.0/0            /* cali:_wjq-Yrma8Ly1Svo */ /* Drop IPIP packets from non-Calico hosts */
                  MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:ss8lEMQsXi-s6qYT */ MARK and 0xfffff
                  cali-forward-check  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:PgIW-V0nEjwPhF_8 */
                  RETURN     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:QMJlDwlS0OjHyfMN */ mark match ! 0x0/0xfff00000
                  cali-wl-to-host  all  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:nDRe73txrna-aZjG */
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:iX2AYvqGXaVqwkro */ mark match 0x10000/0x10000
                  MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:bhpnxD5IRtBP8KW0 */ MARK and 0xfff0ffff
                  cali-from-host-endpoint  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:H5_bccAbHV0sooVy */
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:inBL01YlfurT0dbI */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000
                  
                  Chain cali-OUTPUT (1 references)
                  target     prot opt source               destination         
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Mq1_rAdXXH3YkrzW */ mark match 0x10000/0x10000
                  cali-forward-endpoint-mark  all  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:5Z67OUUpTOM7Xa1a */ mark match ! 0x0/0xfff00000
                  RETURN     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:M2Wf0OehNdig8MHR */
                  ACCEPT     4    --  0.0.0.0/0            0.0.0.0/0            /* cali:AJBkLho_0Qd8LNr3 */ /* Allow IPIP packets to other Calico hosts */ match-set cali40all-hosts-net dst ADDRTYPE match src-type LOCAL
                  MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:iz2RWXlXJDUfsLpe */ MARK and 0xfff0ffff
                  cali-to-host-endpoint  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:hXojbnLundZDgZyw */
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:wankpMDC2Cy1KfBv */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000
                  
                  Chain cali-forward-check (1 references)
                  target     prot opt source               destination         
                  RETURN     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Pbldlb4FaULvpdD8 */ ctstate RELATED,ESTABLISHED
                  cali-set-endpoint-mark  tcp  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:ZD-6UxuUtGW-xtzg */ /* To kubernetes NodePort service */ multiport dports 30000:32767 match-set cali40this-host dst
                  cali-set-endpoint-mark  udp  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:CbPfUajQ2bFVnDq4 */ /* To kubernetes NodePort service */ multiport dports 30000:32767 match-set cali40this-host dst
                  cali-set-endpoint-mark  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:jmhU0ODogX-Zfe5g */ /* To kubernetes service */ ! match-set cali40this-host dst
                  
                  Chain cali-forward-endpoint-mark (1 references)
                  target     prot opt source               destination         
                  cali-from-endpoint-mark  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:O0SmFDrnm7KggWqW */ mark match ! 0x100000/0xfff00000
                  cali-to-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:aFl0WFKRxDqj8oA6 */
                  cali-to-hep-forward  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:AZKVrO3i_8cLai5f */
                  MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:96HaP1sFtb-NYoYA */ MARK and 0xfffff
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:VxO6hyNWz62YEtul */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000
                  
                  Chain cali-from-endpoint-mark (1 references)
                  target     prot opt source               destination         
                  DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:9dpftzl-pNycbr37 */ /* Unknown interface */
                  
                  Chain cali-from-hep-forward (1 references)
                  target     prot opt source               destination         
                  
                  Chain cali-from-host-endpoint (1 references)
                  target     prot opt source               destination         
                  
                  Chain cali-from-wl-dispatch (2 references)
                  target     prot opt source               destination         
                  DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:zTj6P0TIgYvgz-md */ /* Unknown interface */
                  
                  Chain cali-set-endpoint-mark (3 references)
                  target     prot opt source               destination         
                  DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:MN61lcxFj1yWuYBo */ /* Unknown endpoint */
                  MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:nKOjq8N2yzfmS3jk */ /* Non-Cali endpoint mark */ MARK xset 0x100000/0xfff00000
                  
                  Chain cali-to-hep-forward (2 references)
                  target     prot opt source               destination         
                  
                  Chain cali-to-host-endpoint (1 references)
                  target     prot opt source               destination         
                  
                  Chain cali-to-wl-dispatch (2 references)
                  target     prot opt source               destination         
                  DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:7KNphB1nNHw80nIO */ /* Unknown interface */
                  
                  Chain cali-wl-to-host (1 references)
                  target     prot opt source               destination         
                  cali-from-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Ee9Sbo10IpVujdIY */
                  ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:nSZbcOoG1xPONxb8 */ /* Configured DefaultEndpointToHostAction */
                  [root@k8s-master ~]# 
                  [root@k8s-master ~]# kubectl get svc
                  NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
                  kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        33d
                  my-dep       NodePort    10.111.199.51   <none>        80:31734/TCP   30d
                  my-service   NodePort    10.100.228.0    <none>        80:32433/TCP   24d
                  nginx        ClusterIP   None            <none>        80/TCP         4d12h
                  [root@k8s-master ~]# iptables-save |grep nginx
                  [root@k8s-master ~]# iptables-save |grep my-dep
                  [root@k8s-master ~]# iptables-save |grep my-service
                  [root@k8s-master ~]# iptables-save |grep kubernetes
                  -A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD
                  -A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP
                  -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
                  -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
                  -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
                  -A cali-forward-check -p tcp -m comment --comment "cali:ZD-6UxuUtGW-xtzg" -m comment --comment "To kubernetes NodePort service" -m multiport --dports 30000:32767 -m set --match-set cali40this-host dst -g cali-set-endpoint-mark
                  -A cali-forward-check -p udp -m comment --comment "cali:CbPfUajQ2bFVnDq4" -m comment --comment "To kubernetes NodePort service" -m multiport --dports 30000:32767 -m set --match-set cali40this-host dst -g cali-set-endpoint-mark
                  -A cali-forward-check -m comment --comment "cali:jmhU0ODogX-Zfe5g" -m comment --comment "To kubernetes service" -m set ! --match-set cali40this-host dst -j cali-set-endpoint-mark
                  -A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
                  -A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
                  -A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
                  -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
                  
                  1. cni網絡插件是否正常工作?
                    扁平化、轉發、實現不同節點間的通信
                    還是去查看IP,然后去curl那個IP地址,看是否返回正常值

                  小結:

                  1. 關閉kube-apiserver,執行不了kubectl命令
                  2. 關閉kube-controller-manager,創建deployment,但是deployment不會去創建具體的pod
                  3. 關閉kube-scheduler,Pod無法調度
                  4. 關閉kubelet,Node顯示notready
                  5. 關閉kube-proxy,網絡不通
                  隨時隨地學軟件編程-關注百度小程序和微信小程序
                  關于找一找教程網

                  本站文章僅代表作者觀點,不代表本站立場,所有文章非營利性免費分享。
                  本站提供了軟件編程、網站開發技術、服務器運維、人工智能等等IT技術文章,希望廣大程序員努力學習,讓我們用科技改變世界。
                  [Kubernetes CKA認證運維工程師筆記-Kubernetes故障排查]http://www.yachtsalesaustralia.com/tech/detail-275709.html

                  贊(0)
                  關注微信小程序
                  程序員編程王-隨時隨地學編程

                  掃描二維碼或查找【程序員編程王】

                  可以隨時隨地學編程啦!

                  技術文章導航 更多>
                  掃一掃關注最新編程教程
                  国产在线拍揄自揄视频菠萝

                        1. <dd id="erndk"></dd>