简介

整体思路，按官网文档Daemonset+HostNetwork+LB 方式部署Ngin ingress，然后创建七层CLB，将证书卸载到CLB。

方案步骤

• 按官网文档 部署Nginx Ingress，整个过程三条命令完成

  • 给需要运行Nginx Ingress 实例的节点打标签

    kubectl label node 10.0.0.3 nginx-ingress=true
    

  • 创建Nginx Ingress负载运行的命名空间

    kubectl create ns nginx-ingress
    

  • 通过yaml创建Nginx Ingress Controller，HostNetwork 方式部署

    kubectl apply -f <https://raw.githubusercontent.com/TencentCloudContainerTeam/manifest/master/nginx-ingress/nginx-ingress-daemonset-hostnetwork.yaml> -n nginx-ingress
    

    https://raw.githubusercontent.com/TencentCloudContainerTeam/manifest/master/nginx-ingress/nginx-ingress-daemonset-hostnetwork.yaml

  • 查看Nginx Ingress Controller是否创建成功

    kubectl get po -n nginx-ingress
    

    

• 在CLB控制台手动创建CLB，配置七层转发到Nginx Ingress实例所在节点的80端口

  

  附录：
  nginx-ingress-daemonset-hostnetwork.yaml 文件内容如下：

  apiVersion: v1
  kind: ConfigMap
  metadata:
    name: nginx-ingress-controller
    namespace: nginx-ingress
  data:
    # nginx 与 client 保持的一个长连接能处理的请求数量，默认 100，高并发场景建议调高。
    # 参考: <https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive-requests>
    keep-alive-requests: "10000"
    # nginx 与 upstream 保持长连接的最大空闲连接数 (不是最大连接数)，默认 32，在高并发下场景下调大，避免频繁建连导致 TIME_WAIT 飙升。
    # 参考: <https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#upstream-keepalive-connections>
    upstream-keepalive-connections: "200"
    # 每个 worker 进程可以打开的最大连接数，默认 16384。
    # 参考: <https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections>
    max-worker-connections: "65536"
  ---
  apiVersion: v1
  kind: ServiceAccount
  metadata:
    labels:
      app: nginx-ingress
    name: nginx-ingress
    namespace: nginx-ingress
  ---
  apiVersion: v1
  kind: ServiceAccount
  metadata:
    labels:
      app: nginx-ingress
    name: nginx-ingress-backend
    namespace: nginx-ingress
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRole
  metadata:
    labels:
      app: nginx-ingress
    name: nginx-ingress
  rules:
    - apiGroups:
        - ""
      resources:
        - configmaps
        - endpoints
        - nodes
        - pods
        - secrets
      verbs:
        - list
        - watch
    - apiGroups:
        - ""
      resources:
        - nodes
      verbs:
        - get
    - apiGroups:
        - ""
      resources:
        - services
      verbs:
        - get
        - list
        - update
        - watch
    - apiGroups:
        - extensions
        - "networking.k8s.io" # k8s 1.14+
      resources:
        - ingresses
      verbs:
        - get
        - list
        - watch
    - apiGroups:
        - ""
      resources:
        - events
      verbs:
        - create
        - patch
    - apiGroups:
        - extensions
        - "networking.k8s.io" # k8s 1.14+
      resources:
        - ingresses/status
      verbs:
        - update
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRoleBinding
  metadata:
    labels:
      app: nginx-ingress
    name: nginx-ingress
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: ClusterRole
    name: nginx-ingress
  subjects:
    - kind: ServiceAccount
      name: nginx-ingress
      namespace: nginx-ingress
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: Role
  metadata:
    labels:
      app: nginx-ingress
    name: nginx-ingress
    namespace: nginx-ingress
  rules:
    - apiGroups:
        - ""
      resources:
        - namespaces
      verbs:
        - get
    - apiGroups:
        - ""
      resources:
        - configmaps
        - pods
        - secrets
        - endpoints
      verbs:
        - get
        - list
        - watch
    - apiGroups:
        - ""
      resources:
        - services
      verbs:
        - get
        - list
        - update
        - watch
    - apiGroups:
        - extensions
        - "networking.k8s.io" # k8s 1.14+
      resources:
        - ingresses
      verbs:
        - get
        - list
        - watch
    - apiGroups:
        - extensions
        - "networking.k8s.io" # k8s 1.14+
      resources:
        - ingresses/status
      verbs:
        - update
    - apiGroups:
        - ""
      resources:
        - configmaps
      resourceNames:
        - ingress-controller-leader-nginx
      verbs:
        - get
        - update
    - apiGroups:
        - ""
      resources:
        - configmaps
      verbs:
        - create
    - apiGroups:
        - ""
      resources:
        - endpoints
      verbs:
        - create
        - get
        - update
    - apiGroups:
        - ""
      resources:
        - events
      verbs:
        - create
        - patch
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: RoleBinding
  metadata:
    labels:
      app: nginx-ingress
    name: nginx-ingress
    namespace: nginx-ingress
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: Role
    name: nginx-ingress
  subjects:
  - kind: ServiceAccount
    name: nginx-ingress
    namespace: nginx-ingress
  ---
  apiVersion: v1
  kind: Service
  metadata:
    labels:
      app: nginx-ingress
      component: controller
    name: nginx-ingress-controller-metrics
    namespace: nginx-ingress
  spec:
    ports:
    - name: metrics
      port: 9913
      targetPort: metrics
    selector:
      app: nginx-ingress
      component: controller
    type: "ClusterIP"
  ---
  apiVersion: v1
  kind: Service
  metadata:
    labels:
      app: nginx-ingress
      component: default-backend
    name: nginx-ingress-default-backend
    namespace: nginx-ingress
  spec:
    ports:
      - name: http
        port: 80
        protocol: TCP
        targetPort: http
    selector:
      app: nginx-ingress
      component: default-backend
    type: "ClusterIP"
  ---
  apiVersion: apps/v1
  kind: DaemonSet
  metadata:
    labels:
      app: nginx-ingress
      component: controller
    name: nginx-ingress-controller
    namespace: nginx-ingress
  spec:
    selector:
      matchLabels:
        app: nginx-ingress
        component: controller
    template:
      metadata:
        labels:
          app: nginx-ingress
          component: controller
      spec:
        dnsPolicy: ClusterFirst
        initContainers:
        - name: setsysctl
          image: busybox
          securityContext:
            privileged: true
          command:
          - sh
          - -c
          - |
            sysctl -w net.core.somaxconn=65535
            sysctl -w net.ipv4.ip_local_port_range="1024 65535"
            sysctl -w net.ipv4.tcp_tw_reuse=1
            sysctl -w fs.file-max=1048576
        containers:
          - name: nginx-ingress-controller
            image: "ccr.ccs.tencentyun.com/mirrors/nginx-ingress-controller:v0.34.1"
            imagePullPolicy: IfNotPresent
            args:
            - /nginx-ingress-controller
            - --default-backend-service=$(POD_NAMESPACE)/nginx-ingress-default-backend
            - --election-id=ingress-controller-leader
            - --ingress-class=nginx
            - --configmap=$(POD_NAMESPACE)/nginx-ingress-controller
            securityContext:
              capabilities:
                  drop:
                  - ALL
                  add:
                  - NET_BIND_SERVICE
              runAsUser: 101
              allowPrivilegeEscalation: true
            env:
              - name: POD_NAME
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.name
              - name: POD_NAMESPACE
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.namespace
            livenessProbe:
              httpGet:
                path: /healthz
                port: 10254
                scheme: HTTP
              initialDelaySeconds: 10
              periodSeconds: 10
              timeoutSeconds: 1
              successThreshold: 1
              failureThreshold: 3
            ports:
              - name: http
                containerPort: 80
                protocol: TCP
              - name: https
                containerPort: 443
                protocol: TCP
              - name: metrics
                containerPort: 10254
                protocol: TCP
            readinessProbe:
              httpGet:
                path: /healthz
                port: 10254
                scheme: HTTP
              initialDelaySeconds: 10
              periodSeconds: 10
              timeoutSeconds: 1
              successThreshold: 1
              failureThreshold: 3
        hostNetwork: true
        affinity:
          nodeAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              nodeSelectorTerms:
              - matchExpressions:
                - key: nginx-ingress
                  operator: In
                  values:
                  - "true"
        serviceAccountName: nginx-ingress
        terminationGracePeriodSeconds: 60
  ---
  apiVersion: apps/v1
  kind: Deployment
  metadata:
    labels:
      app: nginx-ingress
      component: default-backend
    name: nginx-ingress-default-backend
    namespace: nginx-ingress
  spec:
    selector:
      matchLabels:
        app: nginx-ingress
        component: default-backend
    replicas: 1
    revisionHistoryLimit: 10
    template:
      metadata:
        labels:
          app: nginx-ingress
          component: default-backend
      spec:
        containers:
        - name: nginx-ingress-default-backend
          image: "ccr.ccs.tencentyun.com/mirrors/ingress-defaultbackend-amd64:1.5"
          imagePullPolicy: IfNotPresent
          args:
          securityContext:
            runAsUser: 65534
          livenessProbe:
            httpGet:
              path: /healthz
              port: 8080
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /healthz
              port: 8080
              scheme: HTTP
            initialDelaySeconds: 0
            periodSeconds: 5
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 6
          ports:
          - name: http
            containerPort: 8080
            protocol: TCP
        serviceAccountName: nginx-ingress-backend
        terminationGracePeriodSeconds: 60