基本使用

  • 下载image

    1
    2
    3
    4
    5
    6
    7
    # 获取需要下载的image
    grep image mandatory.yaml
    image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0
    # 下载代理下载image
    docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/nginx-ingress-controller:0.25.0
    # 打上tag
    docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/nginx-ingress-controller:0.25.0 quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0

  • 修改mandatory.yaml文件

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    apiVersion: apps/v1
    kind: Deployment #改成 DaemonSet
    metadata:
      name: nginx-ingress-controller
      namespace: ingress-nginx
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      replicas: 1 # 删除replicas
      selector:
        matchLabels:
          app.kubernetes.io/name: ingress-nginx
          app.kubernetes.io/part-of: ingress-nginx
      template:
        metadata:
          labels:
            app.kubernetes.io/name: ingress-nginx
            app.kubernetes.io/part-of: ingress-nginx
          annotations:
            prometheus.io/port: "10254"
            prometheus.io/scrape: "true"
        spec:
          serviceAccountName: nginx-ingress-serviceaccount
          # 网络模式为hostNetwork
          hostNetwork: true
          # node选择器
          nodeSelector:
          	# app_gateway便签的
            app_gateway: ingress
          containers:
            - name: nginx-ingress-controller
              image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0
              args:
                - /nginx-ingress-controller
                - --configmap=$(POD_NAMESPACE)/nginx-configuration
                - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
                - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
                - --publish-service=$(POD_NAMESPACE)/ingress-nginx
                - --annotations-prefix=nginx.ingress.kubernetes.io
              securityContext:
                allowPrivilegeEscalation: true

  • err services "ingress-nginx" not found异常

    ingress-nginx-service.yaml

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    kind: Service
    apiVersion: v1
    metadata:
      name: ingress-nginx
      namespace: ingress-nginx
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      externalTrafficPolicy: Local
      type: LoadBalancer
      selector:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      ports:
        - name: http
          port: 80
          targetPort: http
        - name: https
          port: 443
          targetPort: https

  • 添加ingress-nginx的pod节点

    1
    2
    3
    4
    5
    6
    # 查看ingress-nginx便签
    kubectl get ds -n ingress-nginx nginx-ingress-controller
    # 在nodel节点添加NODE SELECTOR对应的便签(这里我nginx-ingress的便签是app_gateway=ingress)
    kubectl label node vmware-3 app_gateway=ingress
    # 删除同理,清除node标签
    kubectl label node vmware-3 app_gateway-

  • 获取nginx-ingress-controller.yaml配置文件

    1
    kubectl get daemonsets -n ingress-nginx -o yaml > ingress-nginx-controller.yaml

    删除无用的配置

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    apiVersion: v1
    items:
    - apiVersion: extensions/v1beta1
      kind: DaemonSet
      metadata:
        labels:
          app.kubernetes.io/name: ingress-nginx
          app.kubernetes.io/part-of: ingress-nginx
        name: nginx-ingress-controller
        namespace: ingress-nginx
      spec:
        revisionHistoryLimit: 10
        selector:
          matchLabels:
            app.kubernetes.io/name: ingress-nginx
            app.kubernetes.io/part-of: ingress-nginx
        template:
          metadata:
            annotations:
              prometheus.io/port: "10254"
              prometheus.io/scrape: "true"
            creationTimestamp: null
            labels:
              app.kubernetes.io/name: ingress-nginx
              app.kubernetes.io/part-of: ingress-nginx
          spec:
            containers:
            - args:
              - /nginx-ingress-controller
              - --configmap=$(POD_NAMESPACE)/nginx-configuration
              - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
              - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
              - --publish-service=$(POD_NAMESPACE)/ingress-nginx
              - --annotations-prefix=nginx.ingress.kubernetes.io
              env:
              - name: POD_NAME
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.name
              - name: POD_NAMESPACE
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
              image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1
              imagePullPolicy: IfNotPresent
              lifecycle:
                preStop:
                  exec:
                    command:
                    - /wait-shutdown
              livenessProbe:
                failureThreshold: 3
                httpGet:
                  path: /healthz
                  port: 10254
                  scheme: HTTP
                initialDelaySeconds: 10
                periodSeconds: 10
                successThreshold: 1
                timeoutSeconds: 10
              name: nginx-ingress-controller
              ports:
              - containerPort: 80
                hostPort: 80
                name: http
                protocol: TCP
              - containerPort: 443
                hostPort: 443
                name: https
                protocol: TCP
              readinessProbe:
                failureThreshold: 3
                httpGet:
                  path: /healthz
                  port: 10254
                  scheme: HTTP
                periodSeconds: 10
                successThreshold: 1
                timeoutSeconds: 10
              resources: {}
              securityContext:
                allowPrivilegeEscalation: true
                capabilities:
                  add:
                  - NET_BIND_SERVICE
                  drop:
                  - ALL
                runAsUser: 33
              terminationMessagePath: /dev/termination-log
              terminationMessagePolicy: File
            dnsPolicy: ClusterFirst
            hostNetwork: true
            nodeSelector:
              app_gateway: ingress
            restartPolicy: Always
            schedulerName: default-scheduler
            securityContext: {}
            serviceAccount: nginx-ingress-serviceaccount
            serviceAccountName: nginx-ingress-serviceaccount
            terminationGracePeriodSeconds: 300
        templateGeneration: 1
        updateStrategy:
          rollingUpdate:
            maxUnavailable: 1
          type: RollingUpdate
      status:
        currentNumberScheduled: 1
        desiredNumberScheduled: 1
        numberAvailable: 1
        numberMisscheduled: 0
        numberReady: 1
        observedGeneration: 1
        updatedNumberScheduled: 1
    kind: List
    metadata:
      resourceVersion: ""
      selfLink: ""

  • 测试demo

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    #deploy
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: tomcat-demo
    spec:
      selector:
        matchLabels:
          app: tomcat-demo
      replicas: 1
      template:
        metadata:
          labels:
            app: tomcat-demo
        spec:
          containers:
          - name: tomcat-demo
            image: tomcat:8-slim
            ports:
            - containerPort: 8080
    ---
    #service
    apiVersion: v1
    kind: Service
    metadata:
      name: tomcat-demo
    spec:
      ports:
      - port: 80
        protocol: TCP
        targetPort: 8080
      selector:
        app: tomcat-demo

    ---
    #ingress
    apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
      name: tomcat-demo
    spec:
      rules:
      - host: tomcat.chcks.com
        http:
          paths:
          - path: /
            backend:
              serviceName: tomcat-demo
              servicePort: 80

  • 创建pod

    1
    2
    3
    kubectl create -f ingerss-demo.yaml
    # 查看创建情况
    kubectl get pod -o wide

  • 如果实在有域名的服务器上,则需要修改自己主机的host加入

    1
    192.168.43.112 tomcat.chcks.com

    然后在浏览器上访问:tomcat.chcks.com

四层代理配置

tcp-service.yaml :指定对外服务端口(服务发现)

1
2
3
4
5
6
7
apiVersion: v1
kind: ConfigMap
metadata:
  name: tcp-services
  namespace: ingress-nginx
data:
  "30000": default/tomcat-demo:80 # 指定服务对外的端口

定制配置

nginx的基本配置

1
2
3
4
5
6
7
8
9
10
11
12
# nginx配置
kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app: ingress-nginx
data:
  proxy-body-size: "64m"          # 客户端请求主体的大小
  proxy-read-timeout: "180"       # 设置从代理服务器读取响应的超时(以秒为单位)
  proxy-send-timeout: "180"       # 设置向代理服务器传输请求的超时(以秒为单位)

配置的文档地址:https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/

Header配置

文档地址:https://kubernetes.github.io/ingress-nginx/examples/customization/custom-headers/

  • nginx-all-header.yaml:全局header配置

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    apiVersion: v1
    data:
      proxy-set-headers: "ingress-nginx/custom-headers"  # 定义header配置名称,规范:namespace/名称
    kind: ConfigMap
    metadata:
      name: nginx-configuration
      namespace: ingress-nginx
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    ---
    # 定义全局的header
    apiVersion: v1
    data:
      # 添加header的内容
      X-Different-Name: "true"
      X-Request-Start: t=${msec}
      X-Using-Nginx-Controller: "true"
    kind: ConfigMap
    metadata:
      name: custom-headers
      namespace: ingress-nginx

  • nginx-service-header.yaml:应用于service的header

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
      annotations:
        # 定义header
        nginx.ingress.kubernetes.io/configuration-snippet: |
          more_set_headers "Request-Id: $req_id";
      name: tomcat-demo # service名称
      namespace: default # namespaces
    spec:
      rules:
      - host: tomcat.chcks.com
        http:
          paths:
          - backend:
              serviceName: tomcat-demo
              servicePort: 80
            path: /

template模板配置

文档地址:https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/custom-template/

1
2
3
4
5
6
7
8
9
10
# 到nginx-ingress-controller的pod拷贝nginx.tmpl
# 查看nginx-ingress-controller所在的服务器
kubectl get pod -n ingress-nginx -o wide
# 到所在服务器拷贝文件
docker ps | grep k8s_nginx-ingress-controller
docker cp 3d4006b68967:/etc/nginx/template/nginx.tmpl .
scp nginx.tmpl root@vmware-1:/opt/kubernetes/
# 回到vmware-1的服务器
kubectl create configmap nginx-template --from-file nginx.tmpl -n ingress-nginx
kubectl get configmaps -n ingress-nginx nginx-template -o yaml

应用配置文件到nginx-ingress-controller,修改nginx-ingress-controller.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
apiVersion: v1
items:
- apiVersion: extensions/v1beta1
  kind: DaemonSet
  metadata:
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
    name: nginx-ingress-controller
    namespace: ingress-nginx
  spec:
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    template:
      metadata:
        annotations:
          prometheus.io/port: "10254"
          prometheus.io/scrape: "true"
        creationTimestamp: null
        labels:
          app.kubernetes.io/name: ingress-nginx
          app.kubernetes.io/part-of: ingress-nginx
      spec:
        containers:
        - args:
          - /nginx-ingress-controller
          - --configmap=$(POD_NAMESPACE)/nginx-configuration
          - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
          - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
          - --publish-service=$(POD_NAMESPACE)/ingress-nginx
          - --annotations-prefix=nginx.ingress.kubernetes.io
          volumeMounts:  # 挂载到容器内部的存储卷配置
          - mountPath: /etc/nginx/template
            name: nginx-template-volume
            readOnly: true
          env:
          - name: POD_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /wait-shutdown
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          name: nginx-ingress-controller
          ports:
          - containerPort: 80
            hostPort: 80
            name: http
            protocol: TCP
          - containerPort: 443
            hostPort: 443
            name: https
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          resources: {}
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              add:
              - NET_BIND_SERVICE
              drop:
              - ALL
            runAsUser: 33
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
        dnsPolicy: ClusterFirst
        hostNetwork: true
        nodeSelector:
          app_gateway: ingress
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: nginx-ingress-serviceaccount
        serviceAccountName: nginx-ingress-serviceaccount
        terminationGracePeriodSeconds: 300
        volumes:       #pod上定义共享存储卷列表
        - name: nginx-template-volume
          configMap:
            name: nginx-template
            items:
            - key: nginx.tmpl
              path: nginx.tmpl
    templateGeneration: 1
    updateStrategy:
      rollingUpdate:
        maxUnavailable: 1
      type: RollingUpdate
  status:
    currentNumberScheduled: 1
    desiredNumberScheduled: 1
    numberAvailable: 1
    numberMisscheduled: 0
    numberReady: 1
    observedGeneration: 1
    updatedNumberScheduled: 1
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

如需修改模板文件则

1
kubectl edit configmaps -n ingress-nginx nginx-template

TSL/HTTPS

文档地址:https://kubernetes.github.io/ingress-nginx/user-guide/tls/#default-ssl-certificate

生成证书文件

1
2
3
$ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout chc.key -out chc.pem -subj "/CN=*.chcks.com/O=*.chcks.com"

kubectl create secret tls chc-tls --key chc.key --cert chc.pem -n ingress-nginx

修改配置文件nginx-ingress-controller.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
apiVersion: v1
items:
- apiVersion: extensions/v1beta1
  kind: DaemonSet
  metadata:
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
    name: nginx-ingress-controller
    namespace: ingress-nginx
  spec:
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    template:
      metadata:
        annotations:
          prometheus.io/port: "10254"
          prometheus.io/scrape: "true"
        creationTimestamp: null
        labels:
          app.kubernetes.io/name: ingress-nginx
          app.kubernetes.io/part-of: ingress-nginx
      spec:
        containers:
        - args:
          - /nginx-ingress-controller
          - --configmap=$(POD_NAMESPACE)/nginx-configuration
          - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
          - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
          - --publish-service=$(POD_NAMESPACE)/ingress-nginx
          - --annotations-prefix=nginx.ingress.kubernetes.io
          - --default-ssl-certificate=ingress-nginx/chc-tls #使用证书
          volumeMounts:  # 挂载到容器内部的存储卷配置
          - mountPath: /etc/nginx/template
            name: nginx-template-volume
            readOnly: true
          env:
          - name: POD_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /wait-shutdown
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          name: nginx-ingress-controller
          ports:
          - containerPort: 80
            hostPort: 80
            name: http
            protocol: TCP
          - containerPort: 443
            hostPort: 443
            name: https
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          resources: {}
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              add:
              - NET_BIND_SERVICE
              drop:
              - ALL
            runAsUser: 33
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
        dnsPolicy: ClusterFirst
        hostNetwork: true
        nodeSelector:
          app_gateway: ingress
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: nginx-ingress-serviceaccount
        serviceAccountName: nginx-ingress-serviceaccount
        terminationGracePeriodSeconds: 300
        volumes:       #pod上定义共享存储卷列表
        - name: nginx-template-volume
          configMap:
            name: nginx-template
            items:
            - key: nginx.tmpl
              path: nginx.tmpl
    templateGeneration: 1
    updateStrategy:
      rollingUpdate:
        maxUnavailable: 1
      type: RollingUpdate
  status:
    currentNumberScheduled: 1
    desiredNumberScheduled: 1
    numberAvailable: 1
    numberMisscheduled: 0
    numberReady: 1
    observedGeneration: 1
    updatedNumberScheduled: 1
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

配置域名证书web-https-ingress.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: tomcat-demo       # service名称
  namespace: default      # namespaces名称
spec:
  rules:
  - host: tomcat.chcks.com
    http:
      paths:
      - backend:
          serviceName: tomcat-demo
          servicePort: 80
        path: /
  tls:                    #使用tls
    - hosts:
      - tomcat.chcks.com
      secretName: chc-tls # secrets名称

使用配置文件

1
2
3
kubectl apply -f ingress-nginx-controller.yaml
kubectl apply -f web-https-ingress.yaml
# 访问:https://tomcat.chcks.com/

会话保持

web-session-ingress.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/affinity: cookie           #使用cookie
    nginx.ingress.kubernetes.io/session-cookie-hash: sha1  #获取哈希的方式
    nginx.ingress.kubernetes.io/session-cookie-name: chc #cookie名称
  name: tomcat-demo
  namespace: default
spec:
  rules:
  - host: tomcat.chcks.com
    http:
      paths:
      - backend:
          serviceName: tomcat-demo
          servicePort: 80
        path: /