The evolution of container management

Posted on Views:
一個從 Docker 發展成 Kubernetes 的故事

本文以 世紀帝國2 中的時代為例，封建時代、城堡時代、帝王時代
為什麼沒有 黑暗時代 呢？因為會用 Docker 先贏一半

封建時代

只使用 Docker 在 instances 中運行服務並且利用 reverse proxy forward 到相應的 instance

Docker

在 instance 上用 Docker 運行 nginx

1
docker run -d -p 8080:80 --name=test-nginx nginx:alpine

Apache

在 ingress instance 上運行 Apache

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
# rewrite http to https
<VirtualHost *:80>
    ServerName ngx.seancheng.space

    # Logging
    LogLevel warn
    CustomLog "|/usr/sbin/rotatelogs -l /var/log/httpd/ngx.seancheng.space-access-%Y%m%d.log 604800" combined
    ErrorLog "|/usr/sbin/rotatelogs -l /var/log/httpd/ngx.seancheng.space-error-%Y%m%d.log 604800"

    RewriteEngine On
    RewriteCond %{HTTPS} off
    RewriteRule (.*) https://%{SERVER_NAME}$1 [R,L]
    RequestHeader set X-Forwarded-Proto "https"
    RequestHeader set X-Forwarded-Port "443"
</Virtualhost>

# forward https to nginx's instance
<VirtualHost *:443>
    ServerName ngx.seancheng.space

    # Logging
    LogLevel warn
    CustomLog "|/usr/sbin/rotatelogs -l /var/log/httpd/ngx.seancheng.space-ssl-access-%Y%m%d.log 604800" combined
    ErrorLog "|/usr/sbin/rotatelogs -l /var/log/httpd/ngx.seancheng.space-ssl-error-%Y%m%d.log 604800"

    # SSL Certification
    ## using cf origin wildcard certificate
    SSLEngine On
    SSLCertificateFile /root/cf/seancheng.space/origin.crt
    SSLCertificateKeyFile /root/cf/seancheng.space/private.key

    # Headers
    Header always set Strict-Transport-Security "max-age=63072000;includeSubdomains;"
    Header always set X-Frame-Options DENY
    Header set X-Content-Type-Options "nosniff"
    RequestHeader set X-Forwarded-Proto "https"
    RequestHeader set X-Forwarded-Port "443"

    AllowEncodedSlashes NoDecode
    <IfModule mod_headers.c>
        Header set X-XSS-Protection "1; mode=block"
    </IfModule>

    ProxyRequests Off
    <Proxy *>
        Order allow,deny
        Allow from all
    </Proxy>

    ProxyPreserveHost On
    ProxyPass / http://192.168.28.240:8080/
    ProxyPassReverse / http://192.168.28.240:8080/
</VirtualHost>

Cons

  1. container 不能擴容，因為 8080 port 已被佔用，雖然可以不指定 expose’s port，但是就會變得要用另外的方法知道 expose’s port
  2. reverse-proxy 需要明確知道 service 的 ip 和 port
  3. 當服務多的時候，就要各自設定 apache conf，很蠢

城堡時代

使用 Docker swarm 在 instances 上部署服務並透過 Traefik forward 到相應的容器

Docker swarm

使用 Docker swarm 部署服務

Init docker swarm

1
docker swarm init

Create a network for ingress

1
docker network create -d overlay endpoint

Create a nginx service

run nginx with docker swarm

1
docker stack deploy -c stack.yml test-nginx
Configuration
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# stack.yml
version: '3.8'

services:
  app:
    image: nginx:alpine
    networks:
      - endpoint
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s
    deploy:
      update_config:
        parallelism: 1
        delay: 10s
        failure_action: rollback
      labels:
        - "app=nginx"
        - "traefik.enable=true"
        - "traefik.http.routers.test-nginx.rule=Host(`ngx.seancheng.space`)"
        - "traefik.http.routers.test-nginx.tls=true"
        - "traefik.http.services.test-nginx.loadbalancer.server.port=80"

networks:
  endpoint:
    external: true

Traefik

run Traefik with docker swarm

1
docker stack deploy -c stack.yml traefik

Configuration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# stack.yml
version: "3"

services:
  app:
    image: traefik
    networks:
      - endpoint
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./traefik.yml:/etc/traefik/traefik.yml
      - ./certs:/certs:ro
    labels:
      - "app=traefik"
    deploy:
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.dashboard.rule=Host(`traefik.seancheng.space`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))"
        - "traefik.http.routers.dashboard.tls=true"
        - "traefik.http.routers.dashboard.service=api@internal"
        - "[email protected]=8080"

  whoami:
    image: containous/whoami
    networks:
      - endpoint
    deploy:
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.whoami.service=whoami@docker"
        - "traefik.http.services.whoami.loadbalancer.server.port=80"
        - "traefik.http.routers.whoami.rule=Host(`whoami`)"

networks:
  endpoint:
    external: true

Cons

  1. 如果使用單節點的 docker swarm 又會有延伸的問題，instance 死掉或要進行調整還是會造成 service 的 downtime

帝王時代

利用 Kubernetes 達到容器管理，就不專注在 nodes 上了

// todo: kubernetes
未完待續