跨云部署docker swarm必须使用大于19.03的docker版本

组建跨云服务器的局域网

安装openvpn服务器

  1. 使用GitHub上的一键安装脚本；地址：https://github.com/Nyr/openvpn-install

  2. 配置服务器的配置文件

    • 服务器配置：

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      23
      24
      25
      26
      27
      28
      29
      30
      31
      32
      33
      34
      35
      36
      37
      38
      39
      40
      41
      42
      43
      44
      45
      46
      47
      48
      49
      50
      51
      52
      53
      54
      55
       vim /etc/openvpn/server/server.conf
       
       ;local 172.17.0.10
       port 1194
       proto udp
       dev tun
       ca ca.crt
       cert server.crt
       key server.key
       dh dh.pem
       auth SHA512
       tls-crypt tc.key
       topology subnet
       server 10.8.0.0 255.255.255.0
       push "route 0.0.0.0 0.0.0.0"
       ;push "redirect-gateway def1 bypass-dhcp"
       ifconfig-pool-persist ipp.txt
       ;push "dhcp-option DNS 183.60.83.19"
       ;push "dhcp-option DNS 183.60.82.98"
       keepalive 10 120
       cipher AES-256-CBC
       user nobody
       group nobody
       persist-key
       persist-tun
       status openvpn-status.log
       verb 3
       crl-verify crl.pem
       explicit-exit-notify
       
       ## 主要修改内容：
       1. 增加`push "route 0.0.0.0 0.0.0.0"`
       2. 注释`push "redirect-gateway def1 bypass-dhcp"`,`push "dhcp-option DNS 183.60.83.19"`和`push "dhcp-option DNS 183.60.82.98"`
       
       ## 注意：windows使用openvpn gui时会造成只能访问局域网，不能访问外网；解决方式如下
       原配置：
       client
       dev tun
       proto udp
       remote 49.234.201.19 1194
       resolv-retry infinite
       nobind
       persist-key
       persist-tun
       remote-cert-tls server
       auth SHA512
       cipher AES-256-CBC
       ignore-unknown-option block-outside-dns
       block-outside-dns
       verb 3
       修改内容：
       删除`block-outside-dns`
       windows配置删除block-outside-dns这一行，就行了
      因为这个设置， OpenVPN 会添加 Windows 防火墙记录
       参考网址：https://www.v2ex.com/t/521393#reply19

    • 指定客户端ip

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      已有配置：ifconfig-pool-persist ipp.txt
      修改ipp.txt的文件内容，指定用户的ip


      ## 网上查到的方法
      1、在server.conf文件中增加客户端配目录，ccd可以任意指定：
      	client-config-dir ccd
       
      2、进入ccd目录后，用客户名（就是common name）建立文件。
      例如：客户名：litifeng  ip：10.8.0.6
      则名为litifeng的文件中，写入下面代码：
      	ifconfig-push 10.8.0.6 255.255.255.0

  • 重启服务器的openvpn服务

    1
    2
    systemctl restart openvpn-server@server.service
    systemctl status openvpn-server@server.service

客户端安装vpn

1
2
3
4
5
6
7
yum -y install epel-release
yum -y install openvpn

启动vpn,client.ovpn为服务器生成的文件
openvpn --daemon --cd /etc/openvpn/client --config client.ovpn --log-append /var/log/openvpn.log
查看log
tail -f /var/log/openvpn.log

初始化docker swarm

跨云部署docker swarm时routingMesh等都无法使用，因为docker swarm需要使用4789的udp端口；而部分云服务商将4789端口自用了，所以在19.03版本之前的docker是无法使用docker swarm服务的

  1. 开放iptables端口，所有节点全部开放

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    # docker
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 2376 -j ACCEPT
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 2377 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 4789 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 5789 -j ACCEPT
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 7946 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 7946 -j ACCEPT
    # openvpn
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 1194 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 1194 -j ACCEPT

  1. 初始化docker swarm

    1
    2
    3
    4
    5
    6
    7
    8
    9
    docker swarm init --advertise-addr 10.8.0.1:2377 --listen-addr 10.8.0.1:2377  --data-path-port 5789

    ## 参数说明
    --advertise-addr 指定其他节点用来连接到当前管理节点的 IP 和端口。这一属性是可选的，当节点上有多个 IP 时，可以用于指定使用哪个IP。此外，还可以用于指定一个节点上没有的 IP，比如一个负载均衡的 IP。

    --listen-addr 指定用于承载 Swarm 流量的 IP 和端口。其设置通常与 --advertise-addr 相匹配，但是当节点上有多个 IP 的时候，可用于指定具体某个 IP。并且，如果 --advertise-addr 设置了一个远程 IP 地址（如负载均衡的IP地址），该属性也是需要设置的。建议执行命令时总是使用这两个属性来指定具体 IP 和端口。

    --data-path-port
    该标志允许您配置用于数据路径通信的UDP端口号。提供的端口号必须在1024-49151范围内。如果未设置此标志或将其设置为0，则使用默认端口号4789。数据路径端口只能在初始化群集时配置，并且适用于加入群集的所有节点。以下示例初始化一个新的Swarm，并将数据路径端口配置为UDP端口5789。

  1. 加入docker swarm

    1
    2
    3
    ## 获取加入集群的命令
    docker swarm join-token worker      //获取工作节点的token
    docker swarm join-token manager     //获取管理节点的token

  1. 创建overlay网络

    1
    2
    3
    4
    5
    6
    7
    8
    docker network create \
    --driver overlay \
    --attachable \
    server_net

    // 如果创建的ingress网络未指定--attachable参数，则ingress网络只允许服务连接到overlay网络中
    // 想要连接单一容器到overlay网络，就必须在创建overlay网络时添加--attachable参数，
    // 并且单一容器并不是暴露所有端口到overlay网络中，需要自己发布

  1. 测试

    1
    2
    3
    docker service create --name web --network server_net --publish 80:80 --replicas 2 httpd

    docker service scale web=1

安装kong

  1. 安装postgres

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    # 生成随机密码
    openssl rand -base64 20 >> postgres_password.txt

    # 使用docker-compose
    version: "3.3"
    services:
      postgres:
        container_name: "postgres"
        image: "postgres:9.6"
        restart: "always"
        environment:
          POSTGRES_USER: kong
          POSTGRES_DB: kong
          POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
        secrets:
          - postgres_password
        volumes:
          - "/data/docker/postgres/data:/var/lib/postgresql/data"
        ports:
          - "5432:5432"
    networks:
      default:
        external:
          name: server_net
    secrets:
      postgres_password:
        file: ./postgres_password.txt




    # 直接运行
    docker run -d --restart always --name postgres \
                   --network=server_net \
                   -p 5432:5432 \
                   -e POSTGRES_USER=kong \
                   -e POSTGRES_DB=kong \
                   -e POSTGRES_PASSWORD=123456 \
                   -v /data/docker/postgres/data:/var/lib/postgresql/data \
                   postgres:9.6
                   
                   
    # 创建用户
    create user harbor with password '123456';
    # 创建数据库
    create database harbor owner harbor;
    # 将数据库的权限全部赋予某个用户
    grant all on database harbor to harbor;

  1. 初始化及更新数据库

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    docker run --rm \
         --network=server_net \
         -e "KONG_DATABASE=postgres" \
         -e "KONG_PG_HOST=postgres" \
         -e "KONG_PG_PASSWORD=123456" \
         kong:latest kong migrations bootstrap
         
    docker run --rm \
         --network=server_net \
         -e "KONG_DATABASE=postgres" \
         -e "KONG_PG_HOST=postgres" \
         -e "KONG_PG_PASSWORD=123456" \
         kong:latest kong migrations up

  1. 设置数据库主机label

    1
    docker node update --label-add kong=master [主机名称]

  1. 创建kong服务

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    # 使用docker-compose
    version: "3.3"
    services:
      kong:
        image: "kong:latest"
        environment:
          KONG_DATABASE: postgres
          KONG_PG_HOST: postgres
          KONG_PG_PASSWORD_FILE: /run/secrets/postgres_password
          KONG_PROXY_ACCESS_LOG: /dev/stdout
          KONG_ADMIN_ACCESS_LOG: /dev/stdout
          KONG_PROXY_ERROR_LOG: /dev/stderr
          KONG_ADMIN_ERROR_LOG: /dev/stderr
          KONG_ADMIN_LISTEN: "0.0.0.0:8001, 0.0.0.0:8444 ssl"
        secrets:
          - postgres_password
        ports:
          - "8000:8000"
          - "8001:8001"
          - "8443:8443"
        deploy:
          mode: replicated
          replicas: 1
      konga:
        image: pantsel/konga
        environment:
          DB_ADAPTER: mysql
          DB_HOST: mysql_master
          DB_PORT: 3306
          DB_USER: konga
          DB_PASSWORD: 123456
          DB_DATABASE: konga
        deploy:
          mode: replicated
          replicas: 1
    networks:
      default:
        external:
          name: server_net
    secrets:
      postgres_password:
        file: ./postgres_password.txt


    # 直接运行
    docker service create --name kong \
         --network=server_net \
         -e "KONG_DATABASE=postgres" \
         -e "KONG_PG_HOST=postgres" \
         -e "KONG_PG_PASSWORD=123456" \
         -e "KONG_PROXY_ACCESS_LOG=/dev/stdout" \
         -e "KONG_ADMIN_ACCESS_LOG=/dev/stdout" \
         -e "KONG_PROXY_ERROR_LOG=/dev/stderr" \
         -e "KONG_ADMIN_ERROR_LOG=/dev/stderr" \
         -e "KONG_ADMIN_LISTEN=0.0.0.0:8001, 0.0.0.0:8444 ssl" \
         -p 8000:8000 \
         -p 8001:8001 \
         -p 8443:8443 \
         kong:latest
         
         
    # 创建konga服务和路由
    curl http://127.0.0.1:8001/services -X POST -d 'name=konga&host=konga&port=1337'
    curl http://127.0.0.1:8001/services/konga/routes -X POST -d 'hosts[]=konga.xupengfei.net'

  2. 安装mysql

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    # 生成随机密码
    openssl rand -base64 20 >> mysql_password.txt
    openssl rand -base64 20 >> mysql_konga_password.txt

    # 使用docker-compose
    version: "3.3"
    services:
      mysql:
        container_name: "mysql_master"
        image: "mysql:5.7"
        restart: "always"
        environment:
          MYSQL_ROOT_PASSWORD_FILE: /run/secrets/mysql_password
        secrets:
          - mysql_password
        volumes:
          - "/data/docker/mysql/conf:/etc/mysql/conf.d"
          - "/data/docker/mysql/data:/var/lib/mysql"
        ports:
          - "3306:3306"
    networks:
      default:
        external:
          name: server_net
    secrets:
      mysql_password:
        file: ./mysql_password.txt
        
        
    # 创建新用户
    create user konga identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS konga DEFAULT CHARSET utf8mb4 COLLATE utf8mb4_general_ci;
    # 查看用户权限
    show grants for "konga"@"%";
    # 取消用户权限
    revoke all on *.* from "konga"@"%";
    # 授权
    grant all privileges on konga.* to konga@'%' identified by '123456';
    flush privileges;

  3. 安装redis

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    version: "3.3"
    services:
      redis:
        container_name: "redis"
        image: "redis"
        restart: "always"
        volumes:
          - "/data/docker/redis/data:/data"
        ports:
          - "6379:6379"
    networks:
      default:
        external:
          name: server_net

    docker run -d -p 6379:6379 --restart always --network server_net --name redis -v /data/docker/redis/data:/data redis

  1. 创建konga管理面板

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    # 使用docker-compose
    version: "3.3"
    services:
      konga:
        image: pantsel/konga
        environment:
          DB_ADAPTER: mysql
          DB_HOST: mysql_master
          DB_PORT: 3306
          DB_USER: konga
          DB_PASSWORD: 123456
          DB_DATABASE: konga
        deploy:
          mode: replicated
          replicas: 1
    networks:
      default:
        external:
          name: server_net

    # 直接运行
    docker service create --network server_net --name konga pantsel/konga

  1. 安装gitlab

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    # 使用docker-compose
    version: "3.3"
    services:
      konga:
        container_name: "gitlab"
        image: gitlab/gitlab-ce
        restart: "always"
        volumes:
          - "/data/docker/gitlab/config:/etc/gitlab"
          - "/data/docker/gitlab/logs:/var/log/gitlab"
          - "/data/docker/gitlab/data:/var/opt/gitlab"
        ports:
          - "9922:22"
    networks:
      default:
        external:
          name: server_net


    # 直接运行
    docker run -d \
     -p 9980:80 \
     -p 9922:22 \
     -v /data/docker/gitlab/config:/etc/gitlab  \
     -v /data/docker/gitlab/logs:/var/log/gitlab \
     -v /data/docker/gitlab/data:/var/opt/gitlab \
     --restart always \
     --network server_net \
     --name gitlab \
     gitlab/gitlab-ce
     
    ## 修改配置文件
    vim /data/docker/gitlab/config/gitlab.rb
    external_url 'http://gitlab.xupengfei.net'
    gitlab_rails['gitlab_ssh_host'] = 'gitlab.xupengfei.net'
    gitlab_rails['gitlab_shell_ssh_port'] = 9922
    ## 重启gitlab
    docker restart gitlab

  1. 安装邮件服务器

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28

    # 设置服务器主机名
    hostnamectl set-hostname mail.xupengfei.net

    docker pull centos:7.4.1708
    # 运行新容器并设置主机名,开放端口等
    docker run -itd --name ewomail -h mail.xupengfei.net --privileged=true \
      -p 25:25 \
      -p 110:110 \
      -p 143:143 \
      -p 465:465 \
      -p 587:587 \
      -p 993:993 \
      -p 995:995  \
      -p 13000:8000 \
      -p 13010:8010 \
      -p 13020:8020 \
      --restart always \
      -v /data/docker/ewomail/data:/ewomail centos:7.4.1708 init /bin/bash
    # 进入容器
    docker exec -it ewomail /bin/bash

    yum clean all && rm -rf /var/cache/yum && yum update -y && yum -y install git && cd /root && git clone https://gitee.com/laowu5/EwoMail.git && cd /root/EwoMail/install  && chmod +x start.sh && sh start.sh xupengfei.net
     
    # 获取DKIM（防止被判定为垃圾邮件）
    amavisd -c /etc/amavisd/amavisd.conf showkeys
    # 容器打包成镜像
    docker commit ewomail ewomail

  1. 安装Jenkins

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    # 使用docker-compose
    version: "3.3"
    services:
      jenkins:
        container_name: "jenkins"
        image: jenkins
        restart: "always"
        user: root
        privileged: true
        volumes:
          - "/data/docker/jenkins/data:/var/jenkins_home"
          - "/var/run/docker.sock:/var/run/docker.sock"
          - "/usr/bin/docker:/usr/bin/docker"
          - "/etc/localtime:/etc/localtime"
        ports:
          - "58080:8080"
          - "50000:50000"
    networks:
      default:
        external:
          name: server_net

  1. 安装gogs

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    # 使用docker-compose
    version: "3.3"
    services:
      gogs:
        container_name: "gogs"
        image: gogs/gogs
        restart: "always"
        volumes:
          - "/data/docker/gogs/data:/data"
        ports:
          - "10022:22"
          - "10080:3000"
    networks:
      default:
        external:
          name: server_net

    # mysql 创建用户
    # 创建新用户
    create user gogs identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS gogs DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
    # 查看用户权限
    show grants for "gogs"@"%";
    # 取消用户权限
    revoke all on *.* from "gogs"@"%";
    # 授权
    grant all privileges on gogs.* to gogs@'%' identified by '123456';
    flush privileges;

  1. mysql，postgres添加用户

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    # mysql 
        
    # 创建新用户--konga
    create user konga identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS konga DEFAULT CHARSET utf8mb4 COLLATE utf8mb4_general_ci;
    # 查看用户权限
    show grants for "konga"@"%";
    # 取消用户权限
    revoke all on *.* from "konga"@"%";
    # 授权
    grant all privileges on konga.* to konga@'%' identified by '123456';
    flush privileges;

    # 创建新用户--gogs
    create user gogs identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS gogs DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
    # 查看用户权限
    show grants for "gogs"@"%";
    # 取消用户权限
    revoke all on *.* from "gogs"@"%";
    # 授权
    grant all privileges on gogs.* to gogs@'%' identified by '123456';
    flush privileges;

    # postgres

    # 创建用户--harbor
    create user harbor with password '123456';
    # 创建数据库
    create database harbor owner harbor;
    # 将数据库的权限全部赋予某个用户
    grant all on database harbor to harbor;

  1. harbor

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    # 使用nginx代理harbor
    # 1. 代理nginx增加`client_max_body_size 0;`取消限制上传文件的大小
    # 2. 修改harbor的nginx配置
    # 删除/注释掉common/config/nginx/nginx.conf中的proxy_set_header X-Forwarded-Proto $scheme;

    # 登录
    docker login hub.xupengfei.net -u xupengfei -p 123456
    # 打标签
    docker tag jenkins:proxy hub.xupengfei.net/library/jenkins
    # 推送
    docker push hub.xupengfei.net/library/jenkins