跨云部署docker swarm必须使用大于19.03的docker版本

组建跨云服务器的局域网

安装openvpn服务器

  1. 使用GitHub上的一键安装脚本;地址:https://github.com/Nyr/openvpn-install

  2. 配置服务器的配置文件

    • 服务器配置:

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      23
      24
      25
      26
      27
      28
      29
      30
      31
      32
      33
      34
      35
      36
      37
      38
      39
      40
      41
      42
      43
      44
      45
      46
      47
      48
      49
      50
      51
      52
      53
      54
      55
       vim /etc/openvpn/server/server.conf

      ;local 172.17.0.10
      port 1194
      proto udp
      dev tun
      ca ca.crt
      cert server.crt
      key server.key
      dh dh.pem
      auth SHA512
      tls-crypt tc.key
      topology subnet
      server 10.8.0.0 255.255.255.0
      push "route 0.0.0.0 0.0.0.0"
      ;push "redirect-gateway def1 bypass-dhcp"
      ifconfig-pool-persist ipp.txt
      ;push "dhcp-option DNS 183.60.83.19"
      ;push "dhcp-option DNS 183.60.82.98"
      keepalive 10 120
      cipher AES-256-CBC
      user nobody
      group nobody
      persist-key
      persist-tun
      status openvpn-status.log
      verb 3
      crl-verify crl.pem
      explicit-exit-notify

      ## 主要修改内容:
      1. 增加`push "route 0.0.0.0 0.0.0.0"`
      2. 注释`push "redirect-gateway def1 bypass-dhcp"`,`push "dhcp-option DNS 183.60.83.19"`和`push "dhcp-option DNS 183.60.82.98"`

      ## 注意:windows使用openvpn gui时会造成只能访问局域网,不能访问外网;解决方式如下
      原配置:
      client
      dev tun
      proto udp
      remote 49.234.201.19 1194
      resolv-retry infinite
      nobind
      persist-key
      persist-tun
      remote-cert-tls server
      auth SHA512
      cipher AES-256-CBC
      ignore-unknown-option block-outside-dns
      block-outside-dns
      verb 3
      修改内容:
      删除`block-outside-dns`
      windows配置删除block-outside-dns这一行,就行了
      因为这个设置, OpenVPN 会添加 Windows 防火墙记录
      参考网址:https://www.v2ex.com/t/521393#reply19
    • 指定客户端ip

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      已有配置:ifconfig-pool-persist ipp.txt
      修改ipp.txt的文件内容,指定用户的ip


      ## 网上查到的方法
      1、在server.conf文件中增加客户端配目录,ccd可以任意指定:
      client-config-dir ccd

      2、进入ccd目录后,用客户名(就是common name)建立文件。
      例如:客户名:litifeng ip:10.8.0.6
      则名为litifeng的文件中,写入下面代码:
      ifconfig-push 10.8.0.6 255.255.255.0
  • 重启服务器的openvpn服务

    1
    2
    systemctl restart openvpn-server@server.service
    systemctl status openvpn-server@server.service

客户端安装vpn

1
2
3
4
5
6
7
yum -y install epel-release
yum -y install openvpn

启动vpn,client.ovpn为服务器生成的文件
openvpn --daemon --cd /etc/openvpn/client --config client.ovpn --log-append /var/log/openvpn.log
查看log
tail -f /var/log/openvpn.log

初始化docker swarm

跨云部署docker swarm时routingMesh等都无法使用,因为docker swarm需要使用4789的udp端口;而部分云服务商将4789端口自用了,所以在19.03版本之前的docker是无法使用docker swarm服务的

  1. 开放iptables端口,所有节点全部开放

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    # docker
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 2376 -j ACCEPT
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 2377 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 4789 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 5789 -j ACCEPT
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 7946 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 7946 -j ACCEPT
    # openvpn
    -A INPUT -p tcp -m state --state NEW -m tcp --dport 1194 -j ACCEPT
    -A INPUT -p udp -m state --state NEW -m udp --dport 1194 -j ACCEPT
  1. 初始化docker swarm

    1
    2
    3
    4
    5
    6
    7
    8
    9
    docker swarm init --advertise-addr 10.8.0.1:2377 --listen-addr 10.8.0.1:2377  --data-path-port 5789

    ## 参数说明
    --advertise-addr 指定其他节点用来连接到当前管理节点的 IP 和端口。这一属性是可选的,当节点上有多个 IP 时,可以用于指定使用哪个IP。此外,还可以用于指定一个节点上没有的 IP,比如一个负载均衡的 IP。

    --listen-addr 指定用于承载 Swarm 流量的 IP 和端口。其设置通常与 --advertise-addr 相匹配,但是当节点上有多个 IP 的时候,可用于指定具体某个 IP。并且,如果 --advertise-addr 设置了一个远程 IP 地址(如负载均衡的IP地址),该属性也是需要设置的。建议执行命令时总是使用这两个属性来指定具体 IP 和端口。

    --data-path-port
    该标志允许您配置用于数据路径通信的UDP端口号。提供的端口号必须在1024-49151范围内。如果未设置此标志或将其设置为0,则使用默认端口号4789。数据路径端口只能在初始化群集时配置,并且适用于加入群集的所有节点。以下示例初始化一个新的Swarm,并将数据路径端口配置为UDP端口5789。
  1. 加入docker swarm

    1
    2
    3
    ## 获取加入集群的命令
    docker swarm join-token worker //获取工作节点的token
    docker swarm join-token manager //获取管理节点的token
  1. 创建overlay网络

    1
    2
    3
    4
    5
    6
    7
    8
    docker network create \
    --driver overlay \
    --attachable \
    server_net

    // 如果创建的ingress网络未指定--attachable参数,则ingress网络只允许服务连接到overlay网络中
    // 想要连接单一容器到overlay网络,就必须在创建overlay网络时添加--attachable参数,
    // 并且单一容器并不是暴露所有端口到overlay网络中,需要自己发布
  1. 测试

    1
    2
    3
    docker service create --name web --network server_net --publish 80:80 --replicas 2 httpd

    docker service scale web=1

安装kong

  1. 安装postgres

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    # 生成随机密码
    openssl rand -base64 20 >> postgres_password.txt

    # 使用docker-compose
    version: "3.3"
    services:
    postgres:
    container_name: "postgres"
    image: "postgres:9.6"
    restart: "always"
    environment:
    POSTGRES_USER: kong
    POSTGRES_DB: kong
    POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
    secrets:
    - postgres_password
    volumes:
    - "/data/docker/postgres/data:/var/lib/postgresql/data"
    ports:
    - "5432:5432"
    networks:
    default:
    external:
    name: server_net
    secrets:
    postgres_password:
    file: ./postgres_password.txt




    # 直接运行
    docker run -d --restart always --name postgres \
    --network=server_net \
    -p 5432:5432 \
    -e POSTGRES_USER=kong \
    -e POSTGRES_DB=kong \
    -e POSTGRES_PASSWORD=123456 \
    -v /data/docker/postgres/data:/var/lib/postgresql/data \
    postgres:9.6


    # 创建用户
    create user harbor with password '123456';
    # 创建数据库
    create database harbor owner harbor;
    # 将数据库的权限全部赋予某个用户
    grant all on database harbor to harbor;
  1. 初始化及更新数据库

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    docker run --rm \
    --network=server_net \
    -e "KONG_DATABASE=postgres" \
    -e "KONG_PG_HOST=postgres" \
    -e "KONG_PG_PASSWORD=123456" \
    kong:latest kong migrations bootstrap

    docker run --rm \
    --network=server_net \
    -e "KONG_DATABASE=postgres" \
    -e "KONG_PG_HOST=postgres" \
    -e "KONG_PG_PASSWORD=123456" \
    kong:latest kong migrations up
  1. 设置数据库主机label

    1
    docker node update --label-add kong=master [主机名称]
  1. 创建kong服务

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    # 使用docker-compose
    version: "3.3"
    services:
    kong:
    image: "kong:latest"
    environment:
    KONG_DATABASE: postgres
    KONG_PG_HOST: postgres
    KONG_PG_PASSWORD_FILE: /run/secrets/postgres_password
    KONG_PROXY_ACCESS_LOG: /dev/stdout
    KONG_ADMIN_ACCESS_LOG: /dev/stdout
    KONG_PROXY_ERROR_LOG: /dev/stderr
    KONG_ADMIN_ERROR_LOG: /dev/stderr
    KONG_ADMIN_LISTEN: "0.0.0.0:8001, 0.0.0.0:8444 ssl"
    secrets:
    - postgres_password
    ports:
    - "8000:8000"
    - "8001:8001"
    - "8443:8443"
    deploy:
    mode: replicated
    replicas: 1
    konga:
    image: pantsel/konga
    environment:
    DB_ADAPTER: mysql
    DB_HOST: mysql_master
    DB_PORT: 3306
    DB_USER: konga
    DB_PASSWORD: 123456
    DB_DATABASE: konga
    deploy:
    mode: replicated
    replicas: 1
    networks:
    default:
    external:
    name: server_net
    secrets:
    postgres_password:
    file: ./postgres_password.txt


    # 直接运行
    docker service create --name kong \
    --network=server_net \
    -e "KONG_DATABASE=postgres" \
    -e "KONG_PG_HOST=postgres" \
    -e "KONG_PG_PASSWORD=123456" \
    -e "KONG_PROXY_ACCESS_LOG=/dev/stdout" \
    -e "KONG_ADMIN_ACCESS_LOG=/dev/stdout" \
    -e "KONG_PROXY_ERROR_LOG=/dev/stderr" \
    -e "KONG_ADMIN_ERROR_LOG=/dev/stderr" \
    -e "KONG_ADMIN_LISTEN=0.0.0.0:8001, 0.0.0.0:8444 ssl" \
    -p 8000:8000 \
    -p 8001:8001 \
    -p 8443:8443 \
    kong:latest


    # 创建konga服务和路由
    curl http://127.0.0.1:8001/services -X POST -d 'name=konga&host=konga&port=1337'
    curl http://127.0.0.1:8001/services/konga/routes -X POST -d 'hosts[]=konga.xupengfei.net'
  2. 安装mysql

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    # 生成随机密码
    openssl rand -base64 20 >> mysql_password.txt
    openssl rand -base64 20 >> mysql_konga_password.txt

    # 使用docker-compose
    version: "3.3"
    services:
    mysql:
    container_name: "mysql_master"
    image: "mysql:5.7"
    restart: "always"
    environment:
    MYSQL_ROOT_PASSWORD_FILE: /run/secrets/mysql_password
    secrets:
    - mysql_password
    volumes:
    - "/data/docker/mysql/conf:/etc/mysql/conf.d"
    - "/data/docker/mysql/data:/var/lib/mysql"
    ports:
    - "3306:3306"
    networks:
    default:
    external:
    name: server_net
    secrets:
    mysql_password:
    file: ./mysql_password.txt


    # 创建新用户
    create user konga identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS konga DEFAULT CHARSET utf8mb4 COLLATE utf8mb4_general_ci;
    # 查看用户权限
    show grants for "konga"@"%";
    # 取消用户权限
    revoke all on *.* from "konga"@"%";
    # 授权
    grant all privileges on konga.* to konga@'%' identified by '123456';
    flush privileges;
  3. 安装redis

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    version: "3.3"
    services:
    redis:
    container_name: "redis"
    image: "redis"
    restart: "always"
    volumes:
    - "/data/docker/redis/data:/data"
    ports:
    - "6379:6379"
    networks:
    default:
    external:
    name: server_net

    docker run -d -p 6379:6379 --restart always --network server_net --name redis -v /data/docker/redis/data:/data redis
  1. 创建konga管理面板

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    # 使用docker-compose
    version: "3.3"
    services:
    konga:
    image: pantsel/konga
    environment:
    DB_ADAPTER: mysql
    DB_HOST: mysql_master
    DB_PORT: 3306
    DB_USER: konga
    DB_PASSWORD: 123456
    DB_DATABASE: konga
    deploy:
    mode: replicated
    replicas: 1
    networks:
    default:
    external:
    name: server_net

    # 直接运行
    docker service create --network server_net --name konga pantsel/konga
  1. 安装gitlab

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    # 使用docker-compose
    version: "3.3"
    services:
    konga:
    container_name: "gitlab"
    image: gitlab/gitlab-ce
    restart: "always"
    volumes:
    - "/data/docker/gitlab/config:/etc/gitlab"
    - "/data/docker/gitlab/logs:/var/log/gitlab"
    - "/data/docker/gitlab/data:/var/opt/gitlab"
    ports:
    - "9922:22"
    networks:
    default:
    external:
    name: server_net


    # 直接运行
    docker run -d \
    -p 9980:80 \
    -p 9922:22 \
    -v /data/docker/gitlab/config:/etc/gitlab \
    -v /data/docker/gitlab/logs:/var/log/gitlab \
    -v /data/docker/gitlab/data:/var/opt/gitlab \
    --restart always \
    --network server_net \
    --name gitlab \
    gitlab/gitlab-ce

    ## 修改配置文件
    vim /data/docker/gitlab/config/gitlab.rb
    external_url 'http://gitlab.xupengfei.net'
    gitlab_rails['gitlab_ssh_host'] = 'gitlab.xupengfei.net'
    gitlab_rails['gitlab_shell_ssh_port'] = 9922
    ## 重启gitlab
    docker restart gitlab
  1. 安装邮件服务器

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28

    # 设置服务器主机名
    hostnamectl set-hostname mail.xupengfei.net

    docker pull centos:7.4.1708
    # 运行新容器并设置主机名,开放端口等
    docker run -itd --name ewomail -h mail.xupengfei.net --privileged=true \
    -p 25:25 \
    -p 110:110 \
    -p 143:143 \
    -p 465:465 \
    -p 587:587 \
    -p 993:993 \
    -p 995:995 \
    -p 13000:8000 \
    -p 13010:8010 \
    -p 13020:8020 \
    --restart always \
    -v /data/docker/ewomail/data:/ewomail centos:7.4.1708 init /bin/bash
    # 进入容器
    docker exec -it ewomail /bin/bash

    yum clean all && rm -rf /var/cache/yum && yum update -y && yum -y install git && cd /root && git clone https://gitee.com/laowu5/EwoMail.git && cd /root/EwoMail/install && chmod +x start.sh && sh start.sh xupengfei.net

    # 获取DKIM(防止被判定为垃圾邮件)
    amavisd -c /etc/amavisd/amavisd.conf showkeys
    # 容器打包成镜像
    docker commit ewomail ewomail
  1. 安装Jenkins

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    # 使用docker-compose
    version: "3.3"
    services:
    jenkins:
    container_name: "jenkins"
    image: jenkins
    restart: "always"
    user: root
    privileged: true
    volumes:
    - "/data/docker/jenkins/data:/var/jenkins_home"
    - "/var/run/docker.sock:/var/run/docker.sock"
    - "/usr/bin/docker:/usr/bin/docker"
    - "/etc/localtime:/etc/localtime"
    ports:
    - "58080:8080"
    - "50000:50000"
    networks:
    default:
    external:
    name: server_net
  1. 安装gogs

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    # 使用docker-compose
    version: "3.3"
    services:
    gogs:
    container_name: "gogs"
    image: gogs/gogs
    restart: "always"
    volumes:
    - "/data/docker/gogs/data:/data"
    ports:
    - "10022:22"
    - "10080:3000"
    networks:
    default:
    external:
    name: server_net

    # mysql 创建用户
    # 创建新用户
    create user gogs identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS gogs DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
    # 查看用户权限
    show grants for "gogs"@"%";
    # 取消用户权限
    revoke all on *.* from "gogs"@"%";
    # 授权
    grant all privileges on gogs.* to gogs@'%' identified by '123456';
    flush privileges;
  1. mysql,postgres添加用户

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    # mysql 

    # 创建新用户--konga
    create user konga identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS konga DEFAULT CHARSET utf8mb4 COLLATE utf8mb4_general_ci;
    # 查看用户权限
    show grants for "konga"@"%";
    # 取消用户权限
    revoke all on *.* from "konga"@"%";
    # 授权
    grant all privileges on konga.* to konga@'%' identified by '123456';
    flush privileges;

    # 创建新用户--gogs
    create user gogs identified by '123456';
    # 创建数据库
    CREATE DATABASE IF NOT EXISTS gogs DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
    # 查看用户权限
    show grants for "gogs"@"%";
    # 取消用户权限
    revoke all on *.* from "gogs"@"%";
    # 授权
    grant all privileges on gogs.* to gogs@'%' identified by '123456';
    flush privileges;

    # postgres

    # 创建用户--harbor
    create user harbor with password '123456';
    # 创建数据库
    create database harbor owner harbor;
    # 将数据库的权限全部赋予某个用户
    grant all on database harbor to harbor;
  1. harbor

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    # 使用nginx代理harbor
    # 1. 代理nginx增加`client_max_body_size 0;`取消限制上传文件的大小
    # 2. 修改harbor的nginx配置
    # 删除/注释掉common/config/nginx/nginx.conf中的proxy_set_header X-Forwarded-Proto $scheme;

    # 登录
    docker login hub.xupengfei.net -u xupengfei -p 123456
    # 打标签
    docker tag jenkins:proxy hub.xupengfei.net/library/jenkins
    # 推送
    docker push hub.xupengfei.net/library/jenkins