鱼网混合云

外观

ECS

约 690 字大约 2 分钟

2025-04-18

在本项目ECS的主要作用为互联网请求的入口，接受用户请求并通过VPN转发至私有化集群。

Nginx

Nginx作为本项目主要的转发请求和静态资源存储控件，以下为该项目ECS的Nginx主要配置。

# fishnet.top.conf
server {
    listen 80;
    server_name fishnet.top;

    access_log /var/log/nginx/fishnet.top:80.access.log;
    error_log /var/log/nginx/fishnet.top:80.error.log;
    return 301 https://$host$request_uri;
}

server {
    listen 443 ssl;
    server_name fishnet.top;

    ssl_certificate /etc/letsencrypt/live/fishnet.top/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/fishnet.top/privkey.pem;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers HIGH:!aNULL:!MD5;

    access_log /var/log/nginx/fishnet.top:443.access.log;
    error_log /var/log/nginx/fishnet.top:443.error.log;

    root /var/www/fishnet-top-doc;
    index index.html index.htm index.php;

    location / {
        try_files $uri $uri/ =404;
    }
}

# 私有集群VPN转发
upstream cluster_servers_http {
    server 192.168.100.2:31390 weight=3 max_fails=3 fail_timeout=15s;
    server 192.168.100.3:31390 weight=2 max_fails=3 fail_timeout=15s;
    server 192.168.100.4:31390 weight=1 max_fails=3 fail_timeout=15s;
}

server {
    listen 443 ssl;
    server_name rancher.fishnet.top longhorn.fishnet.top harbor.fishnet.top jenkins.fishnet.top kibana.fishnet.top prometheus.fishnet.top grafana.fishnet.top rabbitmq.fishnet.top;

    ssl_certificate /etc/letsencrypt/live/fishnet.top/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/fishnet.top/privkey.pem;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers HIGH:!aNULL:!MD5;

    access_log /var/log/nginx/cluster.access.log;
    error_log /var/log/nginx/cluster.error.log;

    client_max_body_size 0;
    location / {
        proxy_pass https://cluster_servers_http;
	    proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "Upgrade";
	    proxy_ssl_server_name on;
        proxy_ssl_name $host;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	    proxy_set_header X-Forwarded-Proto $scheme;
    }
}

server {
    listen 443 ssl;
    server_name admin.mall.fishnet.top;

    ssl_certificate /etc/letsencrypt/live/fishnet.top/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/fishnet.top/privkey.pem;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers HIGH:!aNULL:!MD5;

    access_log /var/log/nginx/admin.mall.fishnet.top.access.log;
    error_log /var/log/nginx/admin.mall.fishnet.top.error.log;

    root /var/www/fishnet-mall/admin;
    index index.html index.htm index.php;

    location / {
        try_files $uri $uri/ =404;
    }
}

server {
    listen 443 ssl;
    server_name app.mall.fishnet.top;

    ssl_certificate /etc/letsencrypt/live/fishnet.top/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/fishnet.top/privkey.pem;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers HIGH:!aNULL:!MD5;

    access_log /var/log/nginx/app.mall.fishnet.top.access.log;
    error_log /var/log/nginx/app.mall.fishnet.top.error.log;

    root /var/www/fishnet-mall/app;
    index index.html index.htm index.php;

    location / {
        try_files $uri $uri/ =404;
    }
}

server {
    listen 443 ssl;
    server_name api.admin.mall.fishnet.top;

    ssl_certificate /etc/letsencrypt/live/fishnet.top/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/fishnet.top/privkey.pem;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers HIGH:!aNULL:!MD5;

    access_log /var/log/nginx/api.admin.mall.fishnet.top.access.log;
    error_log /var/log/nginx/api.admin.mall.fishnet.top.error.log;

    root /var/www/fishnet-mall/admin;
    index index.html index.htm index.php;

    location / {
        proxy_pass https://cluster_servers_http;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "Upgrade";
        proxy_ssl_server_name on;
        proxy_ssl_name $host;
	    proxy_pass_request_headers on;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

server {
    listen 443 ssl;
    server_name api.portal.mall.fishnet.top;

    ssl_certificate /etc/letsencrypt/live/fishnet.top/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/fishnet.top/privkey.pem;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers HIGH:!aNULL:!MD5;

    access_log /var/log/nginx/api.portal.mall.fishnet.top.access.log;
    error_log /var/log/nginx/api.portal.mall.fishnet.top.error.log;

    root /var/www/fishnet-mall/admin;
    index index.html index.htm index.php;

    location / {
        proxy_pass https://cluster_servers_http;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "Upgrade";
        proxy_ssl_server_name on;
        proxy_ssl_name $host;
        proxy_pass_request_headers on;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

WireGuard（VPN）

本项目使用WireGuard作为VPN的实现，作为ECS与私有化集群的通信手段。

集群网络

网络类型地址服务器
私网172.29.247.225
192.168.100.1
ECS Ubuntu
私网192.168.31.128
192.168.100.2
私有Ubuntu（Master）
私网192.168.31.129
192.168.100.3
私有Ubuntu（Node1）
私网192.168.31.130
192.168.100.4
私有Ubuntu（Node2）

其中192.168.100.0/24属于WireGuard网络地址范围，192.168.31.0/24为私有化集群网络地址范围，172.29.247.225为阿里云ECS内网地址。

网络配置

ECS

# wg0.conf
[Interface]
PrivateKey = xxx
Address = 192.168.100.1/24
ListenPort = 51820

[Peer]
PublicKey = xxx
AllowedIPs = 192.168.100.2/32, 192.168.31.0/24
PersistentKeepalive = 25

[Peer]
PublicKey = xxx
AllowedIPs = 192.168.100.3/32
PersistentKeepalive = 25

[Peer]
PublicKey = xxx
AllowedIPs = 192.168.100.4/32
PersistentKeepalive = 25

ECS使用私有化集群内网IP访问

ip route add 192.168.31.0/24 dev wg0

Master

# wg0.conf
[Interface]
PrivateKey = xxx
Address = 192.168.100.2/24
ListenPort = 51820

[Peer]
PublicKey = xxx
Endpoint = fishnet.top:51820
AllowedIPs = 192.168.100.1/32
PersistentKeepalive = 25

Node1

# wg0.conf
[Interface]
PrivateKey = xxx
Address = 192.168.100.3/24
ListenPort = 51820

[Peer]
PublicKey = xxx
Endpoint = fishnet.top:51820
AllowedIPs = 192.168.100.1/32
PersistentKeepalive = 25

Node2

# wg0.conf
[Interface]
PrivateKey = xxx
Address = 192.168.100.3/24
ListenPort = 51820

[Peer]
PublicKey = xxx
Endpoint = fishnet.top:51820
AllowedIPs = 192.168.100.1/32
PersistentKeepalive = 25