Построение Full-Mesh VPN-сети с использованием fastd, tinc, VpnCloud и тестирование производительности

# Log warnings and errors to stderr
log level warn;

# Log informational messages to syslog
log to syslog level info;

# Hides IP addresses in log output
hide ip addresses no;

# Hides MAC addresses in log output
hide mac addresses no;

# Sets the mode of the interface; the default is TAP mode.
# In TAP mode, a single interface will be created for all peers
# in multi-TAP and TUN mode, each peer gets its own interface.
# mode multitap;
mode tap;

# Use the L2TP kernel implementation for the “null@l2tp” method.
# Enabling offloading allows for significantly higher throughput,
# as data packets don’t need to be copied between kernel and userspace.
# We do not use methods without encryption so setting this option to
# "yes" does not make any sense
offload l2tp no;

# Set the interface name
interface "mesh-vpn";

# If set to no, fastd will create peer-specific interfaces
# only as long as there’s an active session with the peer.
# Does not have an effect in TUN mode.
#persist interface no;
persist interface yes;

# Set the interface MTU for TAP mode with xsalsa20/aes128 over IPv4 with a base MTU of 1492 (PPPoE)
# (see MTU selection documentation)
# The basic overhead of a fastd packet in TUN mode over IPv4 is 28 bytes plus method-specific overhead
# Method “null” uses 1 additional header byte, “null@l2tp” 8 bytes, and all other methods 24 bytes
# TAP mode needs 14 bytes more than TUN mode
# Tunneling over IPv6 needs 20 bytes more than IPv4
# If your base MTU is 1500 and you want to use TUN mode over IPv4 with any crypto method
# Choose 1500 - 28 - 24 = 1448 bytes.
mtu 1434;

# Enables or disabled forwarding packets between peers.
forward yes;

# Support salsa2012+umac #and null methods, prefer salsa2012+umac
method "salsa2012+umac";

# Bind to a fixed port, IPv4 only
bind 0.0.0.0:10000;

# Sets the user to run fastd as.
user "nobody";

# Sets the group to run fastd as.
# Use group "nogroup" in Debian/Ubuntu and "nobody" on RHEL-based distros.
group "nobody";
#group "nogroup";

# By default, fastd switches to the configured user
# and/or drops its POSIX capabilities after the on-up command has been run
drop capabilities yes;

# Configures a shell command that is run after the interface is created,
# before the interface is destroyed, when a handshake is sent to make
# a new connection, when a new peer connection has been established,
# or after a peer connection has been lost
#on pre-up [ sync | async ] "<command>";
#on up [ sync | async ] "<command>";
#on down [ sync | async ] "<command>";
#on post-down [ sync | async ] "<command>";
#on connect [ sync | async ] "<command>";
#on establish [ sync | async ] "<command>";
#on disestablish [ sync | async ] "<command>";
#on up sync "/usr/local/bin/peer-fastd up virtsrvbr0";
on up "ip addr add 172.16.0.11/24 dev mesh-vpn";

# Secret key generated by `fastd --generate-key`
secret "f0a31725e3750e40c7664599359ef80dc209431c865ed3b56f94346c2009d96c";

# An inline peer configuration.
# peer "peer-01" { remote 198.51.100.11:10000; key "71fdf3236cd5e26a528449b5a9a7ab8c1d78382fa7c740685d4e717a67d21f2d"; }
peer "peer-02" { remote 192.0.2.12:10000; key "b5b9a0187324e9581c332f9701411ee32da2d0d8c875f2bc0e6d1a72b4241ee2"; }
peer "peer-03" { remote 203.0.113.13:10000; key "c7eeafa25a59b910f85dd570dc27dd6889cf3acdf3f1561200f754b377b9d693"; }
# Include peers from the directory 'peers'
#include peers from "peers";

------------------------------------------------------------
Client connecting to 192.0.2.12, TCP port 5001 with pid 13242
Write buffer size:  128 KByte
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[  3] local 198.51.100.11 port 38160 connected with 192.0.2.12 port 5001
[ ID] Interval        Transfer    Bandwidth       Write/Err  Rtry    Cwnd/RTT
[  3] 0.00-1800.00 sec  197.40 GBytes   942 Mbits/sec /0          3      462K/2543 us

peer-01>cat iperf_fastd_btest.txt

------------------------------------------------------------
Client connecting to 172.16.0.12, TCP port 5001 with pid 15233 (1 flows)
Write buffer size: 131072 Byte
TOS set to 0x0 (Nagle on)
TCP window size: 45.0 KByte (default)
------------------------------------------------------------
[  1] local 172.16.0.12%mesh-vpn port 42552 connected with 172.16.0.11 port 5001 (MSS=1382) (sock=3) (irtt/icwnd=475/13) (ct=0.55 ms) on 2022-03-03 07:04:56 (MSK)
[ ID] Interval        Transfer    Bandwidth       Write/Err  Rtry     Cwnd/RTT(var)        NetPwr
……………
[  1] 0.00-1800.03 sec   187 GBytes   894 Mbits/sec  1534056/0      68845      149K/1278(59) us  87406

peer-02>ping -c 120 -s -A 172.16.0.11

……
172.16.0.11 xmt/rcv/%loss = 120/120/0%, min/avg/max = 0.38/0.44/0.80

       1 targets
       1 alive
       0 unreachable
       0 unknown addresses

       0 timeouts (waiting for response)
     120 ICMP Echos sent
     120 ICMP Echo Replies received
       0 other ICMP received

 0.38 ms (min round trip time)
 0.44 ms (avg round trip time)
 0.80 ms (max round trip time)
      120.000 sec (elapsed real time)

sudo cat /etc/vpncloud/vpntest.net
---
device:
  type: tap
  name: mesh-vpn-%d
ip: 172.16.0.12/24
crypto:
  private-key: gLtF7RuP4Jbew55N1wGDr81TXgD10OeipRTjPtL28Hr
  public-key: J1UYpeLbR9Jq51jbsfPKfyfPCKPcgoTpEsecxgSgDIW
  trusted-keys:
    - cG5BMfIyDBZghrNlopOE8QT06aUXYoaXHFJAwZxM84j
    - BQK0a8clKrfiQGsPLDaByiT4ULyRZWbTnQcUYYJVXhZ
listen: 3210
peers:
  - 198.51.100.11:3210
  - 203.0.113.13:3210
peer-timeout: 600
mode: switch
switch-timeout: 600
# claims:
#  - 172.16.0.0/24
port-forwarding: false
user: nobody
group: nogroup

[Unit]
Description=VpnCloud network '%I'
After=network-online.target
Wants=network-online.target
PartOf=vpncloud.target
Documentation=man:vpncloud(1)

[Service]
Type=forking
ExecStart=/usr/bin/vpncloud --config /etc/vpncloud/%i.net --log-file /var/log/vpncloud-%i.log --stats-file /var/log/vpncloud-%i.stats --daemon --daemon --pid-file /run/vpncloud-%i.pid
PIDFile=/run/vpncloud-%i.pid
WorkingDirectory=/etc/vpncloud
RestartSec=5s
Restart=on-failure
TasksMax=10
MemoryMax=50M
PrivateTmp=yes
ProtectHome=yes
ProtectSystem=strict
ReadWritePaths=/var/log /run
CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_SETGID CAP_SETUID CAP_SYS_CHROOT
DeviceAllow=/dev/null rw
DeviceAllow=/dev/net/tun rw

[Install]
WantedBy=multi-user.target

cat iperf_vpncloud_btest.txt

>------------------------------------------------------------
Client connecting to 172.16.0.12, TCP port 5001 with pid 153600 (1 flows)
Write buffer size: 131072 Byte
TOS set to 0x0 (Nagle on)
TCP window size: 45.0 KByte (default)
------------------------------------------------------------
[  1] local 172.16.0.11%mesh-vpn-0 port 42620 connected with 172.16.0.12 port 5001 (MSS=1361) (sock=3) (irtt/icwnd=400/13) (ct=0.42 ms) on 2022-03-03 16:04:08 (MSK)
[ ID] Interval        Transfer    Bandwidth       Write/Err  Rtry     Cwnd/RTT(var)        NetPwr
[  1] 0.00-60.00 sec  6.23 GBytes   892 Mbits/sec  51056/0        300     1133K/10113(61) us  11029
[  1] 60.00-120.00 sec  6.25 GBytes   894 Mbits/sec  51162/0        252     1031K/9197(102) us  12152
[  1] 0.00-120.04 sec  12.5 GBytes   893 Mbits/sec  102219/0        552     1034K/10616(2769) us  10514

sudo cat /etc/tinc/vpntest/hosts/peer02

Address = 192.0.2.12
Subnet = 172.16.0.0/24
-----BEGIN RSA PUBLIC KEY-----
MIIBCgKCAQEAwrGjie2YhqFp+iEe57IHPMeUCILEeQbogbamQVV+0fQeaiqLFjcu
zKEu8FRAQL+e+bxBTZXH0oC0b6FzjYQYv46ag4PyMJUVZyOmonAhBavH74IC5hL1
mOadCS4GCJQiyKqPF9T+oXvUIUwmzUpzMYsSgetEYQYVzl1DH0b6L06Pc2OTuHEb
3LLF0cv1OuZFKDo7rgUxylySxd5WmS3Y1lr+8jOxyMNfJOuy7BgLwADakK6Xem46
J0NbniLBza+B2e/3A7XqVgwb5bbN1QLPubvzkHyNXYEAv17GoDCAwJGJNDUREv+P
WHc/6K+QhdAxRjzY2KA34EAFSujaVP6aKQIDAQAB
-----END RSA PUBLIC KEY-----

sudo cat /etc/tinc/vpntest/hosts/peer03

Address = 203.0.113.13
Subnet = 172.16.0.0/24
-----BEGIN RSA PUBLIC KEY-----
MIIBCgKCAQEAuT1Ddm3zDeGJVDorTnIJ0WoldNaLB318cje8fPb8Gsyu5IcEqQSr
lZJyGhci1aGXbeXox3Pz+VKxCG1LXf7lwhH1I4TdvWbncTT5JpR+3HzWN9Km9pJ4
Sjeco2GMx8urNpfgeKcJJ1/bZpo2oZUVRkeWa6pBtxQXpInzsIYIhxywtDz922nV
4NldkrjLMy0M14ZHQiT3LyWpbUlYz/Y514Rx5T/tRvWFPF9iXcwcaYkmmHMGMO7Z
kdJ4JixLfQGSVccNzT+MyHb3VN7y5mKWyhwfF7GHycmVyXJ4iPJs/RP9PPMdL28f
bpKyGCam9gWF07F05vHkdzV5liWtAjy+OQIDAQAB
-----END RSA PUBLIC KEY-----

peer-01>cat iperf_tinc_btest.txt

------------------------------------------------------------
Client connecting to 172.16.0.12, TCP port 5001 with pid 15407 (1 flows)
Write buffer size: 131072 Byte
TOS set to 0x0 (Nagle on)
TCP window size: 45.0 KByte (default)
------------------------------------------------------------
[  1] local 172.16.0.11%mesh-vpn port 42670 connected with 172.16.0.12 port 5001 (MSS=1385) (sock=3) (irtt/icwnd=474/13) (ct=0.61 ms) on 2022-03-03 18:42:45 (MSK)
[ ID] Interval        Transfer    Bandwidth       Write/Err  Rtry     Cwnd/RTT(var)        NetPwr
[  1] 0.00-60.00 sec  5.42 GBytes   776 Mbits/sec  44390/0       3209      489K/5152(68) us  18822
[  1] 60.00-120.00 sec  5.45 GBytes   780 Mbits/sec  44641/0       2692      403K/3807(116) us  25616
[  1] 0.00-120.05 sec  10.9 GBytes   778 Mbits/sec  89032/0       5901      409K/4249(132) us  22878