搜索文章 Ctrl K

2-6-2-Openvswitch实现容器跨主机网络互联

熊 0 阅读 容器

1. Open vSwitch介绍

Open vSwitch , 即Open Virtual Switch开放虚拟交换机 , 是在开源的Apache2.0许可下的产品级质量的多层虚拟交换机。Open vSwitch通过软件的方式形成交换机部件。

跟传统的物理交换机相比 , 虚拟交换机同样具备众多优点 , 一是配置更加灵活。一台普通的服务器可以配置出数十台甚至上百台虚拟交换机 , 且端口数目可以灵活选择。例如 , VMware的ESX一台服务器可以仿真出248台虚拟交换机 , 且每台交换机预设虚拟端口即可达56个; 二是成本更加低廉 , 通过虚拟交换往往可以获得昂贵的普通交换机才能达到的性能 , 例如微软的Hyper-V平台 , 虚拟机与虚拟交换机之间的联机速度轻易可达10Gbps。

官网: http://www.openvswitch.org/

2. 配置说明

环境准备

主机名

操作系统

宿主机IP

Docker0 IP

容器 IP

centos7

Centos 7.9

192.168.194.130/24

10.10.0.1/16

10.10.0.2/16

centos7-2

Centos 7.9

192.168.194.146/24

10.20.0.1/16

10.20.0.2/16

  1. 修改两台主机的docker0 IP

    #节点A
[root@centos7 ~]# cat /etc/docker/daemon.json 
{ 
    "insecure-registries" : [
      "192.168.194.130",
      "192.168.194.146"
     ],
     "registry-mirrors" : [
       "https://docker.mirrors.ustc.edu.cn"
     ],
     "bip" : "10.10.0.1/16"
     
}
[root@centos7 ~]#systemctl restart docker
[root@centos7 ~]# ip a show docker0
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:9d:64:73:e7 brd ff:ff:ff:ff:ff:ff
    inet 10.10.0.1/16 brd 10.10.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:9dff:fe64:73e7/64 scope link 
       valid_lft forever preferred_lft forever
​
#节点B
[root@centos7-2 ~]# cat /etc/docker/daemon.json 
{ 
    "insecure-registries" : [
      "192.168.194.130",
      "192.168.194.146"
     ],
    "bip" : "10.20.0.1/16"
}
[root@centos7-2 ~]# systemctl restart docker
[root@centos7-2 ~]# ip a show docker0
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:30:fb:53:b9 brd ff:ff:ff:ff:ff:ff
    inet 10.20.0.1/16 brd 10.20.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:30ff:fefb:53b9/64 scope link 
       valid_lft forever preferred_lft forever

  2. 安装openvswitch-switch和bridge-utils

    • 构建rpm包

      #安装依赖
[root@centos7 ~]#  yum install wget openssl-devel  python-sphinx gcc make python-devel openssl-devel kernel-devel graphviz kernel-debug-devel autoconf automake rpm-build redhat-rpm-config libtool python-twisted-core python-zope-interface PyQt4 desktop-file-utils libcap-ng-devel groff checkpolicy selinux-policy-devel gcc-c++ python-six unbound unbound-devel -y
​
#下载ovs压缩包
[root@centos7 ~]#  wget https://www.openvswitch.org/releases/openvswitch-2.11.0.tar.gz
​
#构建build目录, 并且构建rpm包
[root@centos7 ~]#mkdir -p ~/rpmbuild/SOURCES
[root@centos7 ~]#mv openvswitch-2.11.0.tar.gz /root/rpmbuild/SOURCES/
[root@centos7 ~]#cd /root/rpmbuild/SOURCES/
[root@centos7 SOURCES]# tar -xvzf openvswitch-2.11.0.tar.gz
​
#构建rpm包
[root@centos7 ~]# rpmbuild -bb --nocheck openvswitch-2.11.0/rhel/openvswitch-fedora.spec

    • 将构建的rpm包下载到测试的两个节点并且安装

      [root@centos7 ~]# yum localinstall /root/rpmbuild/RPMS/x86_64/openvswitch-2.11.0-1.el7.x86_64.rpm

    • 安装bridge-utils

      [root@centos7 ~]#yum install bridge-utils

    • 启动openvswitch

      [root@centos7-2 ~]# systemctl enable --now openvswitch
​

    • 查看软件安装版本

      [root@centos7-2 ~]# ovs-appctl --version
ovs-appctl (Open vSwitch) 2.11.0
[root@centos7-2 ~]# ovs-ofctl --version
ovs-ofctl (Open vSwitch) 2.11.0
OpenFlow versions 0x1:0x5
[root@centos7-2 ~]# ps -e | grep ovs
  8550 ?        00:00:00 ovsdb-server
  8620 ?        00:00:00 ovs-vswitchd
[root@centos7-2 ~]# brctl show 
bridge name bridge id       STP enabled interfaces
docker0     8000.024230fb53b9   no

  3. 创建br0并且激活

    #节点A配置
[root@centos7 ~]# ovs-vsctl add-br br0
[root@centos7 ~]# ip link set dev br0 up
[root@centos7 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:44:5c:5d brd ff:ff:ff:ff:ff:ff
    inet 192.168.194.130/24 brd 192.168.194.255 scope global noprefixroute dynamic ens33
       valid_lft 1681sec preferred_lft 1681sec
    inet6 fe80::a564:21d6:21e3:7e9c/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:9d:64:73:e7 brd ff:ff:ff:ff:ff:ff
    inet 10.10.0.1/16 brd 10.10.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:9dff:fe64:73e7/64 scope link 
       valid_lft forever preferred_lft forever
820: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether c2:54:53:f4:fd:ed brd ff:ff:ff:ff:ff:ff
821: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 76:82:92:cf:c4:47 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::7482:92ff:fecf:c447/64 scope link 
       valid_lft forever preferred_lft forever
​
​
#节点B配置
[root@centos7-2 ~]# ovs-vsctl add-br br0
[root@centos7-2 ~]# ip link set dev br0 up
[root@centos7-2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:f0:11:49 brd ff:ff:ff:ff:ff:ff
    inet 192.168.194.146/24 brd 192.168.194.255 scope global noprefixroute dynamic ens33
       valid_lft 1175sec preferred_lft 1175sec
    inet6 fe80::a165:fb01:7c53:3452/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:30:fb:53:b9 brd ff:ff:ff:ff:ff:ff
    inet 10.20.0.1/16 brd 10.20.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:30ff:fefb:53b9/64 scope link 
       valid_lft forever preferred_lft forever
75: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 16:70:8a:fb:34:f7 brd ff:ff:ff:ff:ff:ff
76: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether f2:89:d5:4f:1c:46 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::f089:d5ff:fe4f:1c46/64 scope link 
       valid_lft forever preferred_lft forever
​

  4. 创建gre隧道(remote_ip为peer宿主机ip)

    #节点A配置
[root@centos7 ~]# ovs-vsctl add-port br0 gre0 -- set Interface gre0 type=gre options:remote_ip=192.168.194.146
[root@centos7 ~]# ovs-vsctl list-ports br0
gre0
[root@centos7 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:44:5c:5d brd ff:ff:ff:ff:ff:ff
    inet 192.168.194.130/24 brd 192.168.194.255 scope global noprefixroute dynamic ens33
       valid_lft 1475sec preferred_lft 1475sec
    inet6 fe80::a564:21d6:21e3:7e9c/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:9d:64:73:e7 brd ff:ff:ff:ff:ff:ff
    inet 10.10.0.1/16 brd 10.10.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:9dff:fe64:73e7/64 scope link 
       valid_lft forever preferred_lft forever
820: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether c2:54:53:f4:fd:ed brd ff:ff:ff:ff:ff:ff
821: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 76:82:92:cf:c4:47 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::7482:92ff:fecf:c447/64 scope link 
       valid_lft forever preferred_lft forever
822: gre0@NONE: <NOARP> mtu 1476 qdisc noop state DOWN group default qlen 1000
    link/gre 0.0.0.0 brd 0.0.0.0
823: gretap0@NONE: <BROADCAST,MULTICAST> mtu 1462 qdisc noop state DOWN group default qlen 1000
    link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
824: gre_sys@NONE: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc pfifo_fast master ovs-system state UNKNOWN group default qlen 1000
    link/ether e6:50:2a:72:91:75 brd ff:ff:ff:ff:ff:ff
​
​
​
#节点B配置
[root@centos7-2 ~]# ovs-vsctl add-port br0 gre0 -- set Interface gre0 type=gre options:remote_ip=192.168.194.130
[root@centos7-2 ~]# ovs-vsctl list-ports br0
gre0
[root@centos7-2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:f0:11:49 brd ff:ff:ff:ff:ff:ff
    inet 192.168.194.146/24 brd 192.168.194.255 scope global noprefixroute dynamic ens33
       valid_lft 1739sec preferred_lft 1739sec
    inet6 fe80::a165:fb01:7c53:3452/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:30:fb:53:b9 brd ff:ff:ff:ff:ff:ff
    inet 10.20.0.1/16 brd 10.20.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:30ff:fefb:53b9/64 scope link 
       valid_lft forever preferred_lft forever
75: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 16:70:8a:fb:34:f7 brd ff:ff:ff:ff:ff:ff
76: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether f2:89:d5:4f:1c:46 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::f089:d5ff:fe4f:1c46/64 scope link 
       valid_lft forever preferred_lft forever
77: gre0@NONE: <NOARP> mtu 1476 qdisc noop state DOWN group default qlen 1000
    link/gre 0.0.0.0 brd 0.0.0.0
78: gretap0@NONE: <BROADCAST,MULTICAST> mtu 1462 qdisc noop state DOWN group default qlen 1000
    link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
79: gre_sys@NONE: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc pfifo_fast master ovs-system state UNKNOWN group default qlen 1000
    link/ether 92:eb:ac:a7:68:56 brd ff:ff:ff:ff:ff:ff
​

  5. 将br0作为接口加入docker0网桥

    #节点A配置
[root@centos7 ~]# brctl addif docker0 br0
[root@centos7 ~]# ovs-vsctl show 
f6199692-be9e-418e-be7a-b05307a1460b
    Bridge "br0"
        Port "br0"
            Interface "br0"
                type: internal
        Port "gre0"
            Interface "gre0"
                type: gre
                options: {remote_ip="192.168.194.146"}
    ovs_version: "2.11.0"
[root@centos7 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
docker0		8000.02429d6473e7	no		br0


#节点B配置
[root@centos7-2 ~]# brctl addif docker0 br0
[root@centos7-2 ~]# ovs-vsctl show 
c1d80e3c-a882-4de4-9221-fe3b89209e90
    Bridge "br0"
        Port "br0"
            Interface "br0"
                type: internal
        Port "gre0"
            Interface "gre0"
                type: gre
                options: {remote_ip="192.168.194.130"}
    ovs_version: "2.11.0"
[root@centos7-2 ~]# brctl show 
bridge name	bridge id		STP enabled	interfaces
docker0		8000.024230fb53b9	no		br0

  6. 添加静态路由(网段地址为peer Docker网段)

    #节点A配置
[root@centos7 ~]# ip route add 10.20.0.0/16 dev docker0
[root@centos7 ~]# ip r
default via 192.168.194.2 dev ens33 proto dhcp metric 100 
10.10.0.0/16 dev docker0 proto kernel scope link src 10.10.0.1 
10.20.0.0/16 dev docker0 scope link 
192.168.194.0/24 dev ens33 proto kernel scope link src 192.168.194.130 metric 100

#节点B配置
[root@centos7-2 ~]# ip route add 10.10.0.0/16 dev docker0
[root@centos7-2 ~]# ip r
default via 192.168.194.2 dev ens33 proto dhcp metric 100 
10.10.0.0/16 dev docker0 scope link 
10.20.0.0/16 dev docker0 proto kernel scope link src 10.20.0.1 
192.168.194.0/24 dev ens33 proto kernel scope link src 192.168.194.146 metric 100

  7. 测试跨主机连通性

    #节点A中ping节点B的虚拟机
[root@centos7 ~]# docker exec -it 59 bash
root@59bc9224de13:/# ping 10.20.0.2
PING 10.20.0.2 (10.20.0.2) 56(84) bytes of data.
64 bytes from 10.20.0.2: icmp_seq=1 ttl=63 time=1.36 ms
From 10.10.0.1 icmp_seq=2 Redirect Host(New nexthop: 2.0.20.10)
64 bytes from 10.20.0.2: icmp_seq=2 ttl=63 time=0.440 ms
^C
--- 10.20.0.2 ping statistics ---
2 packets transmitted, 2 received, +1 errors, 0% packet loss, time 1002ms
rtt min/avg/max/mdev = 0.440/0.900/1.361/0.460 ms
root@59bc9224de13:/# 

同理另外一个方向也可以ping过来

  8. 设置开机启动脚本

    #节点A配置
[root@centos7 ~]#cat > net.sh <<EOF   
#!/bin/bash
ip link set dev br0 up
brctl addif docker0 br0
ip route add 10.20.0.0/16 dev docker0
EOF
[root@ovs1 ~]#chmod +x net.sh

#节点B配置
[root@ovs2 ~]#cat > net.sh <<EOF   
#!/bin/bash
ip link set dev br0 up
brctl addif docker0 br0
ip route add 10.10.0.0/16 dev docker0
EOF
[root@ovs1 ~]#chmod +x net.sh