2-6-3-flannel实现容器跨主机网络通信
1. flannel简介
flannel是coreos团队针对kubernetes设计的一个网络规划服务。简单来说 , 他的功能是让集群的不同节点主机创建的docker容器都具有全集群唯一的虚拟IP 地址。
在默认的docker配置中 , 每个节点上面的docker服务都会分别负责所在节点容器的分配 , 这样导致的一个问题是 , 不同节点上的容器可能获得相同的内外IP地址。并使这些容器之间能够通过IP地址相互找到 , 也就是相互ping通
flannel的设计目的就是为了集群中的所有节点重新规划IP地址的使用规则, 从而使不同节点上的容器获得“同属一个内网”且不重复的IP地址 , 并让属于不同节点上的容器能够直接通过内网IP通信
flannel实质上是一种"覆盖网络 ( overlay network ) ".即表示运行在一个网上的网(应用层网络) , 并不依靠IP来传递消息 , 而是采用一种映射机制 , 把IP地址和identifies(标识)做映射来资源定位 , 也就是将tcp数据包装在另一种网络包里面进行路由转发和通信 , 目前已经支持udp、vxlan 、aws vpc和GCE路由等数据转发方式。
2. 配置flannel
配置环境
主机配置
#node01配置 [root@centos7 ~]# hostnamectl set-hostname node01 [root@node01 ~]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.194.130 node01 etcd 192.168.194.146 node02 #node02配置 [root@centos7-2 ~]# hostnamectl set-hostname node02 [root@node02 ~]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.194.130 node01 etcd 192.168.194.146 node02安装软件
#node01配置 [root@node01 ~]# yum -y install etcd flannel #node02配置 [root@node02 ~]# yum -y install flannel配置
etcd[root@node01 ~]# cat /etc/etcd/etcd.conf #[Member] #ETCD_CORS="" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_WAL_DIR="" #ETCD_LISTEN_PEER_URLS="http://localhost:2380" ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379,http://0.0.0.0:4001" #修改内容 #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" ETCD_NAME="default" #ETCD_SNAPSHOT_COUNT="100000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" #ETCD_QUOTA_BACKEND_BYTES="0" #ETCD_MAX_REQUEST_BYTES="1572864" #ETCD_GRPC_KEEPALIVE_MIN_TIME="5s" #ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s" #ETCD_GRPC_KEEPALIVE_TIMEOUT="20s" # #[Clustering] #ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380" ETCD_ADVERTISE_CLIENT_URLS="http://etcd:2379,http://etcd:4001" #修改内容 #ETCD_DISCOVERY="" #ETCD_DISCOVERY_FALLBACK="proxy" #ETCD_DISCOVERY_PROXY="" #ETCD_DISCOVERY_SRV="" #ETCD_INITIAL_CLUSTER="default=http://localhost:2380" #ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" #ETCD_INITIAL_CLUSTER_STATE="new" #ETCD_STRICT_RECONFIG_CHECK="true" #ETCD_ENABLE_V2="true" # #[Proxy] #ETCD_PROXY="off" #ETCD_PROXY_FAILURE_WAIT="5000" #ETCD_PROXY_REFRESH_INTERVAL="30000" #ETCD_PROXY_DIAL_TIMEOUT="1000" #ETCD_PROXY_WRITE_TIMEOUT="5000" #ETCD_PROXY_READ_TIMEOUT="0" # #[Security] #ETCD_CERT_FILE="" #ETCD_KEY_FILE="" #ETCD_CLIENT_CERT_AUTH="false" #ETCD_TRUSTED_CA_FILE="" #ETCD_AUTO_TLS="false" #ETCD_PEER_CERT_FILE="" #ETCD_PEER_KEY_FILE="" #ETCD_PEER_CLIENT_CERT_AUTH="false" #ETCD_PEER_TRUSTED_CA_FILE="" #ETCD_PEER_AUTO_TLS="false" # #[Logging] #ETCD_DEBUG="false" #ETCD_LOG_PACKAGE_LEVELS="" #ETCD_LOG_OUTPUT="default" # #[Unsafe] #ETCD_FORCE_NEW_CLUSTER="false" # #[Version] #ETCD_VERSION="false" #ETCD_AUTO_COMPACTION_RETENTION="0" # #[Profiling] #ETCD_ENABLE_PPROF="false" #ETCD_METRICS="basic" # #[Auth] #ETCD_AUTH_TOKEN="simple" 开启并验证
etcd#开启etcd [root@node01 ~]# systemctl enable etcd --now Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service. [root@node01 ~]# ss -tnlp State Recv-Q Send-Q Local Address:Port Peer Address:Port LISTEN 0 128 *:22 *:* users:(("sshd",pid=1087,fd=3)) LISTEN 0 100 127.0.0.1:25 *:* users:(("master",pid=1253,fd=13)) LISTEN 0 128 127.0.0.1:2380 *:* users:(("etcd",pid=1907,fd=5)) LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1087,fd=4)) LISTEN 0 100 [::1]:25 [::]:* users:(("master",pid=1253,fd=14)) LISTEN 0 128 [::]:4001 [::]:* users:(("etcd",pid=1907,fd=7)) LISTEN 0 128 [::]:2379 [::]:* users:(("etcd",pid=1907,fd=6)) #测试etcd #测试etcd是否可以存储数据 [root@node01 ~]# etcdctl set testdir/testkey0 1000 1000 [root@node01 ~]# etcdctl get testdir/testkey0 1000 #测试etcd集群是否健康 [root@node01 ~]# etcdctl -C http://etcd:4001 cluster-health member 8e9e05c52164694d is healthy: got healthy result from http://etcd:2379 cluster is healthy [root@node01 ~]# etcdctl -C http://etcd:2379 cluster-health member 8e9e05c52164694d is healthy: got healthy result from http://etcd:2379 cluster is healthy 配置
flannel[root@node01 ~]# cat /etc/sysconfig/flanneld # Flanneld configuration options # etcd url location. Point this to the server where etcd runs FLANNEL_ETCD_ENDPOINTS="http://etcd:2379" #需要修改内容 # etcd config key. This is the configuration key that flannel queries # For address range assignment FLANNEL_ETCD_PREFIX="/atomic.io/network" #此处的值主要用于指定etcd中key的位置, 后续需要加入指定位置的key # Any additional options that you want to pass #FLANNEL_OPTIONS=""在
etcd中设置flannel需要分配的网段[root@node01 ~]# etcdctl mk /atomic.io/network/config '{ "Network": "10.10.0.0/16" }' { "Network": "10.10.0.0/16" } [root@node01 ~]# etcdctl get /atomic.io/network/config { "Network": "10.10.0.0/16" } #此处将flannel的网段设置为10.10.0.0/16启动
flannel，同时检查etcd设置的flannel网段是否设置成功[root@node01 ~]# systemctl enable --now flanneld [root@node01 ~]# ip a show flannel0 9: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN group default qlen 500 link/none inet 10.10.80.0/16 scope global flannel0 valid_lft forever preferred_lft forever inet6 fe80::56bd:5875:237f:352b/64 scope link flags 800 valid_lft forever preferred_lft forever 将
docker与flannel结合查看
subnet信息, 主要目的是为了查看docker0需要修改的IP#这个文件中的内容是软件自动生成的 [root@node01 ~]# cat /run/flannel/subnet.env FLANNEL_NETWORK=10.10.0.0/16 FLANNEL_SUBNET=10.10.80.1/24 #这个地方便是后续docker0需要设置的IP FLANNEL_MTU=1472 #后续需要将mtu值设置到dockerd服务中 FLANNEL_IPMASQ=false配置
dockerdaemon,并重启docker[root@node01 ~]# cat /etc/docker/daemon.json { "insecure-registries" : [ "192.168.194.130", "192.168.194.146" ], "registry-mirrors" : [ "https://docker.mirrors.ustc.edu.cn" ], "bip" : "10.10.80.1/24", "mtu" : 1472 #这个地方与/run/flannel/subnet.env中的FLANNEL_MTU值对应 }验证容器网络
[root@node01 ~]# ip a show docker0 8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1472 qdisc noqueue state DOWN group default link/ether 02:42:a7:c2:55:f8 brd ff:ff:ff:ff:ff:ff inet 10.10.80.1/24 brd 10.10.80.255 scope global docker0 valid_lft forever preferred_lft forever [root@node01 ~]# ip a show flannel0 9: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN group default qlen 500 link/none inet 10.10.80.0/16 scope global flannel0 valid_lft forever preferred_lft forever inet6 fe80::56bd:5875:237f:352b/64 scope link flags 800 valid_lft forever preferred_lft forever运行一个ubuntu容器, 并检查容器中的IP信息
[root@centos7 ~]# docker run -it -d ubuntu /bin/bash a0aced2d0bf958acac1ba6d29c00ab5b09891f26d691ac3b8df978fd2d0ee794 [root@centos7 ~]# docker inspect a0 ...... "bridge": { "IPAMConfig": null, "Links": null, "Aliases": null, "MacAddress": "02:42:0a:0a:50:02", "NetworkID": "32cfd204be6b1c5b15d11d1073264290768284200805c6eecbdeda2dbf484a80", "EndpointID": "c3ebcc6dd68ec6c8a7d6606b85e10912e730220e090d53a97082fd7383a7442f", "Gateway": "10.10.80.1", "IPAddress": "10.10.80.2", "IPPrefixLen": 24, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "DriverOpts": null, "DNSNames": null } } } } ] #可以看到新容器的IP为10.10.80.2
配置节点2中的
flannel, 配置方式与节点1一样[root@node02 ~]# cat /etc/sysconfig/flanneld # Flanneld configuration options # etcd url location. Point this to the server where etcd runs FLANNEL_ETCD_ENDPOINTS="http://etcd:2379" # etcd config key. This is the configuration key that flannel queries # For address range assignment FLANNEL_ETCD_PREFIX="/atomic.io/network" # Any additional options that you want to pass #FLANNEL_OPTIONS="" [root@node02 ~]# systemctl --now enable flanneld Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service. Created symlink from /etc/systemd/system/docker.service.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service. [root@node02 ~]# ip a show flannel0 9: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN group default qlen 500 link/none inet 10.10.18.0/16 scope global flannel0 valid_lft forever preferred_lft forever inet6 fe80::b225:3d7c:df1d:757/64 scope link flags 800 valid_lft forever preferred_lft forever配置节点
2中的docker服务查看
subnet信息[root@node02 ~]# cat /run/flannel/subnet.env FLANNEL_NETWORK=10.10.0.0/16 FLANNEL_SUBNET=10.10.18.1/24 FLANNEL_MTU=1472 FLANNEL_IPMASQ=false修改
dockerdaemon文件, 然后重启docker[root@node02 ~]# cat /etc/docker/daemon.json { "insecure-registries" : [ "192.168.194.130", "192.168.194.146" ], "bip" : "10.10.18.1/24", "mtu" : 1472 } [root@node02 ~]# systemctl restart docker [root@node02 ~]# ip a show docker0 8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1472 qdisc noqueue state DOWN group default link/ether 02:42:02:82:0c:1d brd ff:ff:ff:ff:ff:ff inet 10.10.18.1/24 brd 10.10.18.255 scope global docker0 valid_lft forever preferred_lft forever
同时修改两个主机的iptables
#node01配置 [root@node01 ~]#iptables -A FORWARD -s 10.10.0.0/16 -j ACCEPT #node02配置 [root@node02 ~]#iptables -A FORWARD -s 10.10.0.0/16 -j ACCEPT测试
[root@node01 ~]# docker exec -it a0 ping -c 4 10.10.18.2 PING 10.10.18.2 (10.10.18.2) 56(84) bytes of data. 64 bytes from 10.10.18.2: icmp_seq=1 ttl=60 time=0.801 ms 64 bytes from 10.10.18.2: icmp_seq=2 ttl=60 time=0.582 ms 64 bytes from 10.10.18.2: icmp_seq=3 ttl=60 time=0.513 ms 64 bytes from 10.10.18.2: icmp_seq=4 ttl=60 time=0.543 ms --- 10.10.18.2 ping statistics --- 4 packets transmitted, 4 received, 0% packet loss, time 3002ms rtt min/avg/max/mdev = 0.513/0.609/0.801/0.113 ms