论坛徽章:: 0

电梯直达

1楼 [收藏(0)] [报告]

发表于 2015-01-05 14:03 |只看该作者 |倒序浏览

20可用积分

lvs采用tun模式，vip和后端rs在不同的网段。
目前异常现象：client访问vip，lvs已经把包转到了rs，三次握手时rs回给client的syn ack client没有收到，不知道具体时什么原因导致的，初步怀疑rs的回包被网络设备丢弃，或者配置隧道时有参数不正常，烦知道的朋友能够帮忙，感谢！
服务器keepalived配置：

virtual_server 10.25.4.170 8080 {
delay_loop 6
lb_algo rr
lb_kind TUN
persistence_timeout 220
protocol TCP
   real_server 10.39.57.18 8080 {
   weight 100
   TCP_CHECK {
   connect_timeout 5
   nb_get_retry 3
   delay_before_retry 3
   connect_port 8080
      }
   }
}

[root@CNSZ023568 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=1048576)
Prot LocalAddress

ort Scheduler Flags
-> RemoteAddress

ort          Forward Weight ActiveConn InActConn
TCP  10.25.4.170:8080 rr persistent 220
  -> 10.39.57.18:8080          Tunnel  100 0       0
[root@CNSZ023568 ~]#

访问时lvs hash表状态：
[root@CNSZ023568 ~]# ipvsadm -lnc
IPVS connection entries
pro expire state    source          virtual          destination
TCP 00:55  SYN_RECV 10.13.52.107:38490 10.25.4.170:8080 10.39.57.18:8080
TCP 03:32  NONE       10.13.52.107:0    10.25.4.170:8080 10.39.57.18:8080
[root@CNSZ023568 ~]#

rs配置：
参数配置：

[root@BIG-ipTest ~]# ifconfig
eth0    Link encap:Ethernet  HWaddr 00:50:56:AA:2A:3D
      inet addr:10.39.57.18  Bcast:10.39.59.255  Mask:255.255.252.0
      inet6 addr: fe80::250:56ff:feaa:2a3d/64 Scope:Link
      UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
      RX packets:2179099 errors:0 dropped:0 overruns:0 frame:0
      TX packets:892718 errors:0 dropped:0 overruns:0 carrier:0
      collisions:0 txqueuelen:1000
      RX bytes:80776525 (77.0 MiB)  TX bytes:127428660 (121.5 MiB)

eth1    Link encap:Ethernet  HWaddr 00:50:56:AA:27:3F
      inet addr:10.39.60.230  Bcast:10.39.63.255  Mask:255.255.252.0
      inet6 addr: fe80::250:56ff:feaa:273f/64 Scope:Link
      UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
      RX packets:8154078 errors:0 dropped:0 overruns:0 frame:0
      TX packets:19910 errors:0 dropped:0 overruns:0 carrier:0
      collisions:0 txqueuelen:1000
      RX bytes:12046403 (11.4 MiB)  TX bytes:2016044 (1.9 MiB)

lo       Link encap:Local Loopback
      inet addr:127.0.0.1  Mask:255.0.0.0
      inet6 addr: ::1/128 Scope:Host
      UP LOOPBACK RUNNING  MTU:16436  Metric:1
      RX packets:243 errors:0 dropped:0 overruns:0 frame:0
      TX packets:243 errors:0 dropped:0 overruns:0 carrier:0
      collisions:0 txqueuelen:0
      RX bytes:14518 (14.1 KiB)  TX bytes:14518 (14.1 KiB)

tunl0    Link encap:IPIP Tunnel  HWaddr
      inet addr:10.25.4.170  Mask:255.255.255.255
      UP RUNNING NOARP  MTU:1480  Metric:1
      RX packets:233 errors:0 dropped:0 overruns:0 frame:0
      TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
      collisions:0 txqueuelen:0
      RX bytes:12000 (11.7 KiB)  TX bytes:0 (0.0 b)

[root@BIG-ipTest ~]#

[root@BIG-ipTest ~]# sysctl -a | grep arp_ignore
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.tunl0.arp_ignore = 1
[root@BIG-ipTest ~]# sysctl -a | grep arp_announce
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.tunl0.arp_announce = 2
[root@BIG-ipTest ~]# sysctl -a | grep rp_f
net.ipv4.conf.tunl0.rp_filter = 0
net.ipv4.conf.tunl0.arp_filter = 0

rs抓包分析
tcpdump -i any host 10.25.4.170 and port 8080
13:57:17.154956 IP 10.13.52.107.38490 > 10.25.4.170.webcache: Flags [S], seq 2282393001, win 8192, options [mss 1460,nop,wscale 2,nop,nop,sackOK], length 0
13:57:17.155035 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:18.354901 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:20.162180 IP 10.13.52.107.38490 > 10.25.4.170.webcache: Flags [S], seq 2282393001, win 8192, options [mss 1460,nop,wscale 2,nop,nop,sackOK], length 0
13:57:20.162204 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:20.354896 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:24.354919 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:26.168315 IP 10.13.52.107.38490 > 10.25.4.170.webcache: Flags [S], seq 2282393001, win 8192, options [mss 1460,nop,nop,sackOK], length 0
13:57:26.168364 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:32.354912 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0
13:57:48.354917 IP 10.25.4.170.webcache > 10.13.52.107.38490: Flags [S.], seq 3874366914, ack 2282393002, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 7], length 0

如上rs已经回包给client但是client没有收到ack包造成重传。