linux下集群heartbeat启动不能剔除失效的realserver

xyfxh

linux下集群heartbeat启动不能剔除失效的realserver

ldirectord.cf


checktimeout=3
checkinterval=1
#fallback=127.0.0.1:80
autoreload=yes
logfile="/var/log/ldirectord.log"
quiescent=yes

virtual=192.168.1.118:80
        real=192.168.1.150:80 gate w 1
        real=192.168.1.128:80 gate w 2
        #real=192.168.6.6:80 gate
        fallback=127.0.0.1:80
        service=http
        checkport=80
        request="test.html"
        receive="this node is available"
        #virtualhost=some.domain.com.au
        scheduler=wrr
        persistent=600
        netmask=255.255.255.0
        protocol=tcp

haresources

node116 LVSSyncDaemonSwap::master IPaddr::192.168.1.118/32/eth0:118/192.168.1.255 lvsdr ldirectord

[root@node117 ha.d]# more /etc/init.d/lvsdr
#!/bin/sh
#create in 20060812 by ghb
# description: start LVS   of  Directorserver
VIP=192.168.1.118
RIP1=192.168.1.128
RIP2=192.168.1.150
#RIPn=192.168.0.128~254
GW=192.168.1.1

. /etc/rc.d/init.d/functions

case "$1" in
    start)
        echo " start LVS  of DirectorServer"
       # set the Virtual  IP Address
       /sbin/ifconfig eth0:0 $VIP broadcast $VIP netmask 255.255.255.255 up
       /sbin/route add -host $VIP dev eth0:0
       #Clear IPVS table
       /sbin/ipvsadm -C
      #set LVS
      /sbin/ipvsadm -A -t $VIP:80 -s rr
      /sbin/ipvsadm -a -t $VIP:80 -r $RIP1:80 -g
      /sbin/ipvsadm -a -t $VIP:80 -r $RIP2:80 -g
      #/sbin/ipvsadm -a -t $VIP:80 -r $RIP3:80 -g
      #Run LVS
      /sbin/ipvsadm
      #end

        ;;
    stop)
        echo "close LVS Directorserver"
        /sbin/ipvsadm -C
        ;;
    *)
        echo "Usage: $0 {start|stop}"
        exit 1
esac



ha-log启动日志

heartbeat[26065]: 2008/03/14_16:28:31 WARN: Logging daemon is disabled --enabling logging daemon is recommended
heartbeat[26065]: 2008/03/14_16:28:31 info: **************************
heartbeat[26065]: 2008/03/14_16:28:31 info: Configuration validated. Starting heartbeat 2.0.6
heartbeat[26066]: 2008/03/14_16:28:31 info: heartbeat: version 2.0.6
heartbeat[26066]: 2008/03/14_16:28:31 info: Heartbeat generation: 58
heartbeat[26066]: 2008/03/14_16:28:31 info: G_main_add_TriggerHandler: Added signal manual handler
heartbeat[26066]: 2008/03/14_16:28:31 info: G_main_add_TriggerHandler: Added signal manual handler
heartbeat[26066]: 2008/03/14_16:28:31 info: Removing /usr/local/var/run/heartbeat/rsctmp failed, recreating.
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: UDP Broadcast heartbeat started on port 694 (694) interface eth1
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: UDP Broadcast heartbeat closed on port 694 interface eth1 - Status: 1
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: ucast: write socket priority set to IPTOS_LOWDELAY on eth0
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: ucast: bound send socket to device: eth0
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: ucast: bound receive socket to device: eth0
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: ucast: started on port 694 interface eth0 to 192.168.1.116
heartbeat[26066]: 2008/03/14_16:28:31 info: glib: ping heartbeat started.
heartbeat[26066]: 2008/03/14_16:28:31 info: G_main_add_SignalHandler: Added signal handler for signal 17
heartbeat[26066]: 2008/03/14_16:28:31 info: Local status now set to: 'up'
heartbeat[26066]: 2008/03/14_16:28:32 info: Link 192.168.1.1:192.168.1.1 up.
heartbeat[26066]: 2008/03/14_16:28:32 info: Status update for node 192.168.1.1: status ping
heartbeat[26066]: 2008/03/14_16:28:32 info: Link node117:eth1 up.
heartbeat[26066]: 2008/03/14_16:30:32 WARN: node node116: is dead
heartbeat[26066]: 2008/03/14_16:30:32 info: Comm_now_up(): updating status to active
heartbeat[26066]: 2008/03/14_16:30:32 info: Local status now set to: 'active'
heartbeat[26066]: 2008/03/14_16:30:32 WARN: No STONITH device configured.
heartbeat[26066]: 2008/03/14_16:30:32 WARN: Shared disks are not protected.
heartbeat[26066]: 2008/03/14_16:30:32 info: Resources being acquired from node116.
harc[26083]:    2008/03/14_16:30:32 info: Running /usr/local/etc/ha.d/rc.d/status status
heartbeat[26084]: 2008/03/14_16:30:32 info: No local resources [/usr/local/lib/heartbeat/ResourceManager listkeys node117] to acquire.
mach_down[26101]:       2008/03/14_16:30:32 info: Taking over resource group IPaddr::192.168.1.118/32/eth0:118
ResourceManager[26123]: 2008/03/14_16:30:32 info: Acquiring resource group: node116 IPaddr::192.168.1.118/32/eth0:118 lvsdr ldirectord
IPaddr[26147]:  2008/03/14_16:30:32 INFO: IPaddr Resource is stopped
ResourceManager[26123]: 2008/03/14_16:30:32 info: Running /usr/local/etc/ha.d/resource.d/IPaddr 192.168.1.118/32/eth0:118 start
IPaddr[26349]:  2008/03/14_16:30:32 INFO: /sbin/ifconfig eth0:0 192.168.1.118 netmask 255.255.255.255 broadcast 192.168.1.118
IPaddr[26349]:  2008/03/14_16:30:32 INFO: Sending Gratuitous Arp for 192.168.1.118 on eth0:0 [eth0]
IPaddr[26349]:  2008/03/14_16:30:32 INFO: /usr/local/lib/heartbeat/send_arp -i 500 -r 10 -p /usr/local/var/run/heartbeat/rsctmp/send_arp/send_arp-192.168.1.118 eth0 192.168.1.118 auto 192.168.1.118 ffffffffffff
IPaddr[26267]:  2008/03/14_16:30:32 INFO: IPaddr Success
ResourceManager[26123]: 2008/03/14_16:30:32 info: Running /etc/init.d/lvsdr  start
ResourceManager[26123]: 2008/03/14_16:30:32 info: Running /etc/init.d/ldirectord  start
mach_down[26101]:       2008/03/14_16:30:32 info: /usr/local/lib/heartbeat/mach_down: nice_failback: foreign resources acquired
mach_down[26101]:       2008/03/14_16:30:32 info: mach_down takeover complete for node node116.
heartbeat[26066]: 2008/03/14_16:30:32 info: mach_down takeover complete.
heartbeat[26066]: 2008/03/14_16:30:32 info: Initial resource acquisition complete (mach_down)
heartbeat[26066]: 2008/03/14_16:30:42 info: Local Resource acquisition completed. (none)
heartbeat[26066]: 2008/03/14_16:30:42 info: local resource transition completed.



[root@node117 ha.d]# ldirectord -d /usr/local/etc/ha.d/ldirectord.cf start调试日志

DEBUG2: check_http: url="http://192.168.1.128:80/test.html" virtualhost="192.168.1.128"
LWP::UserAgent::new: ()
LWP::UserAgent::request: ()
LWP::UserAgent::send_request: GET http://192.168.1.128:80/test.html
LWP::UserAgent::_need_proxy: Not proxied
LWP:rotocol::http::request: ()
LWP:rotocol::collect: read 23 bytes
LWP::UserAgent::request: Simple response: OK
DEBUG2: Enabled server=192.168.1.128
DEBUG2: check_http: http://192.168.1.128:80/test.html is up
DEBUG2: Checking negotiate: real server=negotiate:http:tcp:192.168.1.150:80::80:1:\/test\.html:this\ node\ is\ available (virtual=tcp:192.168.1.118:80)
DEBUG2: check_http: url="http://192.168.1.150:80/test.html" virtualhost="192.168.1.150"
LWP::UserAgent::new: ()
LWP::UserAgent::request: ()
LWP::UserAgent::send_request: GET http://192.168.1.150:80/test.html
LWP::UserAgent::_need_proxy: Not proxied
LWP:rotocol::http::request: ()
LWP:rotocol::collect: read 23 bytes
LWP::UserAgent::request: Simple response: OK
DEBUG2: Enabled server=192.168.1.150
DEBUG2: check_http: http://192.168.1.150:80/test.html is up
DEBUG2: Checking negotiate: real server=negotiate:http:tcp:192.168.1.128:80::80:2:\/test\.html:this\ node\ is\ available (virtual=tcp:192.168.1.118:80)

[ 本帖最后由 xyfxh 于 2008-3-14 16:48 编辑 ]

xyfxh

[root@node117 ha.d]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddressort Scheduler Flags
  -> RemoteAddressort           Forward Weight ActiveConn InActConn
TCP  192.168.1.118:80 rr
  -> 192.168.1.150:80             Route   1      0          0
  -> 192.168.1.128:80             Route   1      0          0

现在的情况就是我把其中一台realserver服务器的HTTP服务停止

[root@node117 ha.d]# ipvsadm -Ln
这里面显示的服务器并没有被剔除，不知道原因在哪请指教


第二个问题就是当两台正式服务器都启动的时候，刷新VIP地址访问服务器的时候
总有一个服务器显示的网页不全，或者显示效果跟没有加载CSS一样，这是什么原因，谢谢

yingweixu

"[root@node117 ha.d]# ipvsadm -Ln
这里面显示的服务器并没有被剔除，不知道原因在哪请指教"
针对你的第一个问题：在ldirectord中，对于失效的real server 他并不会去除，只是在接下来的地址分配时候不给该服务器发送链接请求。如果你想让他去除得话也可以，只要把quiescent=yes改成NO 就可以了。

对于你得第二个问题因为你给的信息不够全，所以很难判断原因，请贴出你两个real server的网络设置，是否对他的网络参数进行更改以及虚拟端口你使用lo:0 还是 eth0:0?

xyfxh

这是150机器的配置
[root@real02 ~]# ip add list
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet 192.168.1.118/32 brd 192.168.1.118 scope global lo:0
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop qlen 1000
    link/ether 00:20:ed:79:c4:a3 brd ff:ff:ff:ff:ff:ff
3: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:20:ed:79:c4:a2 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.150/24 brd 192.168.1.255 scope global eth0
    inet6 fe80::220:edff:fe79:c4a2/64 scope link
       valid_lft forever preferred_lft forever
4: sit0: <NOARP> mtu 1480 qdisc noop
    link/sit 0.0.0.0 brd 0.0.0.0



这是128机器的网络配置
[root@realserver02 ~]# ip add list
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet 192.168.1.118/32 brd 192.168.1.118 scope global lo:0
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop qlen 1000
    link/ether 00:20:ed:7c:12:cf brd ff:ff:ff:ff:ff:ff
3: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:20:ed:7c:12:ce brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.128/24 brd 192.168.1.255 scope global eth0
    inet6 fe80::220:edff:fe7c:12ce/64 scope link
       valid_lft forever preferred_lft forever
4: sit0: <NOARP> mtu 1480 qdisc noop
    link/sit 0.0.0.0 brd 0.0.0.0



[root@node117 ha.d]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddressort Scheduler Flags
  -> RemoteAddressort           Forward Weight ActiveConn InActConn
TCP  vip:http rr
  -> 192.168.1.150:http           Route   1      0          6
  -> 192.168.1.128:http           Route   1      5          1
[root@node117 ha.d]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddressort Scheduler Flags
  -> RemoteAddressort           Forward Weight ActiveConn InActConn
TCP  vip:http rr
  -> 192.168.1.150:http           Route   1      0          13
  -> 192.168.1.128:http           Route   1      4          9
几乎150上没什么响应

[ 本帖最后由 xyfxh 于 2008-3-17 11:10 编辑 ]

yingweixu

从你发的信息看,两个realserver的网络地址配置是没错的,看不出什么问题,但是不知道你是否设置了 real server 关于arp 和 包转发的一系列网络设置参数,而且你没有贴出director server的网络接口参数.所以还是看不出问题.
建议你先使用ip直接访问出现问题的服务器看看是否也会出现类似的问题,如果一样会出现就不是集群的问题,而是你本身服务器的问题了.

xyfxh

[root@node117 ha.d]# ipvsadm -l -c
IPVS connection entries
pro expire state       source             virtual            destination
TCP 00:52  TIME_WAIT   192.168.1.55:41882 vip:http           realserver02:http
TCP 00:35  TIME_WAIT   192.168.1.55:41861 vip:http           realserver02:http
TCP 01:36  TIME_WAIT   192.168.1.55:41892 vip:http           realserver02:http
TCP 00:52  TIME_WAIT   192.168.1.55:41866 vip:http           realserver02:http
TCP 01:28  TIME_WAIT   192.168.1.55:41894 vip:http           real02:http
TCP 01:36  TIME_WAIT   192.168.1.55:41904 vip:http           realserver02:http
TCP 00:27  TIME_WAIT   192.168.1.55:41855 vip:http           real02:http
TCP 00:27  TIME_WAIT   192.168.1.55:41852 vip:http           real02:http
TCP 00:57  CLOSE_WAIT  192.168.1.55:41900 vip:http           realserver02:http
TCP 00:47  TIME_WAIT   192.168.1.55:41880 vip:http           real02:http
TCP 01:28  TIME_WAIT   192.168.1.55:41902 vip:http           real02:http
TCP 01:58  TIME_WAIT   192.168.1.55:41908 vip:http           real02:http
TCP 00:46  TIME_WAIT   192.168.1.55:41872 vip:http           real02:http
TCP 00:52  TIME_WAIT   192.168.1.55:41878 vip:http           realserver02:http
TCP 00:03  TIME_WAIT   192.168.1.55:41829 vip:http           realserver02:http
TCP 00:27  TIME_WAIT   192.168.1.55:41859 vip:http           real02:http
TCP 14:58  ESTABLISHED 192.168.1.55:41914 vip:http           realserver02:http
TCP 01:58  TIME_WAIT   192.168.1.55:41912 vip:http           real02:http
TCP 00:35  TIME_WAIT   192.168.1.55:41857 vip:http           realserver02:http
TCP 00:34  TIME_WAIT   192.168.1.55:41850 vip:http           realserver02:http
TCP 00:30  CLOSE_WAIT  192.168.1.55:41888 vip:http           realserver02:http
TCP 01:36  TIME_WAIT   192.168.1.55:41896 vip:http           realserver02:http
TCP 14:58  ESTABLISHED 192.168.1.55:41910 vip:http           realserver02:http
TCP 01:58  TIME_WAIT   192.168.1.55:41916 vip:http           real02:http
TCP 00:03  TIME_WAIT   192.168.1.55:41841 vip:http           realserver02:http
TCP 00:02  TIME_WAIT   192.168.1.55:41833 vip:http           realserver02:http
TCP 00:52  TIME_WAIT   192.168.1.55:41874 vip:http           realserver02:http
TCP 01:58  TIME_WAIT   192.168.1.55:41920 vip:http           real02:http
TCP 01:23  TIME_WAIT   192.168.1.55:41886 vip:http           real02:http
TCP 14:58  ESTABLISHED 192.168.1.55:41918 vip:http           realserver02:http
TCP 00:34  TIME_WAIT   192.168.1.55:41845 vip:http           realserver02:http
TCP 00:27  TIME_WAIT   192.168.1.55:41847 vip:http           real02:http
TCP 00:03  TIME_WAIT   192.168.1.55:41837 vip:http           realserver02:http
TCP 00:46  TIME_WAIT   192.168.1.55:41876 vip:http           real02:http
TCP 01:28  TIME_WAIT   192.168.1.55:41898 vip:http           real02:http
TCP 00:26  TIME_WAIT   192.168.1.55:41843 vip:http           real02:http
TCP 01:28  TIME_WAIT   192.168.1.55:41893 vip:http           real02:http
TCP 00:35  TIME_WAIT   192.168.1.55:41854 vip:http           realserver02:http
TCP 00:46  TIME_WAIT   192.168.1.55:41868 vip:http           real02:http
TCP 00:46  TIME_WAIT   192.168.1.55:41864 vip:http           real02:http
现在配置成NAT模式了，几乎等待状态的很多

xyfxh

这是LD SERVER的网络配置参数
[root@node117 ha.d]# ip add list
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 100
    link/ether 00:c0:9f:39:b0:fd brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.117/24 brd 192.168.1.255 scope global eth0
    inet 192.168.1.118/32 brd 192.168.1.118 scope global eth0:0
    inet6 fe80::2c0:9fff:fe39:b0fd/64 scope link
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 100
    link/ether 00:c0:9f:39:b0:fe brd ff:ff:ff:ff:ff:ff
    inet 192.168.10.117/24 brd 192.168.10.255 scope global eth1
    inet 192.168.10.118/32 brd 10.0.0.255 scope global eth1:0
    inet6 fe80::2c0:9fff:fe39:b0fe/64 scope link
       valid_lft forever preferred_lft forever
4: sit0: <NOARP> mtu 1480 qdisc noop
    link/sit 0.0.0.0 brd 0.0.0.0


两台NAT真是服务器网络配置参数
[root@real02 network-scripts]# ip add list
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 100
    link/ether 00:20:ed:79:c4:a3 brd ff:ff:ff:ff:ff:ff
    inet 192.168.10.150/24 brd 192.168.10.255 scope global eth1
    inet6 fe80::220:edff:fe79:c4a3/64 scope link
       valid_lft forever preferred_lft forever
3: eth0: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:20:ed:79:c4:a2 brd ff:ff:ff:ff:ff:ff
4: sit0: <NOARP> mtu 1480 qdisc noop
    link/sit 0.0.0.0 brd 0.0.0.0


[root@realserver02 network-scripts]# ip add list
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 100
    link/ether 00:20:ed:7c:12:cf brd ff:ff:ff:ff:ff:ff
    inet 192.168.10.128/24 brd 192.168.10.255 scope global eth1
    inet6 fe80::220:edff:fe7c:12cf/64 scope link
       valid_lft forever preferred_lft forever
3: eth0: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:20:ed:7c:12:ce brd ff:ff:ff:ff:ff:ff
4: sit0: <NOARP> mtu 1480 qdisc noop
    link/sit 0.0.0.0 brd 0.0.0.0

[root@realserver02 network-scripts]# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 4294967295
kernel.shmall = 268435456
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

xyfxh

heartbeat[13078]: 2008/03/18_10:54:48 info: Configuration validated. Starting heartbeat 2.0.6
heartbeat[13079]: 2008/03/18_10:54:48 info: heartbeat: version 2.0.6
heartbeat[13079]: 2008/03/18_10:54:48 info: Heartbeat generation: 82
heartbeat[13079]: 2008/03/18_10:54:48 info: G_main_add_TriggerHandler: Added signal manual handler
heartbeat[13079]: 2008/03/18_10:54:48 info: G_main_add_TriggerHandler: Added signal manual handler
heartbeat[13079]: 2008/03/18_10:54:48 info: Removing /usr/local/var/run/heartbeat/rsctmp failed, recreating.
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: UDP Broadcast heartbeat started on port 694 (694) interface eth1
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: UDP Broadcast heartbeat closed on port 694 interface eth1 - Status: 1
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: ucast: write socket priority set to IPTOS_LOWDELAY on eth0
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: ucast: bound send socket to device: eth0
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: ucast: bound receive socket to device: eth0
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: ucast: started on port 694 interface eth0 to 192.168.1.116
heartbeat[13079]: 2008/03/18_10:54:48 info: glib: ping heartbeat started.
heartbeat[13079]: 2008/03/18_10:54:48 info: G_main_add_SignalHandler: Added signal handler for signal 17
heartbeat[13079]: 2008/03/18_10:54:48 info: Local status now set to: 'up'
heartbeat[13079]: 2008/03/18_10:54:49 info: Link 192.168.1.1:192.168.1.1 up.
heartbeat[13079]: 2008/03/18_10:54:49 info: Status update for node 192.168.1.1: status ping
heartbeat[13079]: 2008/03/18_10:54:49 info: Link node117:eth1 up.
heartbeat[13079]: 2008/03/18_10:56:49 WARN: node node116: is dead
heartbeat[13079]: 2008/03/18_10:56:49 info: Comm_now_up(): updating status to active
heartbeat[13079]: 2008/03/18_10:56:49 info: Local status now set to: 'active'
heartbeat[13079]: 2008/03/18_10:56:49 WARN: No STONITH device configured.
heartbeat[13079]: 2008/03/18_10:56:49 WARN: Shared disks are not protected.
heartbeat[13079]: 2008/03/18_10:56:49 info: Resources being acquired from node116.
heartbeat[13096]: 2008/03/18_10:56:49 debug: notify_world: setting SIGCHLD Handler to SIG_DFL
harc[13096]:    2008/03/18_10:56:49 info: Running /usr/local/etc/ha.d/rc.d/status status
heartbeat[13097]: 2008/03/18_10:56:49 info: No local resources [/usr/local/lib/heartbeat/ResourceManager listkeys node117] to acquire.
heartbeat[13079]: 2008/03/18_10:56:49 debug: StartNextRemoteRscReq(): child count 1
mach_down[13115]:       2008/03/18_10:56:49 info: Taking over resource group LVSSyncDaemonSwap::master
ResourceManager[13136]: 2008/03/18_10:56:49 info: Acquiring resource group: node116 LVSSyncDaemonSwap::master IPaddr::192.168.1.118/32/eth0:118/192.168.1.255 IPaddr::192.168.10.118/32/eth1:118/10.0.0.255 lvsdr ldirectord::fxh.cf
ResourceManager[13136]: 2008/03/18_10:56:49 info: Running /usr/local/etc/ha.d/resource.d/LVSSyncDaemonSwap master start
ResourceManager[13136]: 2008/03/18_10:56:49 debug: Starting /usr/local/etc/ha.d/resource.d/LVSSyncDaemonSwap master start
LVSSyncDaemonSwap[13198]:       2008/03/18_10:56:50 info: ipvs_syncbackup down
LVSSyncDaemonSwap[13198]:       2008/03/18_10:56:50 info: ipvs_syncmaster up
LVSSyncDaemonSwap[13198]:       2008/03/18_10:56:50 info: ipvs_syncmaster obtained
ResourceManager[13136]: 2008/03/18_10:56:50 debug: /usr/local/etc/ha.d/resource.d/LVSSyncDaemonSwap master start done. RC=0
IPaddr[13247]:  2008/03/18_10:56:50 INFO: IPaddr Resource is stopped
ResourceManager[13136]: 2008/03/18_10:56:50 info: Running /usr/local/etc/ha.d/resource.d/IPaddr 192.168.1.118/32/eth0:118/192.168.1.255 start
ResourceManager[13136]: 2008/03/18_10:56:50 debug: Starting /usr/local/etc/ha.d/resource.d/IPaddr 192.168.1.118/32/eth0:118/192.168.1.255 start
eth0:118: warning: name may be invalid
IPaddr[13449]:  2008/03/18_10:56:50 INFO: /sbin/ifconfig eth0:0 192.168.1.118 netmask 255.255.255.255 broadcast 192.168.1.255
IPaddr[13449]:  2008/03/18_10:56:50 INFO: Sending Gratuitous Arp for 192.168.1.118 on eth0:0 [eth0]
IPaddr[13449]:  2008/03/18_10:56:50 INFO: /usr/local/lib/heartbeat/send_arp -i 500 -r 10 -p /usr/local/var/run/heartbeat/rsctmp/send_arp/send_arp-192.168.1.118 eth0 192.168.1.118 auto 192.168.1.118 ffffffffffff
IPaddr[13367]:  2008/03/18_10:56:50 INFO: IPaddr Success
INFO: IPaddr Success
ResourceManager[13136]: 2008/03/18_10:56:50 debug: /usr/local/etc/ha.d/resource.d/IPaddr 192.168.1.118/32/eth0:118/192.168.1.255 start done. RC=0
IPaddr[13542]:  2008/03/18_10:56:50 INFO: IPaddr Resource is stopped
ResourceManager[13136]: 2008/03/18_10:56:50 info: Running /usr/local/etc/ha.d/resource.d/IPaddr 192.168.10.118/32/eth1:118/10.0.0.255 start
ResourceManager[13136]: 2008/03/18_10:56:50 debug: Starting /usr/local/etc/ha.d/resource.d/IPaddr 192.168.10.118/32/eth1:118/10.0.0.255 start
eth1:118: warning: name may be invalid
IPaddr[13743]:  2008/03/18_10:56:50 INFO: /sbin/ifconfig eth1:0 192.168.10.118 netmask 255.255.255.255 broadcast 10.0.0.255
IPaddr[13743]:  2008/03/18_10:56:50 INFO: Sending Gratuitous Arp for 192.168.10.118 on eth1:0 [eth1]
IPaddr[13743]:  2008/03/18_10:56:50 INFO: /usr/local/lib/heartbeat/send_arp -i 500 -r 10 -p /usr/local/var/run/heartbeat/rsctmp/send_arp/send_arp-192.168.10.118 eth1 192.168.10.118 auto 192.168.10.118 ffffffffffff
IPaddr[13661]:  2008/03/18_10:56:50 INFO: IPaddr Success
INFO: IPaddr Success
ResourceManager[13136]: 2008/03/18_10:56:50 debug: /usr/local/etc/ha.d/resource.d/IPaddr 192.168.10.118/32/eth1:118/10.0.0.255 start done. RC=0
ResourceManager[13136]: 2008/03/18_10:56:50 info: Running /etc/init.d/lvsdr  start
ResourceManager[13136]: 2008/03/18_10:56:50 debug: Starting /etc/init.d/lvsdr  start
start LVS  of DirectorServer
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddressort Scheduler Flags
  -> RemoteAddressort           Forward Weight ActiveConn InActConn
TCP  vip:http rr
  -> 192.168.10.150:http          Masq    1      0          0
  -> 192.168.10.128:http          Masq    1      0          0
ResourceManager[13136]: 2008/03/18_10:56:51 debug: /etc/init.d/lvsdr  start done. RC=0
ResourceManager[13136]: 2008/03/18_10:56:51 info: Running /etc/init.d/ldirectord fxh.cf start
ResourceManager[13136]: 2008/03/18_10:56:51 debug: Starting /etc/init.d/ldirectord fxh.cf start
ResourceManager[13136]: 2008/03/18_10:56:51 debug: /etc/init.d/ldirectord fxh.cf start done. RC=0
mach_down[13115]:       2008/03/18_10:56:51 info: /usr/local/lib/heartbeat/mach_down: nice_failback: foreign resources acquired
mach_down[13115]:       2008/03/18_10:56:51 info: mach_down takeover complete for node node116.
heartbeat[13079]: 2008/03/18_10:56:51 info: mach_down takeover complete.
heartbeat[13079]: 2008/03/18_10:56:51 info: Initial resource acquisition complete (mach_down)
heartbeat[13079]: 2008/03/18_10:56:59 info: Local Resource acquisition completed. (none)
heartbeat[13079]: 2008/03/18_10:56:59 info: local resource transition completed.