大神们帮忙看一下这样的错误一般是什么引起的

Juice12

有台服务器监控到昨晚有大概一小时没响应，后来又恢复了，dmesg和/var/log/messages里面看到如下信息：有没大神帮分析下一般是什么导致的，OS是RHEL 6.6。
INFO: task automount:2961 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
automount     D ffff88204cf3a438     0  2961      1 0x10000000
ffff882050b13cf0 0000000000000086 00007fd6c80008ec ffff882050b13da8
ffffffff810b33b9 0000000000000019 ffff88204ed91358 0000000000000000
0000000000000000 ffffffff810fced2 ffff882051a10638 ffff882050b13fd8
Call Trace:
[<ffffffff810b33b9>] ? futex_wait+0x1d9/0x310
[<ffffffff810fced2>] ? ring_buffer_lock_reserve+0xa2/0x160
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc76>] rwsem_down_read_failed+0x26/0x30
[<ffffffff812989c4>] call_rwsem_down_read_failed+0x14/0x30
[<ffffffff8152c174>] ? down_read+0x24/0x30
[<ffffffff810fca64>] ? rb_reserve_next_event+0xb4/0x370
[<ffffffff8104d12e>] __do_page_fault+0x18e/0x480
[<ffffffff810fced2>] ? ring_buffer_lock_reserve+0xa2/0x160
[<ffffffff810fced2>] ? ring_buffer_lock_reserve+0xa2/0x160
[<ffffffff811022d3>] ? trace_nowake_buffer_unlock_commit+0x43/0x60
[<ffffffff81018619>] ? ftrace_raw_event_sys_exit+0xb9/0xc0
[<ffffffff8152ffbe>] do_page_fault+0x3e/0xa0
[<ffffffff8152d375>] page_fault+0x25/0x30
INFO: task automount:28811 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
automount     D ffff88204cf3a438     0 28811      1 0x10000000
ffff880e2aa2de08 0000000000000086 ffff88204e5938c0 ffffffff8122e5cf
ffff880e2aa2dd78 ffff881defbaf9c0 0000000000000000 0000000000000024
ffff880e2aa2de08 ffffffffffffffe9 ffff881ed6e19058 ffff880e2aa2dfd8
Call Trace:
[<ffffffff8122e5cf>] ? security_inode_permission+0x1f/0x30
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff8114545c>] sys_mmap_pgoff+0x5c/0x2d0
[<ffffffff81012499>] sys_mmap+0x29/0x30
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task automount:28812 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
automount     D ffff88204cf3a438     0 28812      1 0x10000000
ffff8820ff6fbe08 0000000000000086 ffff88204e5938c0 ffffffff8122e5cf
ffff8820ff6fbd78 ffff881dee28be00 0000000000000000 0000000000000024
ffff8820ff6fbe08 ffffffffffffffe9 ffff88404eef1af8 ffff8820ff6fbfd8
Call Trace:
[<ffffffff8122e5cf>] ? security_inode_permission+0x1f/0x30
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff8114545c>] sys_mmap_pgoff+0x5c/0x2d0
[<ffffffff81012499>] sys_mmap+0x29/0x30
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task automount:28813 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
automount     D ffff88204cf3a438     0 28813      1 0x10000000
ffff883e19879e08 0000000000000086 ffff88204e5938c0 ffffffff8122e5cf
ffff883e19879d78 ffff8804c04d93c0 0000000000000000 0000000000000024
ffff883e19879e08 ffffffffffffffe9 ffff88404faa3af8 ffff883e19879fd8
Call Trace:
[<ffffffff8122e5cf>] ? security_inode_permission+0x1f/0x30
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff8114545c>] sys_mmap_pgoff+0x5c/0x2d0
[<ffffffff81012499>] sys_mmap+0x29/0x30
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task automount:28814 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
automount     D ffff88204cf3a438     0 28814      1 0x10000000
ffff883e69d3be08 0000000000000086 ffff88204e5938c0 ffffffff8122e5cf
ffff883e69d3bd78 ffff881defbaf600 0000000000000000 0000000000000024
ffff883e69d3be08 ffffffffffffffe9 ffff88404faa3098 ffff883e69d3bfd8
Call Trace:
[<ffffffff8122e5cf>] ? security_inode_permission+0x1f/0x30
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff8114545c>] sys_mmap_pgoff+0x5c/0x2d0
[<ffffffff81012499>] sys_mmap+0x29/0x30
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task nscd:2992 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
nscd          D ffff88404f9c6c78     0  2992      1 0x10000000
ffff882051ce1e08 0000000000000086 ffff88204edb08c0 ffffffff8122e5cf
ffff882051ce1d78 ffff881df3b5cec0 0000000000000000 0000000000000024
ffff882051ce1e08 ffffffffffffffe9 ffff88204fbf5af8 ffff882051ce1fd8
Call Trace:
[<ffffffff8122e5cf>] ? security_inode_permission+0x1f/0x30
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff8114545c>] sys_mmap_pgoff+0x5c/0x2d0
[<ffffffff81012499>] sys_mmap+0x29/0x30
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task nscd:2993 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
nscd          D ffff88404f9c6c78     0  2993      1 0x10000000
ffff88204f849e08 0000000000000086 ffff88204edb08c0 ffffffff8122e5cf
ffff88204f849d78 ffff88014ff1eec0 0000000000000000 0000000000000024
ffff88204f849e08 ffffffffffffffe9 ffff88204f59f098 ffff88204f849fd8
Call Trace:
[<ffffffff8122e5cf>] ? security_inode_permission+0x1f/0x30
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff8114545c>] sys_mmap_pgoff+0x5c/0x2d0
[<ffffffff81012499>] sys_mmap+0x29/0x30
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task khugepaged:303 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
khugepaged    D ffff882051cfcbf8     0   303      2 0x00000000
ffff88204fa29c88 0000000000000046 ffff8820f0c94400 ffff88204fa29c00
ffff8820f0c968c0 0000000000000000 0000000000000001 00000000000168c0
0000000000000000 0000000000000001 ffff88204fa27ab8 ffff88204fa29fd8
Call Trace:
[<ffffffff810873dc>] ? lock_timer_base+0x3c/0x70
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8108579a>] ? ftrace_raw_event_timer_start+0xca/0xd0
[<ffffffff8152cc76>] rwsem_down_read_failed+0x26/0x30
[<ffffffff810874f0>] ? process_timeout+0x0/0x10
[<ffffffff812989c4>] call_rwsem_down_read_failed+0x14/0x30
[<ffffffff8152c174>] ? down_read+0x24/0x30
[<ffffffff811867bf>] khugepaged+0x18f/0x1310
[<ffffffff810096f0>] ? __switch_to+0xd0/0x320
[<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81186630>] ? khugepaged+0x0/0x1310
[<ffffffff8109e66e>] kthread+0x9e/0xc0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffff8109e5d0>] ? kthread+0x0/0xc0
[<ffffffff8100c200>] ? child_rip+0x0/0x20
INFO: task innovus:11502 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
innovus       D ffff882051cfcbf8     0 11502  10437 0x10000000
ffff881def0fbe48 0000000000000082 0000000000000024 ffff881def0fbde0
ffffffff810fae38 ffff881def0fbdb0 ffff881def0fbdb8 ffffffff81015463
ffff881def0fbdc8 ffff884053e44500 ffff881def1c1af8 ffff881def0fbfd8
Call Trace:
[<ffffffff810fae38>] ? __rb_reserve_next+0x178/0x380
[<ffffffff81015463>] ? native_sched_clock+0x13/0x80
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8152cc43>] rwsem_down_write_failed+0x23/0x30
[<ffffffff812989f3>] call_rwsem_down_write_failed+0x13/0x20
[<ffffffff8152c142>] ? down_write+0x32/0x40
[<ffffffff811554a8>] sys_munmap+0x48/0x80
[<ffffffff8100b288>] tracesys+0xd9/0xde
INFO: task khugepaged:303 blocked for more than 120 seconds.
      Not tainted 2.6.32-504.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
khugepaged    D ffff882051cfcbf8     0   303      2 0x00000000
ffff88204fa29c88 0000000000000046 ffff8820f0c94400 ffff88204fa29c00
ffff8820f0c968c0 0000000000000000 0000000000000001 00000000000168c0
0000000000000000 0000000000000001 ffff88204fa27ab8 ffff88204fa29fd8
Call Trace:
[<ffffffff810873dc>] ? lock_timer_base+0x3c/0x70
[<ffffffff8152cae5>] rwsem_down_failed_common+0x95/0x1d0
[<ffffffff8108579a>] ? ftrace_raw_event_timer_start+0xca/0xd0
[<ffffffff8152cc76>] rwsem_down_read_failed+0x26/0x30
[<ffffffff810874f0>] ? process_timeout+0x0/0x10
[<ffffffff812989c4>] call_rwsem_down_read_failed+0x14/0x30
[<ffffffff8152c174>] ? down_read+0x24/0x30
[<ffffffff811867bf>] khugepaged+0x18f/0x1310
[<ffffffff810096f0>] ? __switch_to+0xd0/0x320
[<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81186630>] ? khugepaged+0x0/0x1310
[<ffffffff8109e66e>] kthread+0x9e/0xc0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffff8109e5d0>] ? kthread+0x0/0xc0
[<ffffffff8100c200>] ? child_rip+0x0/0x20

vagrant_1220

貌似soft lockup的bug。所谓，soft lockup就是说，这个bug没有让系统彻底死机，但是若干个进程（或者kernel thread）被锁死在了某个状态（一般在内核区域）。很多情况下这个是由于内核锁的使用的问题，可能是磁盘问题。非法直接插拔移动磁盘等。