Chinaunix
标题:
RHEL5.4根目录100%的不知道什么原因
[打印本页]
作者:
lusklusklusk
时间:
2015-05-25 11:42
标题:
RHEL5.4根目录100%的不知道什么原因
环境RHEL5.4+ORACLE11GR2
早上发现oracle挂了，查询发现linux的根目录满了，进行了一些操作如service syslog restart，umount rman_backup，删除了一些日志文件，依旧无效，根据网上的一些经验操作也没有找到具体原因，重启服务器后恢复，重启之前的详细信息如下，哪位大牛能帮忙看看具体原因是什么
[root@mdsdb03 /]# df -lh
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup00-LogVol00
274G 260G 0 100% /
/dev/sda1 99M 13M 81M 14% /boot
tmpfs 253G 397M 252G 1% /dev/shm
/dev/sdb1 1.1T 58G 985G 6% /rman_backup
[root@mdsdb03 /]# du -skh
76G .
[root@mdsdb03 /]# df -i
Filesystem Inodes IUsed IFree IUse% Mounted on
/dev/mapper/VolGroup00-LogVol00
73990144 212249 73777895 1% /
/dev/sda1 26104 34 26070 1% /boot
tmpfs 66078737 195 66078542 1% /dev/shm
/dev/sdb1 146243584 89 146243495 1% /rman_backup
[root@mdsdb03 /]# df -alh
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup00-LogVol00
274G 260G 0 100% /
proc 0 0 0 - /proc
sysfs 0 0 0 - /sys
devpts 0 0 0 - /dev/pts
/dev/sda1 99M 13M 81M 14% /boot
tmpfs 253G 397M 252G 1% /dev/shm
none 0 0 0 - /proc/sys/fs/binfmt_misc
sunrpc 0 0 0 - /var/lib/nfs/rpc_pipefs
oracleasmfs 0 0 0 - /dev/oracleasm
/dev/sdb1 1.1T 58G 985G 6% /rman_backup
[root@mdsdb03 ~]# cat /etc/fstab
/dev/VolGroup00/LogVol00 / ext3 defaults 1 1
/dev/sdb1 /rman_backup ext3 defaults 1 1
LABEL=/boot /boot ext3 defaults 1 2
tmpfs /dev/shm tmpfs defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
sysfs /sys sysfs defaults 0 0
proc /proc proc defaults 0 0
/dev/VolGroup00/LogVol01 swap swap defaults 0 0
[root@mdsdb03 /]# du -sh *
323M backup
7.7M bin
7.3M boot
398M dev
106M etc
3.0M home
141M lib
20M lib64
16K lost+found
8.0K media
0 misc
8.0K mnt
0 net
3.5M opt
0 proc
68G rman_backup
844K root
77M sbin
8.0K selinux
3.5G software
8.0K srv
0 sys
100K tftpboot
60K tmp
12G u01
2.2G usr
120M var
[root@mdsdb03 dumpdir]# free -g
total used free shared buffers cached
Mem: 504 259 244 0 0 249
-/+ buffers/cache: 9 494
Swap: 275 0 275
[root@mdsdb03 dumpdir]# lsof|grep deleted
sh 10815 oracle 1w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
sh 10815 oracle 2w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
expdp 10816 oracle 1w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
expdp 10816 oracle 2w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
gconfd-2 12044 root 13wW REG 253,0 610 73105413 /tmp/gconfd-root/lock/0t1432520270ut371296u0p12044r463110117k1535203864 (deleted)
syslogd 12516 root 2w REG 253,0 1377 39322222 /var/log/messages (deleted)
syslogd 12516 root 3w REG 253,0 1005 39322477 /var/log/secure (deleted)
syslogd 12516 root 4w REG 253,0 60569 39322479 /var/log/maillog (deleted)
syslogd 12516 root 5w REG 253,0 19916 39322482 /var/log/cron (deleted)
syslogd 12516 root 6w REG 253,0 0 39322480 /var/log/spooler (deleted)
syslogd 12516 root 7w REG 253,0 0 39322481 /var/log/boot.log (deleted)
acpid 13078 root 1w REG 253,0 2378 39322103 /var/log/acpid (deleted)
acpid 13078 root 2w REG 253,0 2378 39322103 /var/log/acpid (deleted)
gpm 13563 root 0u CHR 136,0 2 /dev/pts/0 (deleted)
gpm 13563 root 1u CHR 136,0 2 /dev/pts/0 (deleted)
gpm 13563 root 2u CHR 136,0 2 /dev/pts/0 (deleted)
Xorg 13822 root 0w REG 253,0 59428 39322132 /var/log/Xorg.0.log (deleted)
gpnpd.bin 28468 grid 17uw REG 253,0 0 39354408 /var/tmp/.oracle/ora_gipc_GPNPD_mdsdb03_lock (deleted)
ocssd.bin 28519 grid 17uw REG 253,0 0 39354412 /var/tmp/.oracle/sOCSSD_LL_mdsdb03__lock (deleted)
ocssd.bin 28519 grid 72uw REG 253,0 0 39354414 /var/tmp/.oracle/sOracle_CSS_LclLstnr_mdsdb-cluster_1_lock (deleted)
ocssd.bin 28519 grid 74uw REG 253,0 0 39354416 /var/tmp/.oracle/sOCSSD_LL_mdsdb03_mdsdb-cluster_lock (deleted)
oracle 28780 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28780 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28782 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28782 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28786 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28786 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28788 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28788 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28790 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28790 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28792 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28792 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28794 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28794 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28796 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28796 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28798 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28798 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28800 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28800 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28804 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28804 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28806 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28806 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28808 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28808 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28810 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28810 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28812 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28812 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28814 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28814 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28816 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28816 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28818 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28818 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28820 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28820 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28822 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28822 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28830 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28830 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28858 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28858 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
oracle 28860 grid 6w REG 253,0 10590185 57147567 /u01/app/grid/log/mdsdb03/agent/ohasd/oraagent_grid/oraagent_grid.l10 (deleted)
oracle 28860 grid 15u REG 253,0 0 56533970 /u01/app/grid/dbs/lkinst+ASM1 (deleted)
java 29889 grid 17w REG 253,0 81 57147825 /u01/app/grid/log/mdsdb03/srvm/eonsd.trc.12 (deleted)
作者:
lusklusklusk
时间:
2015-05-25 11:52
根目录270G左右，根目录下面的文件大小才70G左右，不知道那200G去哪里了
作者:
liaosnet
时间:
2015-05-25 12:10
本帖最后由 liaosnet 于 2015-05-25 12:10 编辑
删除的文件没释放掉（inode占用)吧~~
重启后程序释放了这些inode，也就释放出来空间了。
作者:
lusklusklusk
时间:
2015-05-25 13:03
删除的日志不多，都是oracle的日志文件，不超过1000个，总大小不超过1G，重启之前就看到inode还有73777895可用，不是node的问题，不知道什么导致
回复
3#
liaosnet
作者:
dengbao2001
时间:
2015-05-25 14:02
/dev/mapper/VolGroup00-LogVol00
这个LV用来260G，总计有274G，有一定保留空间给管理员使用。你查查这个LV的使用状态
作者:
jixuuse
时间:
2015-05-25 14:08
find / -xdev -size +100M -exec ls -l {} \;
查一下大于100M的文件，也许有惊喜
作者:
lusklusklusk
时间:
2015-05-25 14:52
找到原因了，/backup/dumpdir/nohup.out文件200多G，但是句柄没有释放如
[root@mdsdb03 dumpdir]# lsof|grep deleted
sh 10815 oracle 1w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
sh 10815 oracle 2w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
expdp 10816 oracle 1w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
expdp 10816 oracle 2w REG 253,0 259639676928 65273914 /backup/dumpdir/nohup.out (deleted)
但是为什么如下又只显示323M
[root@mdsdb03 /]# du -sh *
323M backup
作者:
lyhabc
时间:
2015-05-25 22:56
df和du这两个命令方式本来不一样
df命令是用来观察总体磁盘使用量的，要观察局部使用量，需要使用du命令。df命令可以通过读取磁盘的superblock来实现，而du命令则不同，它要搜索所有的inode来计算局部数据，所以du命令的执行效率，经常要比df差很多。
作者:
Shell_HAT
时间:
2015-05-28 13:47
回复
7#
lusklusklusk
http://blog.itpub.net/10173379/viewspace-744440/
欢迎光临 Chinaunix (http://bbs.chinaunix.net/)
Powered by Discuz! X3.2