flock 文件锁和NAS引发的问题！！！！

oract666

大家好：

    我现在遇到一个问题。某些应用用到了文件锁，在一个进程中很多的并发线程以加锁的方式、追加模式打开文件，锁定即将写入的部分，以达到串行化写文件的目的。

    本来这部分程序在SAN结构的存储中运行良好（线程所写的日志放在SAN结构的存储上），但是上周做了存储的迁移，临时把日志文件放在了NAS存储上，结果导致线程被严重阻塞。

    为了进一步说明问题，我专门做了个模拟程序，模拟了这部分功能。编译平台是AIX 5.3，XLC 64位编译器。源代码如附件所示，日志文件放在NAS上。用truss追踪发现线程全部受阻，但是仍旧可以缓慢执行，但是的确很慢，要1-2分钟才能执行一次写文件。顺便说一句，如果日志文件不放置在NAS上，则不会出现阻塞情况。所以我现在怀疑根本原因是flock机制和NAS所支持的文件共享功能之间有冲突所导致。具体还请大家详细指点一下，谢谢！


test1.zip (925 Bytes, 下载次数: 15)

2012-08-29 12:18 上传

点击文件名下载附件
源代码

truss如下：

kwrite(1, " n e w   t h r e a d :  ".., 34)     = 34
_nsleep(0x00000001100404C0, 0x0000000110040590) = 0
Thread 258,bzero successful,j=1
kwrite(1, " T h r e a d   2 5 8 , b".., 32)     = 32
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 3
kioctl(3, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
_nsleep(0x00000001100E64C0, 0x00000001100E6590) = 0
Thread 1029,bzero successful,j=1
kwrite(1, " T h r e a d   1 0 2 9 ,".., 33)     = 33
_nsleep(0x00000001100AF4C0, 0x00000001100AF590) = 0
Thread 772,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 4
kwrite(1, " T h r e a d   7 7 2 , b".., 32)     = 32
kioctl(4, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
_nsleep(0x000000011011E4C0, 0x000000011011E590) = 0
thread_waitlock_(0x0000000110009518, 1, 0x0000000000000000) = 1
thread_unlock(0x000000011000951               = 0
_nsleep(0x00000001108B44C0, 0x00000001108B4590) = 0
_nsleep(0x000000011087D4C0, 0x000000011087D590) = 0
_nsleep(0x00000001108454C0, 0x0000000110845590) = 0
_nsleep(0x000000011080E4C0, 0x000000011080E590) = 0
_nsleep(0x00000001107D64C0, 0x00000001107D6590) = 0
_nsleep(0x000000011079F4C0, 0x000000011079F590) = 0
_nsleep(0x00000001107674C0, 0x0000000110767590) = 0
_nsleep(0x00000001107304C0, 0x0000000110730590) = 0
_nsleep(0x00000001106F84C0, 0x00000001106F8590) = 0
_nsleep(0x00000001106C14C0, 0x00000001106C1590) = 0
_nsleep(0x00000001106894C0, 0x0000000110689590) = 0
_nsleep(0x00000001106514C0, 0x0000000110651590) = 0
_nsleep(0x000000011061A4C0, 0x000000011061A590) = 0
_nsleep(0x00000001105E24C0, 0x00000001105E2590) = 0
_nsleep(0x00000001105AB4C0, 0x00000001105AB590) = 0
_nsleep(0x00000001105734C0, 0x0000000110573590) = 0
_nsleep(0x000000011053C4C0, 0x000000011053C590) = 0
_nsleep(0x00000001105044C0, 0x0000000110504590) = 0
_nsleep(0x00000001104CD4C0, 0x00000001104CD590) = 0
_nsleep(0x00000001104964C0, 0x0000000110496590) = 0
_nsleep(0x000000011045E4C0, 0x000000011045E590) = 0
_nsleep(0x00000001104274C0, 0x0000000110427590) = 0
_nsleep(0x00000001103EF4C0, 0x00000001103EF590) = 0
_nsleep(0x00000001103B84C0, 0x00000001103B8590) = 0
_nsleep(0x00000001103804C0, 0x0000000110380590) = 0
_nsleep(0x00000001103494C0, 0x0000000110349590) = 0
_nsleep(0x00000001103114C0, 0x0000000110311590) = 0
_nsleep(0x00000001102DA4C0, 0x00000001102DA590) = 0
_nsleep(0x00000001102A24C0, 0x00000001102A2590) = 0
_nsleep(0x000000011026B4C0, 0x000000011026B590) = 0
_nsleep(0x00000001102334C0, 0x0000000110233590) = 0
_nsleep(0x00000001101FC4C0, 0x00000001101FC590) = 0
_nsleep(0x00000001101C44C0, 0x00000001101C4590) = 0
_nsleep(0x000000011018D4C0, 0x000000011018D590) = 0
_nsleep(0x00000001101554C0, 0x0000000110155590) = 0
_nsleep(0x00000001100774C0, 0x0000000110077590) = 0
Thread 10281,bzero successful,j=1
kwrite(1, " T h r e a d   1 0 2 8 1".., 34)     = 34
Thread 10024,bzero successful,j=1
kwrite(1, " T h r e a d   1 0 0 2 4".., 34)     = 34
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 5
kioctl(5, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 9767,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 6
kwrite(1, " T h r e a d   9 7 6 7 ,".., 33)     = 33
kioctl(6, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 9510,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 7
kwrite(1, " T h r e a d   9 5 1 0 ,".., 33)     = 33
kioctl(7, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 9253,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 8
kwrite(1, " T h r e a d   9 2 5 3 ,".., 33)     = 33
kioctl(8, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 8996,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 9
kwrite(1, " T h r e a d   8 9 9 6 ,".., 33)     = 33
kioctl(9, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 8739,bzero successful,j=1
kwrite(1, " T h r e a d   8 7 3 9 ,".., 33)     = 33
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 10
kioctl(10, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 8482,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 11
kwrite(1, " T h r e a d   8 4 8 2 ,".., 33)     = 33
kioctl(11, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 8225,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 12
kwrite(1, " T h r e a d   8 2 2 5 ,".., 33)     = 33
kioctl(12, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 7968,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 13
kwrite(1, " T h r e a d   7 9 6 8 ,".., 33)     = 33
kioctl(13, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 7711,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 14
kwrite(1, " T h r e a d   7 7 1 1 ,".., 33)     = 33
kioctl(14, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 7454,bzero successful,j=1
kwrite(1, " T h r e a d   7 4 5 4 ,".., 33)     = 33
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 15
kioctl(15, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
Thread 7197,bzero successful,j=1
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 16
kwrite(1, " T h r e a d   7 1 9 7 ,".., 33)     = 33
kioctl(16, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY
kfcntl(5, 13, 0x00000001108B46E0)               = 0
kwrite(5, " t h i s   i s   t h r e".., 41)     = 41
Thread 6940,bzero successful,j=1
kwrite(1, " T h r e a d   6 9 4 0 ,".., 33)     = 33
kfcntl(6, 13, 0x000000011087D6E0)               = 0
kfcntl(5, 13, 0x00000001108B46E0)               = 0
close(5)                                        = 0
kwrite(4572529856, 0x00000001108B4590, 0)       = 41
kfcntl(7, 13, 0x00000001108456E0)               = 0
_nsleep(0x00000001108B44C0, 0x00000001108B4590) = 0
kfcntl(6, 13, 0x000000011087D6E0)               = 0
kfcntl(8, 13, 0x000000011080E6E0)               = 0
thread_unlock(0x000000011000951               = 0
thread_waitlock_(0x0000000110009518, 1, 0x0000000000000000) = 1
kwrite(7, " t h i s   i s   t h r e".., 40)     = 40
Thread 10281,bzero successful,j=2
kwrite(1, " T h r e a d   1 0 2 8 1".., 34)     = 34
kfcntl(9, 13, 0x00000001107D66E0)               = 0
close(6)                                        = 0
open("/test/test1.log", O_RDWR|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) = 5
kioctl(5, 22528, 0x0000000000000000, 0x0000000000000000) Err#25 ENOTTY

hellioncu

改用fcntl加锁试试

oract666

回复 2# hellioncu


   

现在就是采用的fcntl加锁啊

fcntl_i=fcntl(fd,F_SETLKW,&f1);

windoze

NAS用的什么系统？什么NFS服务器？
之前的SAN十有八九用的是iSCSI，文件系统驱动在本机，所以没问题，改用NAS之后应该是用的NFS，众所周知NFS的锁永远都有问题

file3

如果你的文件在nfs上并只有一个客户端，可以在客户机上做一个映射文件之类的东西。映射文件写到一定大小传到nfs上，客户机上的进程每次对这个文件进行加锁。另可以提供一个同步函数，便于应用上强制同步文件到nfs。

如果是多客户机的nfs就必须在nfs上做自己的同步处理了。客户机同步类似于上面。