求教找出目录下文件内容中完全重复的行

horizonhyg

哇,三层循环加过滤...

jason680

须求不明+1

ywlscpl

按你给的文件顺序1.sql 2.sql 3.sql，6楼代码结果

1. ywlscpl@ubuntu:~$ find . -name "*.sql" -type f -print
   
2. ./2.sql
   
3. ./1.sql
   
4. ./d1/3.sql
   
5. ywlscpl@ubuntu:~$ sh comm.sh
   
6. ./d1/3.sql:line 17==>./2.sql:line 5
   
7. ./d1/3.sql:line 19==>./2.sql:line 7
   
8. ./d1/3.sql:line 21==>./2.sql:line 9
   
9. ./d1/3.sql:line 7==>./1.sql:line 3
   
10. ./d1/3.sql:line 9==>./1.sql:line 5

复制代码

chiyiangel

回复 13# ywlscpl


   谢谢热心的大哥 现在可以了 ...  我把条件改称了 /^go/  可能是awk 的版本问题 ....
不过现在 又有新的问题了   我发现在需要过滤得这些.sql脚本里有很多 空白行 还有很多括号 单占一行 我不想把这些过滤出来 我只想 过滤一下insert 或者 create 的开头的 相同的语句 ....

springwind426

1. 
   
2. /tmp/test.pl   内容如下：
   
3. 
   
4. #!/usr/bin/perl
   
5. while($f=<STDIN>)
   
6. {
   
7.         $i=1;
   
8.         chomp $f;
   
9.         open($fh,"<",$f);
   
10.         while(<$fh>)
    
11.         {
    
12.                 chomp;
    
13.                 next if not /^(insert|create)/;
    
14.                 $a{$_}=sprintf("%s\n%s[%d]",$a{$_},$f,$i++);
    
15.         }
    
16.         close($fh);
    
17. }
    
18. while(my($sql,$files)=each(%a))
    
19. {
    
20.         next if not $files=~/\n.*\n/;
    
21.         printf "%s%s\n",$sql,$files;
    
22. }
    
23. 
    
24. 
    
25. 然后 chmod +x /tmp/test.pl
    
26. 
    
27. find 目录 -type f -name "*.sql" | /tmp/test.pl
    
28. 
    
29. 条件是你的insert或者create语句是单独一行，并且create或者insert是顶格写
    

复制代码

ywlscpl

回复 14# chiyiangel


    空白行 还有很多括号 单占一行

这些都加到例外判断里awk '/^ *go *$/||!NF||/^ *{ *$/||/^ *} *$/{next}

或直接  awk '!/^ *insert/||!/^ *create/{next}

chiyiangel

回复 16# ywlscpl


    非常感谢这位大哥 太热心了  待会儿试下看 ...   谢谢

chiyiangel

回复 15# springwind426

非常感谢... 但是用你的脚本来跑得话 过滤出来的东西 不正确阿...  比如用这个脚本来过滤上面的例子  结果是这个

1. insert into pub_billtemplet_b(cardflag,datatype,defaultshowname,defaultvalue,dr,editflag,editformula,foreground,hyperlinkflag,idcolname,inputlength,itemkey,itemtype,leafflag,listflag,listshowflag,loadformula,lockflag,metadatapath,metadataproperty,metadatarelation,newlineflag,nullflag,options,pk_billtemplet,pk_billtemplet_b,pk_corp,pos,reftype,resid,resid_tabname,reviseflag,showflag,showorder,table_code,table_name,totalflag,ts,userdefflag,userdefine1,userdefine2,userdefine3,usereditflag,userflag,userreviseflag,usershowflag,validateformula,width) values(1,-1,null,null,0,0,null,-1,'N',null,-1,'modifier',0,'N',1,'Y',null,0,'modifier','ncaam.appassetversion.modifier',null,'N',0,null,'1027Z1100000000005G5','1027Z1100000000005GE','@@@@',2,null,null,null,'N',1,3,'audit','审计信息',0,'2010-06-18 15:55:15','N',null,null,null,1,1,'N',1,null,1)
   
2. 
   
3. ./003.sql[4]
   
4. ./002.sql[10]
   
5. insert into pub_billtemplet_b(cardflag,datatype,defaultshowname,defaultvalue,dr,editflag,editformula,foreground,hyperlinkflag,idcolname,inputlength,itemkey,itemtype,leafflag,listflag,listshowflag,loadformula,lockflag,metadatapath,metadataproperty,metadatarelation,newlineflag,nullflag,options,pk_billtemplet,pk_billtemplet_b,pk_corp,pos,reftype,resid,resid_tabname,reviseflag,showflag,showorder,table_code,table_name,totalflag,ts,userdefflag,userdefine1,userdefine2,userdefine3,usereditflag,userflag,userreviseflag,usershowflag,validateformula,width) values(1,-1,null,null,0,1,null,-1,'N',null,-1,'venddate',0,'N',1,'Y',null,0,'venddate','ncaam.appassetversion.venddate',null,'N',0,null,'1027Z1100000000005G5','1027Z1100000000005GC','@@@@',0,null,null,null,'N',1,6,'appassetversionHVO','appassetversionHVO',0,'2010-06-18 15:55:15','N',null,null,null,1,1,'N',1,null,1)
   
6. 
   
7. ./001.sql[4]
   
8. ./002.sql[8]
   
9. insert into pub_billtemplet_b(cardflag,datatype,defaultshowname,defaultvalue,dr,editflag,editformula,foreground,hyperlinkflag,idcolname,inputlength,itemkey,itemtype,leafflag,listflag,listshowflag,loadformula,lockflag,metadatapath,metadataproperty,metadatarelation,newlineflag,nullflag,options,pk_billtemplet,pk_billtemplet_b,pk_corp,pos,reftype,resid,resid_tabname,reviseflag,showflag,showorder,table_code,table_name,totalflag,ts,userdefflag,userdefine1,userdefine2,userdefine3,usereditflag,userflag,userreviseflag,usershowflag,validateformula,width) values(1,-1,null,null,0,0,null,-1,'N',null,-1,'creator',0,'N',1,'Y',null,0,'creator','ncaam.appassetversion.creator',null,'N',0,null,'1027Z1100000000005G5','1027Z1100000000005GD','@@@@',2,null,null,null,'N',1,1,'audit','审计信息',0,'2010-06-18 15:55:15','N',null,null,null,1,1,'N',1,null,1)
   
10. 
    
11. ./003.sql[3]
    
12. ./002.sql[9]
    
13. insert into pub_billtemplet_b(cardflag,datatype,defaultshowname,defaultvalue,dr,editflag,editformula,foreground,hyperlinkflag,idcolname,inputlength,itemkey,itemtype,leafflag,listflag,listshowflag,loadformula,lockflag,metadatapath,metadataproperty,metadatarelation,newlineflag,nullflag,options,pk_billtemplet,pk_billtemplet_b,pk_corp,pos,reftype,resid,resid_tabname,reviseflag,showflag,showorder,table_code,table_name,totalflag,ts,userdefflag,userdefine1,userdefine2,userdefine3,usereditflag,userflag,userreviseflag,usershowflag,validateformula,width) values(1,-1,null,null,0,1,null,-1,'N',null,-1,'vstartdate',0,'N',1,'Y',null,0,'vstartdate','ncaam.appassetversion.vstartdate',null,'N',0,null,'1027Z1100000000005G5','1027Z1100000000005GB','@@@@',0,null,null,null,'N',1,5,'appassetversionHVO','appassetversionHVO',0,'2010-06-18 15:55:15','N',null,null,null,1,1,'N',1,null,1)
    
14. 
    
15. ./001.sql[3]
    
16. ./002.sql[7]
    
17. insert into pub_billtemplet_t(basetab,dr,metadataclass,metadatapath,mixindex,pk_billtemplet,pk_billtemplet_t,pk_layout,pos,position,resid,tabcode,tabindex,tabname,ts,vdef1,vdef2,vdef3) values(null,0,'ncaam.appassetversion',null,null,'1027Z1100000000005G5','1004Z11000000000LEUD','~',0,0,null,'appassetversionHVO',0,'应用资产版本表头','2010-06-18 15:55:15',null,null,null)
    
18. 
    
19. ./003.sql[1]
    
20. ./002.sql[3]
    
21. insert into pub_billtemplet_b(cardflag,datatype,defaultshowname,defaultvalue,dr,editflag,editformula,foreground,hyperlinkflag,idcolname,inputlength,itemkey,itemtype,leafflag,listflag,listshowflag,loadformula,lockflag,metadatapath,metadataproperty,metadatarelation,newlineflag,nullflag,options,pk_billtemplet,pk_billtemplet_b,pk_corp,pos,reftype,resid,resid_tabname,reviseflag,showflag,showorder,table_code,table_name,totalflag,ts,userdefflag,userdefine1,userdefine2,userdefine3,usereditflag,userflag,userreviseflag,usershowflag,validateformula,width) values(1,-1,null,null,0,1,null,-1,'N',null,-1,'vname',0,'N',1,'Y',null,0,'vname','ncaam.appassetversion.vname',null,'N',0,null,'1027Z1100000000005G5','1027Z1100000000005GA','@@@@',0,null,null,null,'N',1,4,'appassetversionHVO','appassetversionHVO',0,'2010-06-18 15:55:15','N',null,null,null,1,1,'N',1,null,1)
    
22. 
    
23. ./001.sql[2]
    
24. ./002.sql[6]

复制代码

ywlscpl

回复 14# chiyiangel

/^ *go/不匹配，/^go/能匹配
    晕，你什么awk版本？ 连/ */都不支持？

chiyiangel

回复 19# ywlscpl


    如果用/^ *insert */ 就过滤不出来
   要用/^insert/ 才行 还好 这些脚本 都是insert 开头的 呵呵