Perl爬虫，爬整站研究

mcshell

回复 13# 墨迹哥


    最近在搞oracle{:3_186:}    和系统自动化{:3_186:}

墨迹哥

回复 21# mcshell


    这两个都是运维的玩意啊。。你怎么跑运维的方向去了？。

mcshell

回复 22# 墨迹哥


    偶一直都是搞运维的{:3_186:}

iakuf

回复 20# 墨迹哥
没见到你讲超出容量的帖子，那个地方指出超出了容量？


   

grshrd49

回复 18# 墨迹哥

我的这两个变量就是排除非目标站点,还有排除各种分页用的
$base_url ;@exclude

grshrd49

回复 23# mcshell


哈哈!你的签名图片终于不动了 哈哈,

grshrd49

phenixghost

回复 6# yakczh_cu


    => 貌似和 , 是等价的吧:wink: 小骆驼刚学

墨迹哥

回复 24# iakuf


    14楼的回复。。。{:2_180:}

墨迹哥

回复 17# grshrd49


    在网上找了一段，感觉这个貌似跟我想法有重叠。
    目前还在分析和整理。。- - ！ 蛋腾了。你也来看看这个，看有什么可以借鉴的？

1. #!/usr/bin/perl
   
2. 
   
3. use URI;
   
4. use LWP;
   
5. 
   
6. open(GC,">process.txt") || die "$!\n";
   
7. print GC "PROCESS:\n";
   
8. close (GC);
   
9. my $t1=time();
   
10. my $mainweb = $ARGV[0];      #首页地址
    
11. my $deep=3;         #爬虫深度
    
12. my $report;         #生成报告
    
13. %hashurls=();        #储存url的深度
    
14. @allurls=();        #所有url列表
    
15. %hashtwo=();        #记录网页对应文件
    
16. 
    
17. $hashurls{$mainweb}=1;
    
18. $allurls[0]=$mainweb;
    
19. $hashtwo{$mainweb}=0;
    
20. my $i=1;         #数组allurls的长度
    
21. my $j=0;         #爬下的网页文件的个数
    
22. my $k=1;         #记录网页对应文件的参数
    
23. my $suc=0;         #成功爬下网页数量
    
24. my $fai=0;         #失败网页数量
    
25. my $dom;         #域名
    
26. 
    
27. print "mainweb=".$mainweb."\n";
    
28. if( $mainweb =~ /http:\/\/www.([^\/\s]{1,500})/i ){
    
29.         $dom=$1;        #域名
    
30.         print "domain is: ".$dom."\n";
    
31.         $report = $report."mainweb=".$mainweb."\n";
    
32.         $report = $report."domain is: ".$dom."\n";
    
33. }
    
34. else {
    
35.         print "wrong input\n";
    
36.         exit;
    
37. }
    
38. 
    
39. my $agent=LWP::UserAgent->new();
    
40. $agent->agent("zzq spider");
    
41. $agent->timeout(5);
    
42. 
    
43. #按数组allurls的顺序爬网页
    
44. foreach $u(@allurls){
    
45.         my $ret = &getweb($u);
    
46.         open(FD,">web/web".$j.".html") || die "$!\n";
    
47.         print FD "$ret\n";
    
48.         close(FD);
    
49.         $j++;
    
50. if($hashurls{$u}<$deep){
    
51.    &geturls($ret,$dom,$hashurls{$u},$u);
    
52. }
    
53. }
    
54. 
    
55. print "Finished!\n";
    
56. open(GC,">>process.txt") || die "$!\n";
    
57. print GC "Finished!\n";
    
58. close (GC);
    
59. 
    
60. #输出报告
    
61. my $t2=time();
    
62. my $costtime=$t2-$t1;
    
63. my $ti1=localtime($t1);
    
64. my $ti2=localtime($t2);
    
65. print "\n\n\n\n\nreport:\n";
    
66. $report = $report."start time = ".$ti1."\tfinish time = ".$ti2."\n";
    
67. $report = $report."Costtime = ".$costtime." seconds\n";
    
68. $report = $report."get ".$j." webs\tsuccess : ".$suc."\tfail : ".$fai."\n";
    
69. foreach $ur(@allurls){
    
70. $report = $report.$ur."\n ======> web".$hashtwo{$ur}.".html\n"
    
71. }
    
72. print "$report";
    
73. open(BG,">report.txt") || die "$!\n";
    
74. print BG "$report";
    
75. close (BG);
    
76. print "You can read this report in report.txt\n";
    
77. 
    
78. 
    
79. #get the urls form the web.
    
80. sub geturls{
    
81. my ($ret,$domain,$tmpdeep,$webu) = @_;
    
82. # print "now start geturls\n";
    
83. while( $ret =~ /href=\"([^\"\?\s>]{1,500})\"/ig )
    
84. {
    
85.    my $realurl = URI->new_abs( $1, $webu );
    
86.    if($realurl =~ /$domain/){
    
87.     if(defined $hashurls{$realurl}){
    
88.      next;
    
89.     }
    
90.     else {
    
91.      print "get a new url:$realurl\n";
    
92.      open(GC,">>process.txt") || die "$!\n";
    
93.      print GC "get a new url:$realurl\n";
    
94.      close (GC);
    
95.      $hashurls{$realurl} = $tmpdeep+1;
    
96.      $allurls[$i] = $realurl;
    
97.      $hashtwo{$realurl} = $k;
    
98.      $k++;
    
99.      $i++;
    
100.     }
     
101.    }
     
102.    else {
     
103.     next;
     
104.    }
     
105. }
     
106. 
     
107. while( $ret =~ /href=\'([^\'\?\s>]{1,500})\'/ig )
     
108. {
     
109.    my $realurl = URI->new_abs( $1, $webu );
     
110.    if($realurl =~ /$domain/){
     
111.     if(defined $hashurls{$realurl}){
     
112.      next;
     
113.     }
     
114.     else {
     
115.      print "get a new url:$realurl\n";
     
116.      open(GC,">>process.txt") || die "$!\n";
     
117.      print GC "get a new url:$realurl\n";
     
118.      close (GC);
     
119.      $hashurls{$realurl} = $tmpdeep+1;
     
120.      $allurls[$i] = $realurl;
     
121.      $hashtwo{$realurl} = $k;
     
122.      $k++;
     
123.      $i++;
     
124.     }
     
125.    }
     
126.    else {
     
127.     next;
     
128.    }
     
129. }
     
130. 
     
131. while( $ret =~ /href=([^\"\?\'\s>]{1,500})/ig )
     
132. {
     
133.    my $realurl = URI->new_abs( $1, $webu );
     
134.    if($realurl =~ /$domain/){
     
135.     if(defined $hashurls{$realurl}){
     
136.      next;
     
137.     }
     
138.     else {
     
139.      print "get a new url:$realurl\n";
     
140.      open(GC,">>process.txt") || die "$!\n";
     
141.      print GC "get a new url:$realurl\n";
     
142.      close (GC);
     
143.      $hashurls{$realurl} = $tmpdeep+1;
     
144.      $allurls[$i] = $realurl;
     
145.      $hashtwo{$realurl} = $k;
     
146.      $k++;
     
147.      $i++;
     
148.     }
     
149.    }
     
150.    else {
     
151.     next;
     
152.    }
     
153. }
     
154. while( $ret =~ /window\.open\(\'([^\'\?\s>]{1,500})/ig )
     
155. {
     
156.    my $realurl = URI->new_abs( $1, $webu );
     
157.    if($realurl =~ /$domain/){
     
158.     if(defined $hashurls{$realurl}){
     
159.      next;
     
160.     }
     
161.     else {
     
162.      print "get a new url:$realurl\n";
     
163.      open(GC,">>process.txt") || die "$!\n";
     
164.      print GC "get a new url:$realurl\n";
     
165.      close (GC);
     
166.      $hashurls{$realurl} = $tmpdeep+1;
     
167.      $allurls[$i] = $realurl;
     
168.      $hashtwo{$realurl} = $k;
     
169.      $k++;
     
170.      $i++;
     
171.     }
     
172.    }
     
173.    else {
     
174.     next;
     
175.    }
     
176. }
     
177. }
     
178. 
     
179. 
     
180. #get a web form internet
     
181. sub getweb
     
182. {
     
183. my ($u) = @_;
     
184. my $h=$j+1;
     
185. print "webaddress: " . $u . "\n";
     
186. print "getting the ".$h." th web of $i in total\n";
     
187. open(GC,">>process.txt") || die "$!\n";
     
188. print GC "webaddress: " . $u . "\n";
     
189. print GC "getting the ".$h." th web of $i in total\n";
     
190. close (GC);
     
191. my $request=HTTP::Request->new(GET,$u);
     
192. my $response=$agent->request($request);
     
193. if ($response->is_success){
     
194.    print "successed!\n";
     
195.    open(GC,">>process.txt") || die "$!\n";
     
196.    print GC "successed!\n";
     
197.    close (GC);
     
198.    $suc++;
     
199.    return $response->content;
     
200. }
     
201. else{
     
202.    print "failed! Status = ".$response->status_line."\n";
     
203.    open(GC,">>process.txt") || die "$!\n";
     
204.    print GC "failed! Status = ".$response->status_line."\n";
     
205.    close (GC);
     
206.    $fai++;
     
207.    return $response->status_line;
     
208. }
     
209. }
     

复制代码