php模拟某网站分页脚本

Galenlong

要对这个网站http://www.cquae.com/search.shtml?ob=ProjectCenter&lx=2&jg=1进行抓取，可是总是只能抓取第一页的，无法模拟分页的过程。分析过该网站的分页参数传递，用snoopy、curl等方法试过之后还是不行，麻烦有心人帮一起研究哈，谢谢!
这是抓取的header

1. FIRST REQUEST:
   
2. 
   
3. GET http://www.cquae.com/search_list.aspx?ob=ProjectCenter&lx=2 HTTP/1.1
   
4. Host: www.cquae.com
   
5. User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13
   
6. Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
   
7. Accept-Language: zh-cn,zh;q=0.5
   
8. Accept-Encoding: gzip,deflate
   
9. Accept-Charset: GB2312,utf-8;q=0.7,*;q=0.7
   
10. Keep-Alive: 115
    
11. Proxy-Connection: keep-alive
    
12. 
    
13. 
    
14. FIRST response:
    
15. 
    
16. HTTP/1.0 200 OK
    
17. Cache-Control: private
    
18. Content-Type: text/html; charset=utf-8
    
19. Server: Microsoft-IIS/7.5
    
20. X-Powered-By: UrlRewriter.NET 2.0.0
    
21. Set-Cookie: ASP.NET_SessionId=5050pcjv24rxsejce1dtbjas; path=/; HttpOnly
    
22. X-AspNet-Version: 2.0.50727
    
23. X-Powered-By: ASP.NET
    
24. Date: Mon, 27 Dec 2010 03:31:38 GMT
    
25. Content-Length: 62646
    
26. Connection: keep-alive
    
27. 
    
28. SECOND REQUEST:
    
29. POST http://www.cquae.com/search_list.aspx?ob=ProjectCenter&lx=2 HTTP/1.1
    
30. Host: www.cquae.com
    
31. User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13
    
32. Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
    
33. Accept-Language: zh-cn,zh;q=0.5
    
34. Accept-Encoding: gzip,deflate
    
35. Accept-Charset: GB2312,utf-8;q=0.7,*;q=0.7
    
36. Keep-Alive: 115
    
37. Proxy-Connection: keep-alive
    
38. Referer: http://www.cquae.com/search_list.aspx?ob=ProjectCenter&lx=2
    
39. Cookie: ASP.NET_SessionId=5050pcjv24rxsejce1dtbjas
    
40. Content-Type: application/x-www-form-urlencoded
    
41. Content-Length: 33393
    
42. 
    
43. 
    
44. SECOND RESOPONSE:
    
45. HTTP/1.0 200 OK
    
46. Cache-Control: private
    
47. Content-Type: text/html; charset=utf-8
    
48. Server: Microsoft-IIS/7.5
    
49. X-Powered-By: UrlRewriter.NET 2.0.0
    
50. X-AspNet-Version: 2.0.50727
    
51. X-Powered-By: ASP.NET
    
52. Date: Mon, 27 Dec 2010 03:32:32 GMT
    
53. Content-Length: 62133
    
54. Connection: keep-alive

复制代码

部分代码如下：

1. 
   
2.         $param['__EVENTARGET']="_ctl11";
   
3.         $param['__EVENTARGUMENT']="";
   
4.         $param['__EVENTVALIDATION']=$eventvalidation;
   
5.         $param['__VIEWSTATE']= $viewstate;
   
6.         $param['wd']="";
   
7.         $param['p1']="";
   
8.         $param['p2']="";
   
9.         $param['yema']="2";
   
10. 
    
11. function get_html_by_url_proxy($url,$options)
    
12. {
    
13.         global $sessionId;
    
14.         $ch = curl_init();
    
15.         if (!is_resource($ch))
    
16.         {
    
17.                 if(DEBUG ==true) log_write("error url: ".$url);
    
18.                 return false;
    
19.         }
    
20.         $proxy  = 'proxy.jgb:8081';
    
21. 
    
22.         $headary= array('Content-Type: multipart/form-data');
    
23.         echo "开启代理$proxy"."\n";
    
24.         var_dump($options);
    
25.         curl_setopt($ch, CURLOPT_PROXY, $proxy);
    
26.         //curl_setopt($ch,CURLOPT_HEADER,0);
    
27.         curl_setopt($ch,CURLOPT_HTTPHEADER,$headary);
    
28.         curl_setopt($ch,CURLOPT_URL,$url);
    
29.         curl_setopt($ch,CURLOPT_COOKIE,$sessionId);
    
30.         curl_setopt($ch,CURLOPT_POSTFIELDS,http_build_query($options));
    
31.         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    
32.         curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
    
33.         $output = curl_exec($ch) ;
    
34.         curl_close($ch);
    
35.         return $output;
    
36. }

复制代码

Galenlong

{:3_194:}