python 抓图片的两段程序

yanite

请教下有没有更智能，更一般化的方法。

1. # -*- coding: utf-8 -*-
   
2. #coding=utf-8
   
3. #vim:fdm=marker:ts=2
   
4. 
   
5. import os
   
6. from os.path import join, getsize
   
7. 
   
8. import re, os, shutil, sys, codecs
   
9. 
   
10. streamWriter=codecs.lookup('utf-8')[-1]
    
11. #sys.stdout = codecs.getwriter(locale.getdefaultlocale()[1])(sys.stdout, 'replace')
    
12. sys.stdout=streamWriter(sys.stdout)
    
13. reload(sys)
    
14. sys.setdefaultencoding('utf8')
    
15. sys.path.append('.')
    
16. sys.path.append('..')
    
17. 
    
18. 
    
19. import string, time, hashlib, types, logging, json, pdb
    
20. 
    
21. #import psyco
    
22. #psyco.full()
    
23. #psyco.profile()
    
24. #psyco.log()
    
25. #from psyco.classes import *
    
26. 
    
27. #from xml.dom import minidom
    
28. #import poplib
    
29. #import email
    
30. 
    
31. from lxml import etree
    
32. 
    
33. from urllib2 import Request, urlopen, URLError, HTTPError
    
34. import urllib, urllib2
    
35. 
    
36. import decimal, datetime
    
37. 
    
38. import socket
    
39. import httplib
    
40. import StringIO
    
41. 
    
42. #import dumper
    
43. import cookielib
    
44. import lxml.html as HTML
    
45. 
    
46. debug_info = 0
    
47. import socket
    
48. socket.setdefaulttimeout(60)
    
49. httplib.HTTPConnection.debuglevel = 1
    
50. 
    
51. class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
    
52.     def http_error_301(self, req, fp, code, msg, headers):
    
53.         result = urllib2.HTTPRedirectHandler.http_error_301(
    
54.             self, req, fp, code, msg, headers)
    
55.         result.status = code
    
56.         return result
    
57. 
    
58.     def http_error_302(self, req, fp, code, msg, headers):
    
59.         # if (req.get_host() == 'container.open.taobao.com'):
    
60.         #return
    
61.         result = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
    
62.         result.status = code
    
63.         return result
    
64. 
    
65. class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
    
66.         def http_error_default(self, req, fp, code, msg, headers):
    
67.                 #print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' ,  code,  headers['Location']
    
68.                 result = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
    
69.                 result.status = code
    
70.                 return result
    
71. 
    
72. # =====================================================================================================================================
    
73. 
    
74. domain = "wallpaper.pconline.com.cn"
    
75. #domain = "219.136.245.183:80"
    
76. root_url = "http://" + domain + "/"
    
77. # 3 is
    
78. img_url1 = root_url + "cate_latest/5/$page_number$.html"
    
79. 
    
80. xdoc_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookielib.CookieJar()), SmartRedirectHandler(), DefaultErrorHandler())
    
81. xdoc_opener.handle_open["http"][0].set_http_debuglevel(debug_info)
    
82. urllib2.install_opener(xdoc_opener)
    
83. 
    
84. vfold = "f:/siteimage/"
    
85. 
    
86. def getitem(url, xpath):
    
87.         global xdoc_opener
    
88.         root_request = urllib2.Request(url)
    
89.         root_page = xdoc_opener.open(root_request)
    
90.         items_list_html = root_page.read()
    
91.         # items_list_html = items_list_html.decode("gbk").encode("UTF-8")
    
92.         lxml  = HTML.document_fromstring(items_list_html)
    
93.         nodes = lxml.getroottree().xpath(xpath)
    
94.         return nodes
    
95. # =====================================================================================================================================
    
96. 
    
97. # every image list html
    
98. for page_number in range(1, 15):
    
99. 
    
100.         list_url = img_url1.replace("$page_number$", str(page_number))
     
101.         list_xpath = "/html/body/div[3]/div/div[2]/ul/li[*]/p/span/a/@href"
     
102. 
     
103.         nodes1 = getitem(list_url, list_xpath)
     
104.         print str(page_number) + " 1: " + list_url
     
105.         for img_url2 in nodes1:
     
106. 
     
107.                 # conve to xtree
     
108.                 #img_url2 = root_url + "photo/3591.html"
     
109.                 single_pageit = "id('scroll')/ul/li/p/a/@href"
     
110.                 #single_pageit = "/html/body/div[3]/div/div/div[2]/div[2]/ul"
     
111. 
     
112.                 nodes = getitem(img_url2, single_pageit)
     
113.                 print "\t\t 2: " + img_url2 + "   " + str(len(nodes)) + r"项"
     
114.                 print
     
115. 
     
116.                 if (len(nodes) < 1):
     
117.                         continue
     
118. 
     
119.                 subfold = img_url2.split("/")[-1]
     
120.                 if (not os.path.isdir(vfold + subfold)):
     
121.                         os.mkdir(vfold + subfold)
     
122. 
     
123.                 # every image html
     
124.                 for inode in nodes:
     
125.                         img_single_pageit = "/html/body/table/tr[2]/td/img/@src"
     
126. 
     
127.                         sub_img_url = root_url + inode.replace("../", "")
     
128.          
     
129.                         inodes = getitem(sub_img_url, img_single_pageit)
     
130.                        
     
131.                         file_name = inodes[0].split("/")[-1]
     
132. 
     
133.                         filelist = os.listdir(vfold + subfold + "/")
     
134.                         if (filelist.count(file_name) > 0):
     
135.                                 #print "\t\t\t is exits " + file_name + "."
     
136.                                 continue
     
137. 
     
138.                         print "\t\t\t 3: " + sub_img_url
     
139.                         if (len(inodes) != 1):
     
140.                                 continue
     
141.                         drc_img_url =  urllib2.Request(inodes[0])
     
142.                         root_page = xdoc_opener.open(drc_img_url)
     
143.                         img = root_page.read()
     
144.                         f = file( vfold + subfold + "/" + file_name , "wb")
     
145.                         f.write(img)
     
146.                         f.close()
     
147.                         print "\t\t\t save pic " + vfold + subfold + "/" + file_name
     
148. 
     
149.         #break
     
150. 
     

复制代码

1. 
   
2. # -*- coding: utf-8 -*-
   
3. #coding=utf-8
   
4. #vim:fdm=marker:ts=2
   
5. 
   
6. import os
   
7. from os.path import join, getsize
   
8. 
   
9. import re, os, shutil, sys, codecs
   
10. 
    
11. streamWriter=codecs.lookup('utf-8')[-1]
    
12. #sys.stdout = codecs.getwriter(locale.getdefaultlocale()[1])(sys.stdout, 'replace')
    
13. sys.stdout=streamWriter(sys.stdout)
    
14. reload(sys)
    
15. sys.setdefaultencoding('utf8')
    
16. sys.path.append('.')
    
17. sys.path.append('..')
    
18. 
    
19. 
    
20. import string, time, hashlib, types, logging, json, pdb
    
21. 
    
22. #import psyco
    
23. #psyco.full()
    
24. #psyco.profile()
    
25. #psyco.log()
    
26. #from psyco.classes import *
    
27. 
    
28. #from xml.dom import minidom
    
29. #import poplib
    
30. #import email
    
31. 
    
32. from lxml import etree
    
33. 
    
34. from urllib2 import Request, urlopen, URLError, HTTPError
    
35. import urllib, urllib2
    
36. 
    
37. import decimal, datetime
    
38. 
    
39. import socket
    
40. import httplib
    
41. import StringIO
    
42. 
    
43. #import dumper
    
44. import cookielib
    
45. import lxml.html as HTML
    
46. 
    
47. debug_info = 0
    
48. 
    
49. socket.setdefaulttimeout(60)
    
50. httplib.HTTPConnection.debuglevel = 1
    
51. 
    
52. class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
    
53.     def http_error_301(self, req, fp, code, msg, headers):
    
54.         result = urllib2.HTTPRedirectHandler.http_error_301(
    
55.             self, req, fp, code, msg, headers)
    
56.         result.status = code
    
57.         return result
    
58. 
    
59.     def http_error_302(self, req, fp, code, msg, headers):
    
60.         # if (req.get_host() == 'container.open.taobao.com'):
    
61.         #return
    
62.         result = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
    
63.         result.status = code
    
64.         return result
    
65. 
    
66. class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
    
67.         def http_error_default(self, req, fp, code, msg, headers):
    
68.                 #print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' ,  code,  headers['Location']
    
69.                 result = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
    
70.                 result.status = code
    
71.                 return result
    
72. 
    
73. # =====================================================================================================================================
    
74. 
    
75. domain = "wallpaper.pconline.com.cn"
    
76. root_url = "http://" + domain + "/"
    
77. # 3 is
    
78. img_url = root_url + "/cate_latest/5/$page_number$.html"
    
79. 
    
80. xdoc_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookielib.CookieJar()), SmartRedirectHandler(), DefaultErrorHandler())
    
81. xdoc_opener.handle_open["http"][0].set_http_debuglevel(debug_info)
    
82. urllib2.install_opener(xdoc_opener)
    
83. 
    
84. def getitem(url, xpath, xpath1 = "",  xpath2= ""):
    
85.         global xdoc_opener
    
86.         root_page = xdoc_opener.open(urllib2.Request(url))
    
87.         items_list_html = root_page.read()
    
88.         # items_list_html = items_list_html.decode("gbk").encode("UTF-8")
    
89.         # print items_list_html
    
90.         lxml  = HTML.document_fromstring(items_list_html)
    
91.         nodes = lxml.getroottree().xpath(xpath)
    
92. 
    
93.         if (xpath1 != ""):
    
94.                 nodes.extend(lxml.getroottree().xpath(xpath1))
    
95.         if (xpath2 != ""):
    
96.                 nodes.extend(lxml.getroottree().xpath(xpath2))
    
97. 
    
98. 
    
99.         return nodes
    
100. # =====================================================================================================================================
     
101. 
     
102. 
     
103. domain = "www.qqqse.com"
     
104. root_url = "http://" + domain + ""
     
105. img_url = root_url + "/html/tupianqu/yazhousetu/list_5_$page_number$.html"
     
106. 
     
107. vfold = "f:/x2/"
     
108. 
     
109. 
     
110. for page_number in range(1, 60):
     
111.         list_url = img_url.replace("$page_number$", str(page_number))
     
112.         list_xpath = "/html/body/div[3]/div[2]/table/tr[1]/td/div/li[*]/a/@href"
     
113.         nodes1 = getitem(list_url, list_xpath)
     
114. 
     
115.         print "1: " + list_url
     
116.         if (len(nodes1) < 1):
     
117.                 print "no item " + str(len(nodes1))
     
118.                 break
     
119. 
     
120.         for img_list in nodes1:
     
121.                 img_list_xpath2 = "/html/body/div[3]/div[2]/table/tr/td/div/div[2]/table/tr/td/img/@src"
     
122.                 img_list_xpath1 = "/html/body/div[3]/div[2]/table/tr/td/div/div[2]/table/tr/td/blockquote/img/@src"
     
123.                 img_list_xpath  = "/html/body/div[3]/div[2]/table/tr/td/div/div[2]/table/tr/td/div/div/img/@src"
     
124. 
     
125.                 img_nodes = getitem(root_url + img_list, img_list_xpath, img_list_xpath1, img_list_xpath2)
     
126.                 if (len(img_nodes) < 1):
     
127.                         print "no items 2: " + str(len(img_nodes))
     
128.                         break
     
129. 
     
130.                 print "\t2: " + img_list + " " + str(len(img_nodes))
     
131.                 subfold = img_list.split("/")[-1]
     
132.                 if ("41517.html" == subfold):
     
133.                         continue
     
134.                 if (not os.path.isdir(vfold + subfold)):
     
135.                         os.mkdir(vfold + subfold)
     
136. 
     
137.                 for uimg in img_nodes:
     
138.                         file_name = uimg.split("/")[-1]
     
139.                         print "\t\t3: " + uimg
     
140. 
     
141.                         filelist = os.listdir(vfold + subfold + "/")
     
142.                         if (filelist.count(file_name) > 0):
     
143.                                 #print "\t\t\t is exits " + file_name + "."
     
144.                                 continue                       
     
145.                         try:
     
146.                                 root_page = xdoc_opener.open(urllib2.Request(uimg))
     
147.                                 img = root_page.read()
     
148.                                 f = file(vfold + subfold + "/" + file_name, "wb")
     
149.                                 f.write(img)
     
150.                                 f.close()
     
151.                                 print "\t\t\t save pic " + vfold + subfold + "/" + file_name
     
152.                         except :
     
153.                                 # print sys.exec_info()
     
154.                                 continue
     
155.                         #break
     
156.                 #break
     

复制代码

yanite

晕， 这个代码不能宽一点么。 伤心。

savagejin

有什么方法处理html中的javascript代码？有些网站是将图片地址放在javascript中的