- 论坛徽章:
- 13
|
本帖最后由 hmchzb19 于 2014-07-01 13:34 编辑
Exception in thread Thread-10 (most likely raised during interpreter shutdown):
Traceback (most recent call last):
File "/usr/lib64/python2.6/threading.py", line 532, in __bootstrap_inner
File "./url_fetch_threaded_part2.py", line 44, in run
File "/usr/lib64/python2.6/Queue.py", line 168, in get
File "/usr/lib64/python2.6/threading.py", line 232, in wait
<type 'exceptions.TypeError'>: 'NoneType' object is not callable
代码如下
- #! /usr/bin/env python
- import Queue
- import threading
- import urllib2
- import time
- from BeautifulSoup import BeautifulSoup
- hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com",
- "http://ibm.com", "http://apple.com"]
- queue = Queue.Queue()
- out_queue = Queue.Queue()
- class ThreadUrl(threading.Thread):
- """Threaded Url Grab"""
- def __init__(self, queue, out_queue):
- threading.Thread.__init__(self)
- self.queue = queue
- self.out_queue = out_queue
- def run(self):
- while True:
- #grabs host from queue
- host = self.queue.get()
- #grabs urls of hosts and then grabs chunk of webpage
- url = urllib2.urlopen(host)
- chunk = url.read()
- #place chunk into out queue
- self.out_queue.put(chunk)
- #signals to queue job is done
- self.queue.task_done()
- class DatamineThread(threading.Thread):
- """Threaded Url Grab"""
- def __init__(self, out_queue):
- threading.Thread.__init__(self)
- self.out_queue=out_queue
- def run(self):
- while True:
- #grabs host from queue
- chunk = self.out_queue.get()
- #parse the chunk
- soup = BeautifulSoup(chunk)
- print soup.findAll(['title'])
- #signals to queue job is done
- self.out_queue.task_done()
- start = time.time()
- def main():
- #populate queue with data
- for host in hosts:
- queue.put(host)
- #spawn a pool of threads, and pass them queue instance
- for i in range(5):
- t = ThreadUrl(queue, out_queue)
- t.setDaemon(True)
- t.start()
- for i in range(5):
- dt = DatamineThread(out_queue)
- dt.setDaemon(True)
- dt.start()
- #wait on the queue until everything has been processed
- queue.join()
- out_queue.join()
- main()
- print "Elapsed Time: %s" % (time.time() - start)
复制代码 |
|