- 论坛徽章:
- 3
|
本帖最后由 venux 于 2018-06-26 15:42 编辑
#coding:utf-8
from gevent import monkey; monkey.patch_socket()
import requests
from bs4 import BeautifulSoup
import os
import gevent
from gevent.queue import Queue
from multiprocessing import Process
import time
'''
#############################
'''
def xiancheng_def(urls):
gevent.joinall([gevent.spawn(get_pic_from_tag_url,i) for i in urls])
def jincheng_def(tasks):
flag=8
urls=[]
i=0
processes=[]
while not tasks.empty():
i+=1
urls.append(tasks.get())
if i == flag:
p=Process(target=xiancheng_def,args=(urls,))
processes.append(p)
urls=[]
i=0
if urls:
p=Process(target=xiancheng_def,args=(urls,))
processes.append(p)
print processes
map(Process.start,tuple(processes))
if __name__ == '__main__':
main_path=raw_input("请如入保存路径 :")
tasks = Queue()
session = requests.Session()
Entry_url='http://www.xxx.com/ent//' #
all_tag_url_queue(Entry_url)
print tasks
# for i in range(17):
# tasks.get()
# gevent.joinall([
# gevent.spawn(get_pic_from_tag_url),
# gevent.spawn(get_pic_from_tag_url),
# gevent.spawn(get_pic_from_tag_url),
# gevent.spawn(get_pic_from_tag_url),
# gevent.spawn(get_pic_from_tag_url),
# gevent.spawn(get_pic_from_tag_url),
# ])
jincheng_def(tasks)
每次都执行一个进程 ,然后报错如下
[err] evmap.c:401: Assertion ctx failed in evmap_io_active
[err] evmap.c:401: Assertion ctx failed in evmap_io_active
使用单进程+gevent没有代码没有问题,网上查不到有关信息,请帮忙看看
|
|