結(jié)果演示

代碼：

#encoding:utf-8 # __author__ = 'donghao' # __time__ = 2018/12/24 15:20 import requests import threading import urllib.request import urllib3 import os import re import time from lxml import etree from queue import Queue#負(fù)責(zé)解析圖片 class Producer(threading.Thread):headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:57.0) Gecko/20100101 Firefox/57.0'}def __init__(self,img_queue,page_queue,*args,**kwargs):super(Producer, self).__init__(*args,**kwargs)self.img_queue = img_queueself.page_queue = page_queuedef run(self):while True:if self.page_queue.empty():breakurl = self.page_queue.get()self.parse_page(url)def parse_page(self,url):resp = requests.get(url=url,headers=self.headers)text = resp.texthtml = etree.HTML(text)imgs = html.xpath("//div[@class='page-content text-center']//img[@class!='gif']")for img in imgs:#獲取圖片urlimg_url = img.get('data-original')#獲取圖片Url的后綴名end = os.path.splitext(img_url)[1]#替換掉url中特殊字符end = re.sub(r'[，。?？,/\\·]','',end)# 獲取圖片描述,并加上后綴name = img.get('alt')+end#存儲(chǔ)圖片url和文件名隊(duì)列self.img_queue.put((img_url,name))#負(fù)責(zé)下載圖片 class Consumer(threading.Thread):def __init__(self,img_queue,page_queue,*args,**kwargs):super(Consumer, self).__init__(*args,**kwargs)self.img_queue = img_queueself.page_queue = page_queuedef run(self):while True:if self.img_queue.empty() and self.page_queue.empty():breakimg_url,filename = self.img_queue.get()urllib.request.urlretrieve(img_url, 'images/' + filename)print(filename+'張圖片下載完成')def main():#爬取10頁(yè)start = time.time()image_queue = Queue(1000)page_queue = Queue(100)tsk = []for x in range(1,10):url = 'http://www.doutula.com/photo/list/?page=%d'%x#存儲(chǔ)頁(yè)面信息page_queue.put(url)for x in range(5):t = Producer(image_queue,page_queue)t.start()tsk.append(t)for x in range(5):t = Consumer(image_queue,page_queue)t.start()tsk.append(t)#終止運(yùn)行，統(tǒng)計(jì)時(shí)間for t in tsk:t.join()end = time.time()print('耗時(shí)：%0.002fs' % (end - start))if __name__ == '__main__':main()

?

總結(jié)

以上是生活随笔為你收集整理的多线程爬取斗图图片的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。