該樓層疑似違規(guī)已被系統(tǒng)折疊?隱藏此樓查看此樓

#-*- coding: UTF-8 -*-

import scrapy

from hoho.items import HohoItem

import re

from scrapy.selector import Selector

import sys

reload(sys)

sys.setdefaultencoding( "UTF-8" )

class tongSpider(scrapy.Spider):

name = 'guwen'

start_urls=['http://www.shicifuns.com/v2/wenyan/list']

def parse(self,response):

papers = response.xpath('//div[@class="css_content"]/div/div[@class="css_body_left"]/div[@class="every_day"]/ul')

for paper in papers:

for p in paper.xpath('li'):

name = p.xpath('a/div/div[@class="poem_title"]/span/text()').extract()[0]

url = p.xpath('a/@href').extract()[0]

content = p.xpath('a/div/div[@class="poem_content"]/text()').extract()[0].strip("\r\n ")

author = p.xpath('a/div/div[@class="poem_info"]/span[@class="dynasty"]/text()').extract()[0]

pinfen = p.xpath('a/div/div[@class="poem_info"]/span[@class="dynasty"]/text()').extract()[1]

item = HohoItem(name = name,url="http://www.shicifuns.com"+url,content=content,author=author,pinfen=pinfen)

yield item

next = response.xpath("//div[@class='css_content']/div/div[@class='css_body_left']/div[@class='pagination']/ul/li/a[@class='next page focus']/@href").extract()

if next:

yield scrapy.Request(url = "http://www.shicifuns.com" + next[0],callback=self.parse)

與50位技術(shù)專家面對(duì)面20年技術(shù)見證，附贈(zèng)技術(shù)全景圖

總結(jié)

以上是生活随笔為你收集整理的python 爬虫爬不出来_爬虫爬不进下一页了，怎么办的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。