1. 如何在requests中設置ip代理

最直接的用法，在get中添加proxies設置ip代理

proxies = {'https': 'http://183.148.153.147:9999/''https': 'http://183.148.153.147:9999/'}) requests.get(url=url, headers=headers, proxies=proxies)

當ip被網站ban掉時，我們就需要使用大量的ip來進行替換，由此引出了下面的內容，爬取西刺提供的免費ip

2. 爬取西刺免費ip代理,并存入mysql

。。具體的字段分析，先待定吧，我ip被西刺ban了。。。233
直接上代碼了

import time from random import randomimport requests from scrapy.selector import Selector import pymysqlheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER" }# 創建數據庫時，使用的字段是ip(varchar)(主鍵) port(varchar) proxy_type(varchar) speed(float) ，數據庫名字(ips)，表名(ip_pond),user,passwd，這些都填自己的 conn = pymysql.connect(host='127.0.0.1', user='root', passwd='123456', db='ips', charset='utf8') cursor = conn.cursor()# 定義一個隨機延時，因為西刺這網站太容易封ip了，， def rand_sleep_time():sleep_time = random() * 100return time.sleep(sleep_time)# 用于更新ip池 def update_ip_pond():# 這個網站目前一共有3637頁，這里獲取前面的10個頁面for i in range(1, 11):resp = requests.get('https://www.xicidaili.com/nn/%s' % i, headers=headers)if resp.status_code != 200:print('第%s頁獲取失敗' % i)else:print('已獲取第%s頁內容' % i)selector = Selector(text=resp.text)# 使用xpath找到ip_list這個idall_items = selector.xpath('//*[@id="ip_list"]//tr')ip_list = []#第一行不是我們需要的，過濾掉，從第1列開始for item in all_items[1:]:# 這里使用xpath從網頁提取speed_str = item.xpath('td[7]/div/@title').get()if speed_str:speed = float(speed_str.split('秒')[0])ip = item.xpath('td[2]/text()').get()port = item.xpath('td[3]/text()').get()proxy_type = item.xpath('td[6]/text()').get().lower()ip_list.append((ip, port, proxy_type, speed))for ip_info in ip_list:# sql的作用為：插入并更新相應的字段cursor.execute("insert ip_pond(ip,port,proxy_type,speed) values ('{0}','{1}','{2}','{3}') ON DUPLICATE KEY UPDATE ip=VALUES(ip),port=VALUES(port),proxy_type=VALUES(proxy_type),speed=VALUES(speed)".format(ip_info[0], ip_info[1], ip_info[2], ip_info[3]))#延時，防拉黑rand_sleep_time()conn.commit()

3. 定義GetIp類，用于從mysql中取出ip

class GetIp(object):# 刪除不可用的Ipdef delete_ip(self, ip):delete_sql = """DELETE FROM ip_pond WHERE ip='{0}'""".format(ip)cursor.execute(delete_sql)conn.commit()return True# 驗證ip是否可用def judge_ip(self, ip, port, proxy_type):#通過百度來驗證吧http_url = 'https://www.baidu.com'proxy_url = '{0}://{1}:{2}'.format(proxy_type, ip, port)try:#對http.https進行區分if proxy_type == 'http':proxy_dict = {'http': proxy_url,}response = requests.get(http_url, proxies=proxy_dict)else:proxy_dict = {'https': proxy_url,}response = requests.get(http_url, proxies=proxy_dict, verify=False)except Exception as e:print('invalid ip and port')self.delete_ip(ip)return Falseelse:code = response.status_codeif code >= 200 and code < 300:print('effective ip')return Trueelse:print('invalid ip and port')self.delete_ip(ip)return False# 從數據庫中隨機選擇def get_random_ip(self):random_sql = """SELECT ip,port,proxy_type,speed FROM ip_pond ORDER BY RAND() LIMIT 1"""cursor.execute(random_sql)for ip_info in cursor.fetchall():ip = ip_info[0]port = ip_info[1]proxy_type = ip_info[2]judge_re = self.judge_ip(ip, port, proxy_type)if judge_re:return '{0}://{1}:{2}'.format(proxy_type, ip, port)else:return self.get_random_ip()# 從數據庫中選速度最快的 (大部分和上面的一樣，只是sql語句不一樣)def get_optimum_ip(self):optimum_sql = """SELECT ip,port,proxy_type,speed FROM ip_pond ORDER BY speed LIMIT 1"""cursor.execute(optimum_sql)for ip_info in cursor.fetchall():ip = ip_info[0]port = ip_info[1]proxy_type = ip_info[2]judge_re = self.judge_ip(ip, port, proxy_type)if judge_re:return '{0}://{1}:{2}'.format(proxy_type, ip, port)else:return self.get_optimum_ip()#對獲取的ip簡單封裝了下，方便使用def get_proxies(self):getip = GetIp()ip = getip.get_random_ip()print(ip)proxy_type = ip.split(':')[0]proxies = {proxy_type: ip}return proxies

4. 正確的使用方式

if __name__ == '__main__':# 當取到的ip是https的時候，會有點慢# 先確認是否存在ip_pond這表sql = """SELECT * FROM ip_pond"""check_table = cursor.execute(sql)if check_table:#測試用的url，這個隨便寫url = 'https://www.baidu.com'headers = {"User-Agent": "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"}#上一步的簡單封裝，直接獲取到proxiesproxies = GetIp().get_proxies()res = requests.get(url=url, headers=headers, proxies=proxies)else:update_ip_pond()

源碼請點擊這里

總結

以上是生活随笔為你收集整理的python 爬取西刺ip代理池的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。