1、requests的post請(qǐng)求

# 1、requests 的post請(qǐng)求 import requests import re headers={'User-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36' } response=requests.get(url='https://github.com/login',headers=headers) print(response.text) 把login頁(yè)返回的cookies信息轉(zhuǎn)換成字典 login_cookies=response.cookies.get_dict() authenticity_token=re.findall(' name="authenticity_token" value="(.*?)"',response.text,re.S)[0] print(authenticity_token) #拼接請(qǐng)求頭信息 headers2={'Referer':'https://github.com/login','User-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36' } #拼接請(qǐng)求體信息 form_data={'commit':'Sign in',"utf-8":"?","authenticity_token":authenticity_token,"login":"852653835","password":"******","webauthn-support":"supported" } # 往session發(fā)送請(qǐng)求，攜帶請(qǐng)求頭、請(qǐng)求體、login頁(yè)的cookies信息 response2=requests.post(url='https://github.com/session',data=form_data,headers=headers2,cookies=login_cookies) print(response2.status_code) with open('github.html','w',encoding='utf-8')as f:f.write(response2.text)
#響應(yīng)response import requestsheaders = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.36', } # response = requests.get('https://www.github.com', headers=headers) print(response.status_code) # 獲取響應(yīng)狀態(tài)碼 print(response.url) # 獲取url地址 print(response.text) # 獲取文本 print(response.content) # 獲取二進(jìn)制流 print(response.headers) # 獲取頁(yè)面請(qǐng)求頭信息 print(response.history) # 上一次跳轉(zhuǎn)的地址 print(response.cookies) # # 獲取cookies信息 print(response.cookies.get_dict()) # 獲取cookies信息轉(zhuǎn)換成字典 print(response.cookies.items()) # 獲取cookies信息轉(zhuǎn)換成字典 print(response.encoding) # 字符編碼 print(response.elapsed) # 訪問(wèn)時(shí)間

?

?

2、requests的高級(jí)用法

# 2、requests高級(jí)用法 # https=http+ssl import requests #王音頻地址發(fā)送get請(qǐng)求 url='http://hc.yinyuetai.com/uploads/videos/common/3B7201685F78BF2954FEEB32CB6EBD82.mp4' response=requests.get(url,stream=True) #stream=True把content設(shè)置為一個(gè)迭代器對(duì)象 print(response.content) with open('music.mp4','wb')as f:for content in response.iter_content():f.write(content)

3、selenium模塊

''' '''例1''' from selenium import webdriver # 用來(lái)驅(qū)動(dòng)瀏覽器的 from selenium.webdriver import ActionChains # 破解滑動(dòng)驗(yàn)證碼的時(shí)候用的 可以拖動(dòng)圖片 from selenium.webdriver.common.by import By # 按照什么方式查找，By.ID,By.CSS_SELECTOR，By.ClassName from selenium.webdriver.common.keys import Keys # 鍵盤(pán)按鍵操作 from selenium.webdriver.support import expected_conditions as EC # 和下面WebDriverWait一起用的 from selenium.webdriver.support.wait import WebDriverWait # 等待頁(yè)面加載某些元素 # import time # 通過(guò)谷歌瀏覽器驅(qū)動(dòng)打谷歌瀏覽器 # 1、 webdriver.Chrome(r'C:\Users\85265\Downloads\chromedriver.exe') chrome=webdriver.Chrome(r'C:\Users\85265\Downloads\chromedriver.exe') chrome.get('https://www.cnblogs.com/kermitjam')#2、 chrome=webdriver.Chrome() 若try出現(xiàn)異常 try:驅(qū)動(dòng)一參數(shù)對(duì)象，驅(qū)動(dòng)二等待時(shí)間wait=WebDriverWait(chrome,10)訪問(wèn)百度chrome.get('https://www.baidu.com')查找input輸入窗input_tag=wait.until(EC.presence_of_element_located(此處可以寫(xiě)一個(gè)參數(shù)(By.ID,'kw')))#沒(méi)ID找class搜索一拳超人input_tag.send_keys('一拳超人')按鍵盤(pán)回車鍵input_tag.send_keys(Keys.ENTER)time.sleep(3) finally:chrome.close()
from selenium import webdriver # 用來(lái)驅(qū)動(dòng)瀏覽器的 from selenium.webdriver import ActionChains # 破解滑動(dòng)驗(yàn)證碼的時(shí)候用的 可以拖動(dòng)圖片 from selenium.webdriver.common.by import By # 按照什么方式查找，By.ID,By.CSS_SELECTOR，By.ClassName from selenium.webdriver.common.keys import Keys # 鍵盤(pán)按鍵操作 from selenium.webdriver.support import expected_conditions as EC # 和下面WebDriverWait一起用的 from selenium.webdriver.support.wait import WebDriverWait # 等待頁(yè)面加載某些元素 import time 通過(guò)谷歌瀏覽器驅(qū)動(dòng)打開(kāi)谷歌瀏覽器 1、 webdriver.Chrome(r'C:\Users\85265\Downloads\chromedriver.exe') chrome=webdriver.Chrome(r'C:\Users\85265\Downloads\chromedriver.exe') chrome.get('https://www.cnblogs.com/kermitjam')#2、 chrome=webdriver.Chrome() 若try出現(xiàn)異常 try:顯示等待（等待某個(gè)元素加載），驅(qū)動(dòng)一參數(shù)對(duì)象，驅(qū)動(dòng)二等待時(shí)間wait=WebDriverWait(chrome,10)訪問(wèn)百度chrome.get('https://www.jd.com')查找input輸入窗input_tag=wait.until(EC.presence_of_element_located(此處可以寫(xiě)一個(gè)參數(shù)(By.ID,'key')))#沒(méi)ID找class 搜索唐詩(shī)三百首input_tag.send_keys('唐詩(shī)三百首')根據(jù)class屬性名查找標(biāo)簽search_button=wait.until(EC.presence_of_element_located((By.CLASS_NAME,'button')))search_button.click()time.sleep(8) finally:chrome.close()

?

?

4、今日作業(yè)：

　　

'''''' ''' 爬取京東商品信息:請(qǐng)求url:https://www.jd.com/提取商品信息:1.商品詳情頁(yè)2.商品名稱3.商品價(jià)格4.評(píng)價(jià)人數(shù)5.商品商家 ''' from selenium import webdriver from selenium.webdriver.common.keys import Keys import timedef get_good(driver):try:# 通過(guò)JS控制滾輪滑動(dòng)獲取所有商品信息js_code = '''window.scrollTo(0,5000);'''driver.execute_script(js_code) # 執(zhí)行js代碼# 等待數(shù)據(jù)加載time.sleep(2)# 3、查找所有商品div# good_div = driver.find_element_by_id('J_goodsList')good_list = driver.find_elements_by_class_name('gl-item')n = 1for good in good_list:# 根據(jù)屬性選擇器查找# 商品鏈接good_url = good.find_element_by_css_selector('.p-img a').get_attribute('href')# 商品名稱good_name = good.find_element_by_css_selector('.p-name em').text.replace("\n", "--")# 商品價(jià)格good_price = good.find_element_by_class_name('p-price').text.replace("\n", ":")# 評(píng)價(jià)人數(shù)good_commit = good.find_element_by_class_name('p-commit').text.replace("\n", " ")good_content = f'''商品鏈接: {good_url}商品名稱: {good_name}商品價(jià)格: {good_price}評(píng)價(jià)人數(shù): {good_commit}\n'''print(good_content)with open('jd.txt', 'a', encoding='utf-8') as f:f.write(good_content)next_tag = driver.find_element_by_class_name('pn-next')next_tag.click()time.sleep(2)# 遞歸調(diào)用函數(shù) get_good(driver)time.sleep(10)finally:driver.close()if __name__ == '__main__':good_name = input('請(qǐng)輸入爬取商品信息:').strip()driver = webdriver.Chrome()driver.implicitly_wait(10)# 1、往京東主頁(yè)發(fā)送請(qǐng)求driver.get('https://www.jd.com/')# 2、輸入商品名稱，并回車搜索input_tag = driver.find_element_by_id('key')input_tag.send_keys(good_name)input_tag.send_keys(Keys.ENTER)time.sleep(2)get_good(driver)

?

轉(zhuǎn)載于:https://www.cnblogs.com/wjlblog/p/11041635.html

創(chuàng)作挑戰(zhàn)賽新人創(chuàng)作獎(jiǎng)勵(lì)來(lái)咯，堅(jiān)持創(chuàng)作打卡瓜分現(xiàn)金大獎(jiǎng)

總結(jié)

以上是生活随笔為你收集整理的Day5的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。