看到網上其他同學的思路是爬取所有歌單，然后篩選出評論過萬的歌曲。但我覺得不同歌單之間會有交叉，這種方式可能效率不高，而且可能會有漏網之魚。所以我準備爬取所有歌手，再爬取他們的熱門50單曲，從中篩選評論過萬的歌曲。現階段幾乎沒有歌手有超過50首評論過萬的歌曲，所以這種方法目前是可行的。

查看歌手頁面，歌手被分成了華語男歌手、華語女歌手、歐美男歌手……共計15個板塊，板塊代號如下：

group = ['1001', '1002', '1003', '2001', '2002', '2003', '6001', '6002', '6003', '7001', '7002', '7003', '4001', '4002', '4003']

而每個板塊又按照首字母分成了27個子頁面（包括熱門歌手頁面），子頁面代號如下：

initial = ['0'] for i in range(65, 91):initial.append(str(i))

15*27=405，我們要爬取405個歌手子頁面，可以利用上述代號拼接出這405個歌手子頁面鏈接：

urls = [] for g in group:for i in initial:url = 'http://music.163.com/discover/artist/cat?id=' + g + '&initial=' + iurls.append(url)

然后就是用爬蟲從這些頁面上爬取歌手的id：

def get_artist(url):k = 0t = []while True:try:resp = request.urlopen(url)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')l = soup.find_all('a', class_='nm nm-icn f-thide s-fc0')p = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:t.append(re.match(p, i['href']).group(1))return texcept Exception as e:print(e)k += 1if k > 10:print('頁面' + url + '發生錯誤')return Nonet = []continue

獲得歌手id以后，再讓爬蟲爬取歌手的個人頁面，獲取熱門50單曲的歌曲id：

def get_song(artist_id):k = 0t = []while True:url = 'http://music.163.com/artist?id=' + artist_idtry:req = request.Request(url)req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4399.400 QQBrowser/9.7.12777.400')req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')resp = request.urlopen(req)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')except Exception as e:k += 1if k > 10:print('歌手' + artist_id + '發生錯誤')print(e)return Nonecontinuetry:a = soup.find('ul', class_='f-hide')l = a.childrenp = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:music_id = re.match(p, i.a['href']).group(1)data = (music_id, artist_id)t.append(data)return texcept Exception as e:print(e)print('歌手' + artist_id + '發生錯誤')return None

利用歌曲id訪問歌曲頁面，獲取歌曲評論數，這里遇到了難點。評論信息都是動態加載的，直接獲取評論數的結果總是0，所以這里借鑒了知乎用戶平胸小仙女的回答，方法如下：

# -*- coding: utf-8 -*- from Crypto.Cipher import AES import base64 import requests import json import codecs import time import random#代理ip proxy_host = '122.72.18.35' proxy = {'http':proxy_host}# 頭部信息 headers={'Host':'music.163.com','Accept':'*/*','Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3','Accept-Encoding':'gzip, deflate','Content-Type':'application/x-www-form-urlencoded','Referer':'http://music.163.com/song?id=347597','Content-Length':'484','Cookie':'__s_=1; _ntes_nnid=f17890f7160fd145486752ebbf2066df,1505221478108; _ntes_nuid=f17890f7160fd145486752ebbf2066df; JSESSIONID-WYYY=Z99pE%2BatJVOAGco1d%2FJpojOK94Xe9GHqe0epcCOj23nqP2SlHt1XwzWQ2FXTwaM2xgIN628qJGj8%2BikzfYkv%2FXAUo%2FSzwMxjdyO9oeQlGKBvH6nYoFpJpVlA%2F8eP57fkZAVEsuB9wqkVgdQc2cjIStE1vyfE6SxKAlA8r0sAgOnEun%2BV%3A1512200032388; _iuqxldmzr_=32; __utma=94650624.1642739310.1512184312.1512184312.1512184312.1; __utmc=94650624; __utmz=94650624.1512184312.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); playerid=10841206','Connection':'keep-alive','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}# offset的取值為:(評論頁數-1)*20,total第一頁為true，其余頁為false first_param = '{rid:"", offset:"0", total:"true", limit:"20", csrf_token:""}' second_param = "010001" third_param = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7" forth_param = "0CoJUm6Qyw8W8jud"# 獲取參數 def get_params(page): # page為傳入頁數iv = "0102030405060708"first_key = forth_paramsecond_key = 16 * 'F'if(page == 1): # 如果為第一頁first_param = '{rid:"", offset:"0", total:"true", limit:"20", csrf_token:""}'h_encText = AES_encrypt(first_param, first_key, iv)else:offset = str((page-1)*20)first_param = '{rid:"", offset:"%s", total:"%s", limit:"20", csrf_token:""}' %(offset,'false')h_encText = AES_encrypt(first_param, first_key, iv)h_encText = AES_encrypt(h_encText, second_key, iv)return h_encText# 獲取 encSecKey def get_encSecKey():encSecKey = "257348aecb5e556c066de214e531faadd1c55d814f9be95fd06d6bff9f4c7a41f831f6394d5a3fd2e3881736d94a02ca919d952872e7d0a50ebfa1769a7a62d512f5f1ca21aec60bc3819a9c3ffca5eca9a0dba6d6f7249b06f5965ecfff3695b54e1c28f3f624750ed39e7de08fc8493242e26dbc4484a01c76f739e135637c"return encSecKey# 解密過程 def AES_encrypt(text, key, iv):pad = 16 - len(text) % 16text = text + pad * chr(pad)encryptor = AES.new(key, AES.MODE_CBC, iv)encrypt_text = encryptor.encrypt(text)encrypt_text = base64.b64encode(encrypt_text)encrypt_text = str(encrypt_text, encoding="utf-8") #注意一定要加上這一句，沒有這一句則出現錯誤return encrypt_textdef get_json(url, params, encSecKey):data = {"params": params,"encSecKey": encSecKey}response = requests.post(url, headers=headers, data=data, proxies=proxy)return response.content#外部調用方法 def get_comments_total(id):url = 'http://music.163.com/weapi/v1/resource/comments/R_SO_4_'+str(id)+'?csrf_token='params = get_params(1)encSecKey = get_encSecKey()json_text = get_json(url,params,encSecKey)json_dict = json.loads(json_text)comments_num = int(json_dict['total'])return comments_num

最后再將獲得的數據逐條寫入數據庫就可以了
總的代碼如下：

# _*_ coding: utf-8 _*_ from urllib import request import requests import json from bs4 import BeautifulSoup as bs from Crypto.Cipher import AES import base64 import re import mysql.connector import get_comments_total as gct import threadinggroup = ['1001', '1002', '1003', '2001', '2002', '2003', '6001', '6002', '6003', '7001', '7002', '7003', '4001', '4002','4003']initial = ['0'] for i in range(65, 91):initial.append(str(i))urls = [] for g in group:for i in initial:url = 'http://music.163.com/discover/artist/cat?id=' + g + '&initial=' + iurls.append(url)#寫入數據庫 def write(L):try:conn = mysql.connector.connect(user='root', password='lixiao187.', database='cloudmusic', charset='utf8')cursor = conn.cursor()for l in L:try:cursor.execute('insert into music(music_id, music_name, artist_id, artist_name, comments) values (%s, %s, %s, %s, %s)',l)conn.commit()except Exception as e:print(e)print(l)continuecursor.close()conn.close()except Exception as e:print(e)print(L)# 獲得某字母頁面上的歌手id列表 def get_artist(url):k = 0t = []while True:try:resp = request.urlopen(url)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')l = soup.find_all('a', class_='nm nm-icn f-thide s-fc0')p = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:t.append(re.match(p, i['href']).group(1))return texcept Exception as e:print(e)k += 1if k > 10:print('頁面' + url + '發生錯誤')return Nonet = []continue# 獲得某歌手的熱門歌曲id列表 def get_song(artist_id):k = 0t = []while True:url = 'http://music.163.com/artist?id=' + artist_idtry:req = request.Request(url)req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4399.400 QQBrowser/9.7.12777.400')req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')resp = request.urlopen(req)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')except Exception as e:k += 1if k > 10:print('歌手' + artist_id + '發生錯誤')print(e)return Nonecontinuetry:a = soup.find('ul', class_='f-hide')l = a.childrenp = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:music_id = re.match(p, i.a['href']).group(1)data = (music_id, artist_id)t.append(data)return texcept Exception as e:print(e)print('歌手' + artist_id + '發生錯誤')return None# 獲得全部所需信息 def get_data(music_id, artist_id):k = 0while True:try:comments = gct.get_comments_total(music_id)print('歌曲'+music_id+'，評論數：'+str(comments))if comments < 10000:return Noneurl = 'http://music.163.com/song?id=' + music_iddata = []req = request.Request(url)req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4399.400 QQBrowser/9.7.12777.400')req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')resp = request.urlopen(req)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')d = soup.find('div', class_='tit')p = soup.find('p', class_='des s-fc4')s = soup.find('span', class_='j-flag')music_name = d.em.textartist_name = p.span['title']data.append(music_id)data.append(music_name)data.append(artist_id)data.append(artist_name)data.append(comments)return dataexcept Exception as e:k += 1if k > 10:print('歌曲' + music_id + '發生錯誤')return Nonecontinue# 逐條寫入 def get_and_write(artists, name):data = []for a in artists:songs = get_song(a)if songs == None:continuefor s in songs:d = get_data(s[0], a)if d == None:continuedata.append(d)if len(data) > 0:write(data)# 歌手子頁面爬取線程 def crawl(url, name):L = []artists = get_artist(url)if artists == None:returnfor a in artists:L.append(a)if len(L) > 9:t = threading.Thread(target=get_and_write, args=(L, ''))t.start()L = []t = threading.Thread(target=get_and_write, args=(L, ''))t.start()# 總方法 def threads_crawl(start, end):L = []for i in range(start - 1, end):t = threading.Thread(target=crawl, args=(urls[i], ''))L.append(t)for t in L:t.start()for t in L:t.join()threads_crawl(1, 405)

總結

以上是生活随笔為你收集整理的爬取网易云音乐评论过万歌曲的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。