一直想練習(xí)下java多線(xiàn)程抓取數(shù)據(jù)。

有天被我發(fā)現(xiàn)，鈴聲多多的官網(wǎng)(http://www.shoujiduoduo.com/main/)有大量的數(shù)據(jù)。

通過(guò)觀察他們前端獲取鈴聲數(shù)據(jù)的ajax

http://www.shoujiduoduo.com/ringweb/ringweb.php?type=getlist&listid={類(lèi)別ID}&page={分頁(yè)頁(yè)碼}

很容易就能發(fā)現(xiàn)通過(guò)改變 listId和page就能從服務(wù)器獲取鈴聲的json數(shù)據(jù)， 通過(guò)解析json數(shù)據(jù)，

可以看到都帶有{"hasmore":1,"curpage":1}這樣子的指示，通過(guò)判斷hasmore的值，決定是否進(jìn)行下一頁(yè)的抓取。

但是通過(guò)上面這個(gè)鏈接返回的json中不帶有鈴聲的下載地址

很快就可以發(fā)現(xiàn)，點(diǎn)擊頁(yè)面的“下載”會(huì)看到

通過(guò)下面的請(qǐng)求，就可以獲取鈴聲的下載地址了

http://www.shoujiduoduo.com/ringweb/ringweb.php?type=geturl&act=down&rid={鈴聲ID}

所以，他們的數(shù)據(jù)是很容易被偷的。于是我就開(kāi)始...

github：https://github.com/yongbo000/DuoduoAudioRobot

上代碼：

package me.yongbo.DuoduoRingRobot;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.URL;

import java.net.URLConnection;

import java.util.Iterator;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import com.google.gson.Gson;

import com.google.gson.JsonArray;

import com.google.gson.JsonElement;

import com.google.gson.JsonParser;

/* * @author yongbo_ * @created 2013/4/16 * * */

public class DuoduoRingRobotClient implements Runnable {

public static String GET_RINGINFO_URL = "http://www.shoujiduoduo.com/ringweb/ringweb.php?type=getlist&listid=%1$d&page=%2$d";

public static String GET_DOWN_URL = "http://www.shoujiduoduo.com/ringweb/ringweb.php?type=geturl&act=down&rid=%1$d";

public static String ERROR_MSG = "listId為 %1$d 的Robot發(fā)生錯(cuò)誤，已自動(dòng)停止。當(dāng)前page為 %2$d";public static String STATUS_MSG = "開(kāi)始抓取數(shù)據(jù)，當(dāng)前l(fā)istId： %1$d，當(dāng)前page： %2$d";

public static String FILE_DIR = "E:/RingData/";public static String FILE_NAME = "listId=%1$d.txt";private boolean errorFlag = false;private int listId;private int page;

private int endPage = -1;private int hasMore = 1;

private DbHelper dbHelper;

/** * 構(gòu)造函數(shù) * @param listId 菜單ID * @param page 開(kāi)始頁(yè)碼 * @param endPage 結(jié)束頁(yè)碼 * */

public DuoduoRingRobotClient(int listId, int beginPage, int endPage)

{this.listId = listId;this.page = beginPage;this.endPage = endPage;this.dbHelper = new DbHelper();}

/** * 構(gòu)造函數(shù) * @param listId 菜單ID * @param page 開(kāi)始頁(yè)碼 * */

public DuoduoRingRobotClient(int listId, int page) {this(listId, page, -1);}

/** * 獲取鈴聲 * */public void getRings() {String url = String.format(GET_RINGINFO_URL, listId, page);String responseStr = httpGet(url);hasMore = getHasmore(responseStr);

page = getNextPage(responseStr);

ringParse(responseStr.replaceAll("\\{\"hasmore\":[0-9]*,\"curpage\":[0-9]*\\},", "").replaceAll(",]", "]"));}/** * 發(fā)起http請(qǐng)求 * @param webUrl 請(qǐng)求連接地址 * */public String httpGet(String webUrl){URL url;URLConnection conn;StringBuilder sb = new StringBuilder();String resultStr = "";try {url = new URL(webUrl);conn = url.openConnection();conn.connect();InputStream is = conn.getInputStream();InputStreamReader isr = new InputStreamReader(is);BufferedReader bufReader = new BufferedReader(isr);String lineText;while ((lineText = bufReader.readLine()) != null) {sb.append(lineText);}resultStr = sb.toString();} catch (Exception e) {errorFlag = true;//將錯(cuò)誤寫(xiě)入txtwriteToFile(String.format(ERROR_MSG, listId, page));}return resultStr;}/** * 將json字符串轉(zhuǎn)化成Ring對(duì)象，并存入txt中 * @param json Json字符串 * */public void ringParse(String json) {Ring ring = null;JsonElement element = new JsonParser().parse(json);JsonArray array = element.getAsJsonArray();// 遍歷數(shù)組Iterator it = array.iterator();

Gson gson = new Gson();while (it.hasNext() && !errorFlag) {JsonElement e = it.next();// JsonElement轉(zhuǎn)換為JavaBean對(duì)象ring = gson.fromJson(e, Ring.class);ring.setDownUrl(getRingDownUrl(ring.getId()));if(isAvailableRing(ring)) {System.out.println(ring.toString());

//可選擇寫(xiě)入數(shù)據(jù)庫(kù)還是寫(xiě)入文本//writeToFile(ring.toString());writeToDatabase(ring);}}}

/** * 寫(xiě)入txt * @param data 字符串 * */public void writeToFile(String data)

{String path = FILE_DIR + String.format(FILE_NAME, listId);File dir = new File(FILE_DIR);File file = new File(path);FileWriter fw = null;if(!dir.exists()){dir.mkdirs();

}try {if(!file.exists()){file.createNewFile();}fw = new FileWriter(file, true);

fw.write(data);fw.write("\r\n");fw.flush();} catch (IOException e) {

// TODO Auto-generated catch blocke.printStackTrace();

}finally {try {if(fw != null){fw.close();}} catch (IOException e) {

// TODO Auto-generated catch blocke.printStackTrace();}}}/** * 寫(xiě)入數(shù)據(jù)庫(kù) * @param ring 一個(gè)Ring的實(shí)例 * */

public void writeToDatabase(Ring ring) {dbHelper.execute("addRing", ring);}

@Overridepublic void run() {while(hasMore == 1 && !errorFlag){if(endPage != -1){if(page > endPage) { break; }}System.out.println(String.format(STATUS_MSG, listId, page));

getRings();System.out.println(String.format("該頁(yè)數(shù)據(jù)寫(xiě)入完成"));}System.out.println("ending...");}

private int getHasmore(String resultStr){Pattern p = Pattern.compile("\"hasmore\":([0-9]*),\"curpage\":([0-9]*)");

Matcher match = p.matcher(resultStr);

if (match.find()) { return Integer.parseInt(match.group(1));

} return 0;

}

private int getNextPage(String resultStr){Pattern p = Pattern.compile("\"hasmore\":([0-9]*),\"curpage\":([0-9]*)");Matcher match = p.matcher(resultStr);if (match.find()) {return Integer.parseInt(match.group(2));}return 0;}

/** * 判斷當(dāng)前Ring是否滿(mǎn)足條件。當(dāng)Ring的name大于50個(gè)字符或是duration為小數(shù)則不符合條件，將被剔除。 * @param ring 當(dāng)前Ring對(duì)象實(shí)例 * */private boolean isAvailableRing(Ring ring){Pattern p = Pattern.compile("^[1-9][0-9]*$");

Matcher match = p.matcher(ring.getDuration());

if(!match.find()){return false;}if(ring.getName().length() > 50 || ring.getArtist().length() > 50 || ring.getDownUrl().length() == 0){return false;}return true;}

/** * 獲取鈴聲的下載地址 * @param rid 鈴聲的id * */

public String getRingDownUrl(String rid){String url = String.format(GET_DOWN_URL, rid);

String responseStr = httpGet(url);return responseStr;}}

總結(jié)

以上是生活随笔為你收集整理的java铃声类_java多线程抓取铃声多多官网的铃声数据的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。