gzip的命名空間引用如下：using System.IO.Compression; 獲取網頁編碼和信息 ///?<summary>
????????///?create?time?2012-09-25
????????///?獲取頁面信息
????????///?</summary>
????????///?<param?name="strURL">網頁鏈接</param>
????????///?<param?name="charSet">編碼格式</param>
????????///?<param?name="strReferer">URL引用</param>
????????///?<returns></returns>
????????public?string?GetPage(string?strURL,?string?charSet,?string?strReferer)
????????{
????????????string?strPage?=?string.Empty;
????????????HttpWebRequest?request?=?(HttpWebRequest)HttpWebRequest.Create(strURL);
????????????//設置http頭
????????????request.AllowAutoRedirect?=?true;
????????????request.AllowWriteStreamBuffering?=?true;
????????????//request.Referer?=?"";
????????????//request.Timeout?=?10?*?1000;
????????????//request.UserAgent?=?"";
????????????//request.UserAgent?=?"Mozilla/5.0?(Windows?NT?5.2;?rv:10.0.2)?Gecko/20100101?Firefox/10.0.2";
????????????request.UserAgent?=?"?Mozilla/4.0?(compatible;?MSIE?8.0;?Windows?NT6.0)";
????????????request.KeepAlive?=?true;
????????????request.Timeout?=?300000;
????????????//request.AllowAutoRedirect?=?false;
????????????request.MaximumAutomaticRedirections?=?300;
????????????System.Net.CookieContainer?c?=?new?System.Net.CookieContainer();
????????????request.CookieContainer?=?c;
????????????int?count?=?0;
????????????char[]?buffer?=?null;
????????????StreamReader?reader?=?null;
????????????HttpWebResponse?response?=?null;
????????????string?tempCharSet?=?string.Empty;
????????????try
????????????{
????????????????string?characterSet?=?string.Empty;//分析后的編碼格式
????????????????response?=?(HttpWebResponse)request.GetResponse();
????????????????string?ce?=?response.ContentEncoding;

????????????????if?(charSet?!=?"")
????????????????{
????????????????????if?(ce.ToLower()?==?"gzip")
????????????????????{
????????????????????????GZipStream?gzip?=?new?GZipStream(response.GetResponseStream(),?CompressionMode.Decompress);
????????????????????????using?(reader?=?new?StreamReader(gzip,?System.Text.Encoding.GetEncoding(charSet)))
????????????????????????{
????????????????????????????strPage?=?reader.ReadToEnd();
????????????????????????}
????????????????????}
????????????????????else
????????????????????{
????????????????????????reader?=?new?System.IO.StreamReader(response.GetResponseStream(),?System.Text.Encoding.GetEncoding(charSet));
????????????????????????strPage?=?reader.ReadToEnd();
????????????????????????reader.Close();
????????????????????}
????????????????}
????????????????else
????????????????{
????????????????????#region?若沒有確定編碼格式，那么需要確定格式，并讀取
????????????????????if?(response.StatusCode?==?HttpStatusCode.OK)
????????????????????{

????????????????????????//根據http應答的http頭來判斷編碼????????????????????
????????????????????????Encoding?encode?=?null;
????????????????????????if?(charSet?==?"")//沒有傳入編碼格式，需初步確定編碼格式
????????????????????????{
????????????????????????????characterSet?=?response.CharacterSet;
????????????????????????????if?(characterSet?!=?"")
????????????????????????????{
????????????????????????????????if?(characterSet?==?"ISO-8859-1")
????????????????????????????????{
????????????????????????????????????characterSet?=?"gb2312";
????????????????????????????????}
????????????????????????????????try
????????????????????????????????{
????????????????????????????????????encode?=?Encoding.GetEncoding(characterSet);
????????????????????????????????}
????????????????????????????????catch?(Exception)
????????????????????????????????{
????????????????????????????????????encode?=?Encoding.Default;
????????????????????????????????}
????????????????????????????}
????????????????????????????else
????????????????????????????{
????????????????????????????????encode?=?Encoding.Default;
????????????????????????????}
????????????????????????}
????????????????????????else
????????????????????????{
????????????????????????????encode?=?Encoding.GetEncoding(charSet);
????????????????????????}
????????????????????????MemoryStream?mStream?=?new?MemoryStream();
????????????????????????//聲明一個內存流來保存http應答流
????????????????????????Stream?receiveStream?=?response.GetResponseStream();
????????????????????????if?(ce.ToLower()?==?"gzip")
????????????????????????{
????????????????????????????GZipStream?gzip?=?new?GZipStream(receiveStream,?CompressionMode.Decompress);
????????????????????????????using?(reader?=?new?StreamReader(gzip,?encode))
????????????????????????????{
????????????????????????????????strPage?=?reader.ReadToEnd();
????????????????????????????}
????????????????????????}
????????????????????????else
????????????????????????{


????????????????????????????byte[]?bf?=?new?byte[1024];
????????????????????????????count?=?receiveStream.Read(bf,?0,?1024);
????????????????????????????while?(count?>?0)
????????????????????????????{
????????????????????????????????mStream.Write(bf,?0,?count);
????????????????????????????????count?=?receiveStream.Read(bf,?0,?1024);
????????????????????????????}
????????????????????????????receiveStream.Close();

????????????????????????????mStream.Seek(0,?SeekOrigin.Begin);

????????????????????????????//從內存流里讀取字符串
????????????????????????????reader?=?new?StreamReader(mStream,?encode);
????????????????????????????buffer?=?new?char[1024];
????????????????????????????count?=?reader.Read(buffer,?0,?1024);
????????????????????????????while?(count?>?0)
????????????????????????????{
????????????????????????????????strPage?+=?new?String(buffer,?0,?count);
????????????????????????????????count?=?reader.Read(buffer,?0,?1024);
????????????????????????????}
????????????????????????}

????????????????????????if?(charSet?==?""?&&?IsLuan(strPage))//沒有傳入編碼格式，再次確定編碼格式
????????????????????????{
????????????????????????????//從解析出的字符串里判斷charset，如果和http應答的編碼不一直
????????????????????????????//那么以頁面聲明的為準，再次從內存流里重新讀取文本
????????????????????????????Regex?reg?=?new?Regex("<meta\\s[^>].*?charset=(\\S*)\"?[^><]*/?>",?RegexOptions.Multiline?|?RegexOptions.IgnoreCase);
????????????????????????????MatchCollection?mc?=?reg.Matches(strPage);
????????????????????????????if?(mc.Count?>?0)
????????????????????????????{
????????????????????????????????tempCharSet?=?mc[0].Result("$1");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("\"",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("?",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("/",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace(">",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("“",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("”",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("\r",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("\n",?"");
????????????????????????????????tempCharSet?=?tempCharSet.Replace("'",?"");
????????????????????????????????if?(tempCharSet.IndexOf("ISO-8859-1")?>?-1)
????????????????????????????????{
????????????????????????????????????tempCharSet?=?"gb2312";
????????????????????????????????}
????????????????????????????????else?if?(tempCharSet.ToLower().IndexOf("utf-8")?>?-1)
????????????????????????????????{
????????????????????????????????????tempCharSet?=?"utf-8";
????????????????????????????????}
????????????????????????????????else?if?(tempCharSet.ToLower().IndexOf("gb2312")?>?-1)
????????????????????????????????{
????????????????????????????????????tempCharSet?=?"gb2312";
????????????????????????????????}
????????????????????????????????

????????????????????????????????if?(string.Compare(tempCharSet,?characterSet,?true)?!=?0?&&?mc.Count?==?0)
????????????????????????????????{
????????????????????????????????????encode?=?Encoding.GetEncoding(tempCharSet);
????????????????????????????????????strPage?=?string.Empty;
????????????????????????????????????mStream.Seek(0,?SeekOrigin.Begin);
????????????????????????????????????if?(ce.ToLower()?==?"gzip")
????????????????????????????????????{
????????????????????????????????????????GZipStream?gzip?=?new?GZipStream(mStream,?CompressionMode.Decompress);
????????????????????????????????????????using?(reader?=?new?StreamReader(gzip,?encode))
????????????????????????????????????????{
????????????????????????????????????????????strPage?=?reader.ReadToEnd();
????????????????????????????????????????}
????????????????????????????????????}
????????????????????????????????????else
????????????????????????????????????{
????????????????????????????????????????reader?=?new?StreamReader(mStream,?encode);
????????????????????????????????????????buffer?=?new?char[1024];
????????????????????????????????????????count?=?reader.Read(buffer,?0,?1024);
????????????????????????????????????????while?(count?>?0)
????????????????????????????????????????{
????????????????????????????????????????????strPage?+=?new?String(buffer,?0,?count);
????????????????????????????????????????????count?=?reader.Read(buffer,?0,?1024);
????????????????????????????????????????}
????????????????????????????????????}
????????????????????????????????}
????????????????????????????}
????????????????????????}
????????????????????????reader.Close();
????????????????????????mStream.Close();
????????????????????}
????????????????????#endregion
????????????????}
????????????}
????????????catch?(Exception?ex)
????????????{
????????????????strGetPageErr?=?ex.Message;

????????????}
????????????finally
????????????{
????????????????if?(response?!=?null)
????????????????????response.Close();
????????????}
????????????string?strPattern?=?string.Empty;
???????????
????????????strPage?=?Regex.Replace(strPage,?"\t",?"",?RegexOptions.IgnoreCase?|?RegexOptions.Multiline?|?RegexOptions.Compiled);
????????????strPage?=?Regex.Replace(strPage,?"\r",?"",?RegexOptions.IgnoreCase?|?RegexOptions.Multiline?|?RegexOptions.Compiled);
????????????strPage?=?Regex.Replace(strPage,?"\n",?"",?RegexOptions.IgnoreCase?|?RegexOptions.Multiline?|?RegexOptions.Compiled);

????????????return?strPage;
????????}


///?<summary>
????????///?判斷中文是否亂碼
????????///?</summary>
????????///?<param?name="strPage"></param>
????????///?<returns></returns>
????????public?bool?IsLuan(string?strPage)
????????{
????????????Regex?reg?=?new?Regex("[一-龥]");
????????????int?iLuanNum?=?0;
????????????MatchCollection?mc?=?reg.Matches(strPage);
????????????const?int?iMaxForNum?=?50;
????????????if?(mc.Count?>?0)
????????????{
????????????????reg?=?new?Regex("[^一乙二十丁廠七卜八人入兒九幾了乃刀力又三干于虧士土工才下寸丈大與萬上小口山巾千乞川億個么久勺丸夕凡及廣亡門義之尸已弓己衛子也女飛刃習叉馬鄉豐王井開夫天元無云專扎藝木五支廳不太犬區歷友尤匹車巨牙屯比互切瓦止少日中貝內水岡見手午牛毛氣升長仁什片仆化仇幣仍僅斤爪反介父從今兇分乏公倉月氏勿風欠丹勻烏勾鳳六文方火為斗憶計訂戶認心尺引丑巴孔隊辦以允予勸雙書幻玉刊末未示擊打巧正撲扒功扔去甘世古節本術可丙左厲石右布龍平滅軋東卡北占業舊帥歸目旦且叮葉甲申號電田由只央史兄叼叫叨另嘆四生失禾丘付仗代仙們儀白仔他斥瓜乎叢令用甩印樂句匆冊犯外處冬鳥務包饑主市立閃蘭半汁匯頭漢寧穴它討寫讓禮訓必議訊記永司尼民出遼奶奴加召皮邊孕發圣對臺矛糾母幼絲式刑動扛寺吉扣考托老圾鞏執擴掃地揚場耳共芒亞芝朽樸機權過臣再協西壓厭在百有存而頁匠夸奪灰達列死成夾軌邪劃邁畢至此貞師塵尖劣光當早吐嚇蟲曲團同吊吃因吸嗎嶼帆歲回豈則剛網肉年朱先丟舌竹遷喬偉傳乒乓休伍伏優伐延件任傷價份華仰仿伙偽自血向似后行舟全會殺合兆企眾爺傘創肌朵雜危旬旨負各名多爭色壯沖冰莊慶亦劉齊交次衣產決充妄閉問闖羊并關米燈州汗污江池湯忙興宇守宅字安講軍許論農諷設訪尋那迅盡導異孫陣陽收階陰防奸如婦好她媽戲羽觀歡買紅纖約級紀馳巡壽弄麥形進戒吞遠違運扶撫壇技壞擾拒找批扯址走抄壩貢攻赤折抓扮搶孝均拋投墳坑抗坊抖護殼志塊扭聲把報卻劫芽花芹芬蒼芳嚴蘆勞克蘇桿杜杠材村杏極李楊求更束豆兩麗醫辰勵否還殲來連步堅旱盯呈時吳助縣里呆園曠圍呀噸足郵男困吵串員聽吩吹嗚吼吧別崗帳財釘針告我亂利禿秀私每兵估體何但伸作伯伶傭低你住位伴身皂佛近徹役返余希坐谷妥含鄰岔肝肚腸龜免狂猶角刪條卵島迎飯飲系言凍狀畝況床庫療應冷這序辛棄冶忘閑間悶判灶燦弟汪沙汽沃泛溝沒沈沉懷憂快完宋宏牢究窮災良證啟評補初社識訴診詞譯君靈即層尿尾遲局改張忌際陸阿陳阻附妙妖妨努忍勁雞驅純紗綱納縱駁紛紙紋紡驢紐奉玩環武青責現表規抹攏拔揀坦擔押抽拐拖者拍頂拆擁抵拘勢抱垃拉攔幸拌招坡披撥擇抬其取苦若茂蘋苗英范直茄莖茅林枝杯柜析板松槍構杰述枕喪或畫臥事刺棗雨賣礦碼廁奔奇奮態歐壟妻轟頃轉斬輪軟到非叔肯齒些虎虜腎賢尚旺具果味昆國昌暢明易昂典固忠咐呼鳴詠呢岸巖帖羅幟嶺凱敗販購圖釣制知垂牧物乖刮稈和季委佳侍供使例版侄偵側憑僑佩貨依的迫質欣征往爬彼徑所舍金命斧爸采受乳貪念貧膚肺肢腫脹朋股肥服脅周昏魚兔狐忽狗備飾飽飼變京享店夜廟府底劑郊廢凈盲放刻育閘鬧鄭券卷單炒炊炕炎爐沫淺法泄河沾淚油泊沿泡注瀉泳泥沸波潑澤治怖性怕憐怪學寶宗定宜審宙官空簾實試郎詩肩房誠襯衫視話誕詢該詳建肅隸錄居屆刷屈弦承孟孤陜降限妹姑姐姓始駕參艱線練組細駛織終駐駝紹經貫奏春幫珍玻毒型掛封持項垮挎城撓政赴趙擋挺括拴拾挑指墊掙擠拼挖按揮挪某甚革薦巷帶草繭茶荒茫蕩榮故胡南藥標枯柄棟相查柏柳柱柿欄樹要咸威歪研磚厘厚砌砍面耐耍牽殘殃輕鴉皆背戰點臨覽豎省削嘗是盼眨哄啞顯冒映星昨畏趴胃貴界虹蝦蟻思螞雖品咽罵嘩咱響哈咬咳哪炭峽罰賤貼骨鈔鐘鋼鑰鉤卸缸拜看矩怎牲選適秒香種秋科重復竿段便倆貨順修保促侮儉俗俘信皇泉鬼侵追俊盾待律很須敘劍逃食盆膽勝胞胖脈勉狹獅獨狡獄狠貿怨急饒蝕餃餅彎將獎哀亭亮度跡庭瘡瘋疫疤姿親音帝施聞閥閣差養美姜叛送類迷前首逆總煉炸炮爛剃潔洪灑澆濁洞測洗活派洽染濟洋洲渾濃津恒恢恰惱恨舉覺宣室宮憲突穿竊客冠語扁襖祖神祝誤誘說誦墾退既屋晝費陡眉孩除險院娃姥姨姻嬌怒架賀盈勇怠柔壘綁絨結繞驕繪給絡駱絕絞統耕耗艷泰珠班素蠶頑盞匪撈栽捕振載趕起鹽捎捏埋捉捆捐損都哲逝撿換挽熱恐壺挨恥耽恭蓮莫荷獲晉惡真框桂檔桐株橋桃格校核樣根索哥速逗栗配翅辱唇夏礎破原套逐烈殊顧轎較頓斃致柴桌慮監緊黨曬眠曉鴨晃晌暈蚊哨哭恩喚啊唉罷峰圓賊賄錢鉗鉆鐵鈴鉛缺氧特犧造乘敵秤租秧積秩稱秘透筆笑筍債借值倚傾倒倘俱倡候俯倍倦健臭射躬息徒徐艦艙般航途拿爹愛頌翁脆脂胸胳臟膠腦貍狼逢留皺餓戀槳漿衰高席準座癥病疾疼疲脊效離唐資涼站剖競部旁旅畜閱羞瓶拳粉料益兼烤烘煩燒燭煙遞濤浙澇酒涉消浩海涂浴浮流潤浪浸漲燙涌悟悄悔悅害寬家宵宴賓窄容宰案請朗諸讀扇襪袖袍被祥課誰調冤諒談誼剝懇展劇屑弱陵陶陷陪娛娘通能難預桑絹繡驗繼球理捧堵描域掩捷排掉推堆掀授教掏掠培接控探據掘職基著勒黃萌蘿菌菜萄菊萍菠營械夢梢梅檢梳梯桶救副票戚爽聾襲盛雪輔輛虛雀堂常匙晨睜瞇眼懸野啦晚啄距躍略蛇累唱患唯崖嶄崇圈銅鏟銀甜梨犁移笨籠笛符第敏做袋悠償偶偷您售停偏假得銜盤船斜盒鴿悉欲彩領腳脖臉脫象夠猜豬獵貓猛餡館湊減毫麻癢痕廊康庸鹿盜章竟商族旋望率著蓋粘粗粒斷剪獸清添淋淹渠漸混漁淘液淡深婆梁滲情惜慚悼懼惕驚慘慣寇寄宿窯密謀謊禍謎逮敢屠彈隨蛋隆隱婚嬸頸績緒續騎繩維綿綢綠琴斑替款堪塔搭越趁趨超提堤博揭喜插揪搜煮援裁擱摟攪握揉斯期欺聯散惹葬葛董葡敬蔥落朝辜葵棒棋植森椅椒棵棍棉棚棕惠惑逼廚廈硬確雁殖裂雄暫雅輩悲紫輝敞賞掌晴暑最量噴晶喇遇喊景踐跌跑遺蛙蛛蜓喝喂喘喉幅帽賭賠黑鑄鋪鏈銷鎖鋤鍋銹鋒銳短智毯鵝剩稍程稀稅筐等筑策篩筒答筋箏傲傅牌堡集焦傍儲奧街懲御循艇舒番釋禽臘脾腔魯猾猴然饞裝蠻就痛童闊善羨普糞尊道曾焰港湖渣濕溫渴滑灣渡游滋溉憤慌惰愧愉慨割寒富竄窩窗遍裕褲裙謝謠謙屬屢強粥疏隔隙絮嫂登緞緩騙編緣瑞魂肆攝摸填搏塌鼓擺攜搬搖搞塘攤蒜勤鵲藍墓幕蓬蓄蒙蒸獻禁楚想槐榆樓概賴酬感礙碑碎碰碗碌雷零霧雹輸督齡鑒睛睡睬鄙愚暖盟歇暗照跨跳跪路跟遣蛾蜂嗓置罪罩錯錫鑼錘錦鍵鋸矮辭稠愁籌簽簡毀舅鼠催傻像躲微愈遙腰腥腹騰腿觸解醬痰廉新韻意糧數煎塑慈煤煌滿漠源濾濫滔溪溜滾濱粱灘慎譽塞謹福群殿辟障嫌嫁疊縫纏靜碧璃墻嘉摧截誓境摘摔撇聚慕暮蔑蔽模榴榜榨歌遭酷釀酸磁愿需裳顆嗽蜻蠟蠅蜘賺鍬鍛舞穩算籮管僚鼻魄貌膜膊膀鮮疑饅裹敲豪膏遮腐瘦辣竭端旗精歉弊熄熔漆漂漫滴演漏慢寨賽察蜜譜嫩翠熊凳騾縮慧撕撒趣趟撐播撞撤增聰鞋蕉蔬橫槽櫻橡飄醋醉震霉瞞題暴瞎影踢踏踩蹤蝶蝴囑墨鎮靠稻黎稿稼箱箭篇僵躺僻德艘膝膛熟摩顏毅糊遵潛潮懂額慰劈操燕薯薪薄顛橘整融醒餐嘴蹄器贈默鏡贊籃邀衡膨雕磨凝辨辯糖糕燃澡激懶壁避繳戴擦鞠藏霜霞瞧蹈螺穗繁辮贏糟糠燥臂翼驟鞭覆蹦鐮翻鷹警攀蹲顫瓣爆疆壤耀躁嚼嚷籍魔灌蠢霸露囊罐]");

????????????????int?iTotalChinese?=?mc.Count?;//所有中文數量
????????????????int?iStart?=?iTotalChinese?/3?+?1;//所有中文的三分之一開始取，盡量確定正文部分

????????????????int?iForNum?=?(iTotalChinese?-?iStart?-?1)?<?iMaxForNum???(iTotalChinese?-?iStart?-?1)?:?iMaxForNum;

????????????????for?(int?i?=?iStart;?i?<?iForNum;?i++)
????????????????{
????????????????????MatchCollection?mc1?=?reg.Matches(mc[i].ToString());
????????????????????if?(mc1.Count?>?0)
????????????????????{
????????????????????????iLuanNum?+=?mc1.Count;
????????????????????????if?(iLuanNum?>?20)
????????????????????????{
????????????????????????????return?true;
????????????????????????}
????????????????????}
????????????????}

????????????}
????????????return?false;

轉載于:https://www.cnblogs.com/hanguoji/archive/2012/11/27/2790181.html

總結

以上是生活随笔為你收集整理的爬取网页时自动获取网页编码信息，并对特殊的乱码页面（压缩过的网页内容）用gzip进行解码。...的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。