C# 模拟http请求网页数据 [网页爬虫]

时间:2021-09-23 04:47:55

using System; using System.Collections.Specialized; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Text.RegularExpressions; using FTE.Framework.Log4NetService; namespace Proxy.BllServices { /// <summary> /// http访问类 /// </summary> public class HttpHelper { /// <summary> /// 访问失败的统一返回字符 /// </summary> public String ErrorReturn { get; private set; } = "HttpHelper access error!"; /// <summary> /// 登录后保存的cookie /// </summary> private CookieContainer Cookie = new CookieContainer(); /// <summary> /// http post 访问网页 /// </summary> /// <param></param> /// <param></param> /// <returns></returns> public string HttpPostString(string Url, string postDataStr) { try { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url); request.Method = "POST"; request.ContentType = "application/x-www-form-urlencoded"; request.ContentLength = Encoding.UTF8.GetByteCount(postDataStr); request.CookieContainer = Cookie; Stream myRequestStream = request.GetRequestStream(); StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312")); myStreamWriter.Write(postDataStr); myStreamWriter.Close(); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); response.Cookies = Cookie.GetCookies(response.ResponseUri); Stream myResponseStream = response.GetResponseStream(); StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8")); string retString = myStreamReader.ReadToEnd(); myStreamReader.Close(); myResponseStream.Close(); return retString; } catch (Exception ex) { LoggerManagerSingle.Instance.Error("http post 网站出错", ex); } return ErrorReturn; } public string HttpGet(string Url, string postDataStr) { try { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url + (postDataStr == "" ? "" : "?") + postDataStr); request.Method = "GET"; request.ContentType = "text/html;charset=UTF-8"; request.CookieContainer = Cookie; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream myResponseStream = response.GetResponseStream(); StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8")); string retString = myStreamReader.ReadToEnd(); myStreamReader.Close(); myResponseStream.Close(); return retString; } catch (Exception ex) { LoggerManagerSingle.Instance.Error("http get 网站出错", ex); } return ErrorReturn; } /// <summary> /// 使用form方式post数据[不包含文件] /// </summary> /// <param></param> /// <param></param> /// <returns></returns> public string HttpPostForm(string url, NameValueCollection stringDict) { try { string responseContent; var memStream = new MemoryStream(); var webRequest = (HttpWebRequest)WebRequest.Create(url); // 边界符 var boundary = "---------------" + DateTime.Now.Ticks.ToString("x"); // 边界符 var beginBoundary = Encoding.ASCII.GetBytes("--" + boundary + "\r\n"); // 最后的结束符 var endBoundary = Encoding.ASCII.GetBytes("--" + boundary + "--\r\n"); // 设置属性 webRequest.CookieContainer = Cookie; webRequest.Method = "POST"; webRequest.ContentType = "multipart/form-data; boundary=" + boundary; // 写入字符串的Key var stringKeyHeader = "\r\n--" + boundary + "\r\nContent-Disposition: form-data; name=\"{0}\"" + "\r\n\r\n{1}\r\n"; foreach (byte[] formitembytes in from string key in stringDict.Keys select string.Format(stringKeyHeader, key, stringDict[key]) into formitem select Encoding.UTF8.GetBytes(formitem)) { memStream.Write(formitembytes, 0, formitembytes.Length); } // 写入最后的结束边界符 memStream.Write(endBoundary, 0, endBoundary.Length); webRequest.ContentLength = memStream.Length; var requestStream = webRequest.GetRequestStream(); memStream.Position = 0; var tempBuffer = new byte[memStream.Length]; memStream.Read(tempBuffer, 0, tempBuffer.Length); memStream.Close(); requestStream.Write(tempBuffer, 0, tempBuffer.Length); requestStream.Close(); var httpWebResponse = (HttpWebResponse)webRequest.GetResponse(); using (var httpStreamReader = new StreamReader(httpWebResponse.GetResponseStream(), Encoding.GetEncoding("utf-8"))) { responseContent = httpStreamReader.ReadToEnd(); } httpWebResponse.Close(); webRequest.Abort(); return responseContent; } catch (Exception ex) { LoggerManagerSingle.Instance.Error("http form post 网站出错", ex); } return ErrorReturn; } } }

              调用例子:

//1. helper.HttpPostString("http://192.168.1.1/", "luci_username=root&luci_password=password"); //2. NameValueCollection stringDict = new NameValueCollection(); stringDict.Add("token", token); stringDict.Add("cbid.wireless.default_radio1.ssid", "everTestWifi"); helper.HttpPostData("http://192.168.1.1/cgi-bin/luci/admin/network/wireless/radio1.network2", stringDict);

  参考连接: