public static class HttpRequestHelper { private static Encoding DEFAULT_ENCODING = Encoding.UTF8; private static string ACCEPT = "application/json, text/javascript, */*; q=0.01"; private static string CONTENT_TYPE = "application/x-www-form-urlencoded"; private static string USERAGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36"; public static string GetHtmlContent(string url) { return GetHtmlContent(url, DEFAULT_ENCODING); } public static string GetHtmlContent(string url, Encoding encoding) { var request = (HttpWebRequest)WebRequest.Create(url); request.UserAgent = USERAGENT; request.Credentials = CredentialCache.DefaultCredentials; using (var webResponse = (HttpWebResponse)request.GetResponse()) { var bytes = GetWebResponseData(webResponse); return encoding.GetString(bytes); } } public static string GetCookie(string url) { string cookie = string.Empty; var request = WebRequest.Create(url); request.Credentials = CredentialCache.DefaultCredentials; using (var response = request.GetResponse()) { cookie = response.Headers.Get("Set-Cookie"); } return cookie; } /// <summary> /// /// </summary> /// <param name="url"></param> /// <param name="data"></param> /// <param name="referer"></param> /// <param name="cookieHeader">name=value,name=value</param> /// <param name="cookies"></param> /// <returns></returns> public static byte[] Post(string url, byte[] data = null, string[] headers = null, string referer = null, string cookieHeader = null, IList<Cookie> cookies = null) { return Submit(url, "POST", data, headers, referer, cookieHeader, cookies); } /// <summary> /// /// </summary> /// <param name="url"></param> /// <param name="data"></param> /// <param name="referer"></param> /// <param name="cookieHeader">name=value,name=value</param> /// <param name="cookies"></param> /// <param name="headers">name=value,name=value</param> /// <returns></returns> private static byte[] Submit(string url, string method, byte[] data = null, string[] headers = null, string referer = null, string cookieHeader = null, IList<Cookie> cookies = null) { if (string.IsNullOrEmpty(url)) throw new ArgumentNullException("url不能为空。"); if (string.IsNullOrEmpty(method)) throw new ArgumentNullException("method不能为空。"); var httpWebRequest = (HttpWebRequest)WebRequest.Create(url); httpWebRequest.Accept = ACCEPT; httpWebRequest.Referer = referer ?? url; httpWebRequest.UserAgent = USERAGENT; httpWebRequest.Method = method; httpWebRequest.CookieContainer = GetCookieContainer(url, cookieHeader, cookies); if (headers != null) { foreach (var item in headers) { httpWebRequest.Headers.Add(item); } } if (method == "POST") { httpWebRequest.ContentType = CONTENT_TYPE; } if (data != null) { httpWebRequest.ContentLength = (long)data.Length; using (var stream = httpWebRequest.GetRequestStream()) { stream.Write(data, 0, data.Length); } } using (var webResponse = (HttpWebResponse)httpWebRequest.GetResponse()) { return GetWebResponseData(webResponse); } } private static byte[] GetWebResponseData(HttpWebResponse response) { using (var stream = response.GetResponseStream()) { if (response.ContentEncoding.ToLower().Contains("gzip")) { using (var gZipStream = new GZipStream(stream, CompressionMode.Decompress)) { var bytes = ReadFully(gZipStream); return bytes; } } else { var bytes = ReadFully(stream); return bytes; } } } private static byte[] ReadFully(Stream stream) { byte[] buffer = new byte[128]; byte[] result; using (var ms = new MemoryStream()) { while (true) { int read = stream.Read(buffer, 0, buffer.Length); if (read <= 0) { break; } ms.Write(buffer, 0, read); } result = ms.ToArray(); } return result; } /// <summary> /// /// </summary> /// <param name="url"></param> /// <param name="cookieHeader">name=value,name=value</param> /// <param name="cookies"></param> /// <returns></returns> public static byte[] Get(string url, string[] headers = null, string referer = null, string cookieHeader = null, IList<Cookie> cookies = null) { return Submit(url, "GET", null, headers, referer, cookieHeader, cookies); } private static CookieContainer GetCookieContainer(string url, string cookieHeader = null, IList<Cookie> cookies = null) { if (string.IsNullOrEmpty(cookieHeader) && (cookies == null || cookies.Count == 0)) return null; var container = new CookieContainer(); if (!string.IsNullOrEmpty(cookieHeader)) { container.SetCookies(new Uri(url), cookieHeader); } if (cookies != null) { foreach (var item in cookies) { container.Add(item); } } return container; } }