HttpRequestHelp

using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Web;
using System.Text.RegularExpressions;

public class HttpRequestHelp
{


    #region  获取登陆的Cookies信息

    /// <summary>登录，获取Cookice信息
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="paramList">参数</param>
    /// <param name="cookieCon">Cookice信息</param>
    /// <returns>返回登陆的页面</returns>
    public static string Login(string url, string paramList, ref CookieContainer cookieCon)
    {
        HttpWebResponse res = null;
        string strResult = "";
        try
        {
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
            req.Method = "POST";
            req.ContentType = "application/x-www-form-urlencoded";
            req.AllowAutoRedirect = false;
            cookieCon = new CookieContainer();
            req.CookieContainer = cookieCon;
            StringBuilder UrlEncoded = new StringBuilder();
            Char[] reserved = { '?', '=', '&' };
            byte[] SomeBytes = null;
            if (paramList != null)
            {
                int i = 0, j;
                while (i < paramList.Length)
                {
                    j = paramList.IndexOfAny(reserved, i);
                    if (j == -1)
                    {
                        UrlEncoded.Append(HttpUtility.UrlEncode(paramList.Substring(i, paramList.Length - i)));
                        break;
                    }
                    UrlEncoded.Append(HttpUtility.UrlEncode(paramList.Substring(i, j - i)));
                    UrlEncoded.Append(paramList.Substring(j, 1));
                    i = j + 1;
                }
                SomeBytes = Encoding.UTF8.GetBytes(UrlEncoded.ToString());
                req.ContentLength = SomeBytes.Length;
                Stream newStream = req.GetRequestStream();
                newStream.Write(SomeBytes, 0, SomeBytes.Length);
                newStream.Close();
            }
            else
            {
                req.ContentLength = 0;
            }

            res = (HttpWebResponse)req.GetResponse();
            string cookieheader = req.CookieContainer.GetCookieHeader(new Uri(url));//cookies头
            Stream ReceiveStream = res.GetResponseStream();
            Encoding encode = System.Text.Encoding.GetEncoding("GBK");
            StreamReader sr = new StreamReader(ReceiveStream, encode);
            Char[] read = new Char[256];
            int count = sr.Read(read, 0, 256);
            while (count > 0)
            {
                String str = new String(read, 0, count);
                strResult += str;
                count = sr.Read(read, 0, 256);
            }
        }
        catch (Exception e)
        {
            strResult = e.ToString();
        }
        finally
        {
            if (res != null)
            {
                res.Close();
            }
        }
        return strResult;
    }
    /// <summary>
    /// 获取页面HTML
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="paramList">参数</param>
    /// <param name="cookieCon">Cookice信息</param>
    /// <returns>返回要处理的页面</returns>
    public static string getPage(string url, string paramList, ref CookieContainer cookieCon)
    {
        HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
        //req.Headers["If-None-Match"] = "36d0ed736e88c71:d9f";
        //req.Referer = "http://bbs.dospy.com";
        //  CookieContainer cookieCon =cookieCon;
        req.CookieContainer = cookieCon;
        //req.CookieContainer.SetCookies(new Uri(url), cookieheader);
        HttpWebResponse res = (HttpWebResponse)req.GetResponse();
        StreamReader sr = new StreamReader(res.GetResponseStream(), Encoding.Default);
        string strResult = sr.ReadToEnd();
        sr.Close();
        return strResult;
    }

    #endregion

    #region  HttpWebRequest

    private static CookieContainer cookie = new CookieContainer();
    private static string contentType = "application/x-www-form-urlencoded";
    private static string accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
    private static string userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";

    public static string GetHtmlEx(string url)
    {
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
        request.UserAgent = userAgent;
        request.ContentType = contentType;
        request.CookieContainer = cookie;
        request.Accept = accept;
        request.Method = "get";

        WebResponse response = request.GetResponse();
        Stream responseStream = response.GetResponseStream();
        Encoding encoding = null;
        for (int i = 0; i < response.Headers.Count; i++)
        {
            Match m = Regex.Match(response.Headers[i].ToString(), "(?i)(?<=charset=)[^ ]+");
            if (!m.Success) continue;
            encoding = Encoding.GetEncoding(m.Value);
            break;
        }
        StreamReader reader = new StreamReader(responseStream, encoding);
        String html = reader.ReadToEnd();
        response.Close();

        return html;
    }

    #endregion

    #region  WebClient
    public static string DownloadData(string url)
    {
        System.Net.WebClient wc = new System.Net.WebClient();
        Byte[] pageData = wc.DownloadData(url);
        string s = System.Text.Encoding.Default.GetString(pageData);
        return s;
    }
    #endregion

    #region  获取页面信息，有证书时返回True
    /// <summary>
    /// 传入URL返回网页的html代码
    /// </summary>
    /// <param name="Url">URL</param>
    /// <returns></returns>
    public string GetUrltoHtml(string Url)
    {
        StringBuilder content = new StringBuilder();
        try
        {
            // 与指定URL创建HTTP请求
            ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);//验证

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
            request.KeepAlive = false;
            request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8; .NET4.0C; .NET4.0E)";
            request.Method = "GET";
            request.Accept = "*/*";

            //创建证书文件
            System.Security.Cryptography.X509Certificates.X509Certificate objx509 = new System.Security.Cryptography.X509Certificates.X509Certificate(System.Windows.Forms.Application.StartupPath + "\\123.cer");

            //添加到请求里
            request.ClientCertificates.Add(objx509);

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            // 获取对应HTTP请求的响应
            // 获取响应流
            Stream responseStream = response.GetResponseStream();
            // 对接响应流(以"GBK"字符集)
            StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("GBK"));
            // 开始读取数据
            Char[] sReaderBuffer = new Char[256];
            int count = sReader.Read(sReaderBuffer, 0, 256);
            while (count > 0)
            {
                String tempStr = new String(sReaderBuffer, 0, count);
                content.Append(tempStr);
                count = sReader.Read(sReaderBuffer, 0, 256);
            }
            // 读取结束
            sReader.Close();
        }
        catch (Exception)
        {
            content = new StringBuilder("Runtime Error");
        }

        return content.ToString();
    }
    //回调验证证书问题
    public bool CheckValidationResult(object sender, System.Security.Cryptography.X509Certificates.X509Certificate certificate, System.Security.Cryptography.X509Certificates.X509Chain chain, System.Net.Security.SslPolicyErrors errors)
    {   // 总是接受
        return true;
    }

    #endregion

}

//最近在做数据采集工作，顺便总结一下几种请求方式。呵呵，我自己懒的写字，就给出几个网上比较经典的程序吧。
public class RequestWeb
{
    private readonly static int TIMEOUT = 15000;
    private CookieContainer _cookieCon = new CookieContainer();
    private CredentialCache _credentials;

    /// <summary>
    /// 通过url请求数据
    /// </summary>
    /// <param name="url">被请求页面的url</param>
    /// <param name="proxyServer">代理服务器</param>
    /// <returns>返回页面内容</returns>
    public string GetPageContent(string url, string proxyServer)
    {
        StringBuilder ret = new StringBuilder("");
        HttpWebResponse rsp = null;

        try
        {
            Uri uri = new Uri(url);
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
            if (!string.IsNullOrEmpty(proxyServer))
            {
                req.Proxy = new WebProxy(proxyServer);
            }
            req.CookieContainer = this._cookieCon;
            req.Headers.Add("Accept-Language: zh-cn");
            req.AllowAutoRedirect = true;
            req.Timeout = TIMEOUT;

            if (this._credentials != null)
            {
                req.PreAuthenticate = true;
                req.Credentials = this._credentials;
            }
            rsp = (HttpWebResponse)req.GetResponse();

            Stream rspStream = rsp.GetResponseStream();
            StreamReader sr = new StreamReader(rspStream, System.Text.Encoding.Default);

            //获取数据
            Char[] read = new Char[256];
            int count = sr.Read(read, 0, 256);
            while (count > 0)
            {
                ret.Append(read, 0, count);
                count = sr.Read(read, 0, 256);
            }
        }
        catch (Exception e)
        {
            ret.Append(e.Message);
        }
        finally
        {
            if (rsp != null)
            {
                rsp.Close();
            }
        }
        return ret.ToString();
    }


    /// <summary>
    /// 通过url请求数据(Post方法)
    /// </summary>
    /// <param name="url">被请求页面的url</param>
    /// <param name="param">POST的内容</param>
    /// <param name="proxyServer">代理</param>
    /// <returns>返回页面内容</returns>
    public string GetPageContent(string url, string param, string proxyServer)
    {
        StringBuilder ret = new StringBuilder("");
        HttpWebResponse rsp = null;

        try
        {
            Uri uri = new Uri(url);
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
            if (!string.IsNullOrEmpty(proxyServer))
            {
                req.Proxy = new WebProxy(proxyServer);
            }
            req.Method = "POST";
            req.ContentType = "application/x-www-form-urlencoded";
            req.Headers.Add("Accept-Language: zh-cn");
            req.CookieContainer = _cookieCon;
            req.Timeout = TIMEOUT;
            req.AllowAutoRedirect = true;
            if (_credentials != null)
            {
                req.PreAuthenticate = true;
                req.Credentials = _credentials;
            }

            //传入POST参数的分析
            if (param != null)
            {
                string temp = EncodeParams(param, System.Text.Encoding.Default);
                byte[] bytes = Encoding.UTF8.GetBytes(temp);
                req.ContentLength = bytes.Length;
                Stream rspStream = req.GetRequestStream();
                rspStream.Write(bytes, 0, bytes.Length);
                rspStream.Close();
            }
            else
            {
                req.ContentLength = 0;
            }

            //取得请求后返回的的数据
            rsp = (HttpWebResponse)(req.GetResponse());
            Stream ReceiveStream = rsp.GetResponseStream();
            StreamReader sr = new StreamReader(ReceiveStream, System.Text.Encoding.Default);

            Char[] read = new Char[256];
            int count = sr.Read(read, 0, 256);
            while (count > 0)
            {
                ret.Append(read, 0, count);
                count = sr.Read(read, 0, 256);
            }
        }
        //catch (Exception e)
        //{
        //    string err = e.ToString();
        //}
        finally
        {
            if (rsp != null)
            {
                rsp.Close();
            }
        }
        return ret.ToString();
    }

    /// <summary>
    /// 通过传入的url请求文件数据
    /// </summary>
    /// <param name="url">图片的URL</param>
    /// <param name="ProxyServer">代理服务器</param>
    /// <returns>图片内容</returns>
    public byte[] GetFile(string url, string proxyServer)
    {
        WebResponse rsp = null;
        byte[] retBytes = null;

        try
        {
            Uri uri = new Uri(url);
            WebRequest req = WebRequest.Create(uri);

            rsp = req.GetResponse();
            Stream stream = rsp.GetResponseStream();

            if (!string.IsNullOrEmpty(proxyServer))
            {
                req.Proxy = new WebProxy(proxyServer);
            }

            using (MemoryStream ms = new MemoryStream())
            {
                int b;
                while ((b = stream.ReadByte()) != -1)
                {
                    ms.WriteByte((byte)b);
                }
                retBytes = ms.ToArray();
            }
        }
        catch (Exception ex)
        {
            retBytes = null;
        }
        finally
        {
            if (rsp != null)
            {
                rsp.Close();
            }
        }
        return retBytes;
    }

    private string EncodeParams(string param, Encoding enc)
    {
        StringBuilder ret = new StringBuilder();
        char[] reserved = { '?', '=', '&', '%', '+' };

        if (param != null)
        {
            int i = 0, j;
            while (i < param.Length)
            {
                j = param.IndexOfAny(reserved, i);
                if (j == -1)
                {
                    ret.Append(HttpUtility.UrlEncode(param.Substring(i, param.Length - i), enc));
                    break;
                }
                ret.Append(HttpUtility.UrlEncode(param.Substring(i, j - i), enc));
                ret.Append(param.Substring(j, 1));
                i = j + 1;
            }
        }
        return ret.ToString();
    }
}