网站页面的抓取

 //抓取网页上的全部内容
        //webrequest类获取网页源代码
        protected void btn_click_Click(object sender, EventArgs e)
        {
            string strMsg = string.Empty;
            try
            {
                //WebRequest.Create方法，返回WebRequest的子类HttpWebRequest
                WebRequest request = WebRequest.Create(txt_url.Text.Trim());//为指定的URI方案初始化新的WebRequest实例
                WebResponse response = request.GetResponse();//WebRequest.GetResponse方法，返回对 Internet 请求的响应
                //Response.Write( response.ContentType);获取输出的内容的格式
                //Stream resStream = response.GetResponseStream(); //WebResponse.GetResponseStream 方法，从 Internet 资源返回数据流。 
                //Encoding enc = Encoding.GetEncoding("GB2312"); // 如果是乱码就改成 utf-8 / GB2312
                StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("UTF-8"));
                // StreamReader 类实现一个 TextReader (TextReader类，表示可读取连续字符系列的读取器)，使其以一种特定的编码从字节流中读取字符。 
                strMsg = reader.ReadToEnd();//输出HTML代码

                reader.Close();
                reader.Dispose();
                response.Close();
            }
            catch
            { }
            if (strMsg != "")
            {
                divShow.InnerHtml = strMsg;

            }
            else
            {
                Response.Write("获取内容为空");
            }
        }

 string strMag = string.Empty;
            WebClient wc = new WebClient();//创建WebClient实例提供向URI 标识的资源发送数据和从URI 标识的资源接收数据
            wc.Credentials = CredentialCache.DefaultCredentials;// 获取或设置用于对向 Internet 资源的请求进行身份验证的网络凭据。
            //方法一
            //Encoding enc = Encoding.GetEncoding("UTF-8");
            //Byte[] pageData = wc.DownloadData(txt_url.Text);// 从资源下载数据并返回字节数组。
            //strMag = enc.GetString(pageData);
            //divShow.InnerHtml = strMag;
            //方法二
            Stream resStream = wc.OpenRead(txt_url.Text);//以流的形式打开URL
            Encoding enc = Encoding.GetEncoding("UTF-8");
            StreamReader sr = new StreamReader(resStream, enc);//以指定的编码方式读取数据流
            divShow.InnerHtml = sr.ReadToEnd();
            resStream.Close();
            wc.Dispose();

 //将获取页面内容，并转化为*.txt保存
        //string strMsg = string.Empty;
        //try
        //{
        //    WebRequest request = WebRequest.Create(txt_url.Text);
        //    WebResponse response = request.GetResponse();
        //    Stream reader = response.GetResponseStream();

        //    //可根据实际保存为具体文件  
        //    FileStream writer = new FileStream("D://logo.txt", FileMode.OpenOrCreate, FileAccess.Write);
        //    byte[] buff = new byte[512];
        //    int c = 0; //实际读取的字节数   
        //    while ((c = reader.Read(buff, 0, buff.Length)) > 0)
        //    {
        //        writer.Write(buff, 0, c);
        //    }
        //    writer.Close();
        //    writer.Dispose();

        //    reader.Close();
        //    reader.Dispose();
        //    response.Close();

        //    strMsg = "保存成功";
        //}
        //catch
        //{ }
        //Response.Write(strMsg); 

网页抓取类

http://www.soaspx.com/dotnet/asp.net/tech/tech_20110913_8052.html

http://blog.csdn.net/jiang1984j/article/details/5793239

http://www.cftea.com/c/2007/08/H89S2ILKP2SPAKT7.asp