使用C#抓取网页内容并分析获取数据

    private void button5_Click(object sender, EventArgs e)
    {
        string html = "";
        WebHeaderCollection header = new WebHeaderCollection();
        header.Set("Pragma", "no-cache");
        html = getHtml("http://www.biomart.cn/info/infoDemand.htm?pge=1", header);
        Regex regex = new Regex("<!-- 列表 -->(?<1>.*)<!-- /列表 -->");
        //MessageBox.Show(regex.Match(html).Groups.Count.ToString());  
        html = regex.Match(html).Groups[1].Value;
        regex = new Regex("href="(?<1>http://www\.biomart\.cn/infodemand/\w+\.htm)"");
        MatchCollection ms = regex.Matches(html);
        header.Set(HttpRequestHeader.Cookie, "__utma=124945049.1686326021.1305093063.1305164868.1305187067.3; __utmz=124945049.1305093063.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); JSESSIONID=9D7F7F4B5D73F453DA54B40A53D5E7C8; __utmc=124945049; __utmb=124945049.2.10.1305187067");
        foreach (Match m in ms)
        {
            MessageBox.Show(m.Groups[1].Value);
            String content = getHtml(m.Groups[1].Value, header);
            regex = new Regex("<div class="product_card">(?<1>.*)\s+</p>\s+</div>");
            MessageBox.Show(regex.Match(content).Groups[1].Value);
        }
    }


    private String getHtml(String url, WebHeaderCollection header)
    {
        WebHeaderCollection header = new WebHeaderCollection();
        header.Set("", "");
        header.Set(HttpRequestHeader.Cookie, "");
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.biomart.cn/info/infoDemand.htm?pge=1");
        request.Timeout = 30000;
        request.Headers = header;
        HttpWebResponse response = (HttpWebResponse)request.GetResponse();
        Stream stream = response.GetResponseStream();
        Encoding encoding = Encoding.GetEncoding("UTF-8");
        StreamReader reader = new StreamReader(stream);
        String content = reader.ReadToEnd();
        content = Regex.Replace(content, "\t|\r|\n", "");
        return content;
    }