1)①爬取中国新闻网科技相关部分新闻

__author__ = 'minmin'
#coding:utf-8
import re,urllib,sgmllib,os

#根据当前的url获取html
def getHtml(url):
    page = urllib.urlopen(url)
    html = page.read()
    page.close()
    return html

#根据html获取想要的文章内容
def func(str):
     result =   re.findall(r"<p.*?>([^<>]*)</p>",getHtml(url),re.M)
     artical =''

     for j in result:
         if len(j)<>0:
             j = j.replace("<strong>","    ")
             j = j.replace("</strong>","    ")
             j = j.replace("<br>","   ")
             j = j.replace("&nbsp;"," ")
             j = j.replace("&ldquo;"," ")
             j = j.replace("&rdquo;"," ")
             j = j.replace("&middot;"," ")
             artical = artical + j + '
'
     return  artical

#html链接的标签是“a”，链接的属性是“href”，也就是要获得html中所有tag=a，attrs=href 值。
class URLPaser(sgmllib.SGMLParser):
    def reset(self):
        sgmllib.SGMLParser.reset(self)
        self.urls = []

    def start_a(self,attrs):
        href = [v for k,v in attrs if k == 'href']
        if href:
            self.urls.extend(href)

IParser = URLPaser()
socket = urllib.urlopen("http://economy.china.com/internet/")#打开这个网页

#fout = file('qq_art_urls.txt','w')#要把这个链接写到这个文件中
IParser.feed(socket.read())#分析啦

reg = 'http://economy.china.com/internet/.*'#这个是用来匹配符合条件的链接，使用正则表达式匹配

pattern = re.compile(reg)

os.getcwd()#获得当前文件夹路径
os.path.sep#当前系统路径分隔符

#判断文件是否存在
if os.path.exists('china_news_Technology')==False:
     os.makedirs('china_news_Technology')
i = 0
url2 = []
for url in IParser.urls:#链接都存在urls里
    url = "http://economy.china.com"+url
    if pattern.match(url):
        if url not in url2:
            url2.append(url)
            url = url.replace(".html","_all.html#page_2")#由于一篇文章被分成几页，找到显示全页的那一个页面
            artical = func(url)
            if len(artical)<>0:
                  print artical
                  print url
                  i = i + 1
                  f = open("china_news_Technology/"+str(i) + '.txt','a+')
                  f.write(artical)
                  f.close()