大二下学期团队项目（豆瓣爬取）

前面已能够爬取豆瓣电影所需要的所有信息，今日主要修改了爬取豆瓣的代码，进行了一些优化，主要两部分代码：

def insert_data(data_beans,headers,cursor,conn):
    try:
        for data_bean in data_beans:
            #20个电影数据
            for i in data_bean:
                # 分配数据
                score = i["rate"].replace(" ", "")
                director = i["directors"]  # []
                director_str = ""
                for j in director:
                    director_str = director_str + " " + j
                name = i["title"].replace(" ", "")
                img = i["cover"].replace(" ", "")
                star = i["casts"]  # []
                star_str = ""
                for j in star:
                    star_str = star_str + " " + j
                # 分配数据

                # 获取电影详细数据的网址
                url_details = i["url"].replace(" ", "")
                r = requests.get(url_details, headers=headers)
                soup_bean = BeautifulSoup(r.text, "lxml")
                # 获取详细数据
                span = soup_bean.find_all("span", {"property": "v:genre"})
                type = ""
                for i in span:
                    type = type + " " + i.text
                span = soup_bean.find_all("span", {"property": "v:runtime"})
                timelen = span[0].text.replace(" ", "")
                span = soup_bean.find_all("span", {"property": "v:initialReleaseDate"})
                date = span[0].text.replace(" ", "")
                span = soup_bean.find("a", {"class", "rating_people"})
                scorenum = span.text.replace(" ", "")
                span = soup_bean.find("span", {"property": "v:summary"})
                summary = span.text.replace(" ", "")  # 将空格去掉
                ex = ' <span class="pl">制片国家/地区:</span> (.*?)<br/>'
                test = re.findall(ex, r.text, re.S)
                area = test[0].replace(" ", "")
                ex2 = '<span class="pl">语言:</span> (.*?)<br/>'
                test = re.findall(ex2, r.text, re.S)
                language = test[0].replace(" / ", " ")
                print(url_details)
                # 获取详细数据
                sql = "insert into moviebean values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
                cursor.execute(sql,
                               [name, star_str, director_str, type, area, date, summary, score, language, img, scorenum,
                                timelen])
        conn.commit()  # 提交事务 update delete insert操作 //*[@id="info"]/text()[2]
        print(f"{time.asctime()}插入数据完毕")
    except:
        traceback.print_exc()

def get_tencent_data():
    #豆瓣的网址
    url_bean = 'https://movie.douban.com/j/new_search_subjects?sort=T&range=0,10&tags=%E7%94%B5%E5%BD%B1&start='

    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
    }
    cursor = None
    conn = None
    conn, cursor = get_conn()
    data_beans=[]
    num=3240#1440/3020/2760/3100/3180
    b=0;
    while b<=500:
        a = 1
        b=b+1
        while a <= 1:
            num_str = '%d' % num
            num = num + 20
            a = a + 1;
            # 获取豆瓣页面电影数据
            r = requests.get(url_bean + num_str, headers=headers)
            print(num_str)
            res_bean = json.loads(r.text);
            print(url_bean+num_str)
            data_beans.append(res_bean["data"])
            print(f"{time.asctime()}开始插入数据")
        insert_data(data_beans, headers,cursor,conn)
        data_beans=[]
    print(f"{time.asctime()}所有数据插入完毕")
    close_conn(conn, cursor)