• 爬取链家二手房


    1.将爬取出来的数据存储到scv, mysql, mongo数据库中,利用正则的方式爬取。

    import csv
    import warnings
    
    import pymongo
    import pymysql
    import requests
    import re
    
    
    class LIANJIA:
    
        def __init__(self):
            self.url = "https://%s.lianjia.com/ershoufang/"
            # self.proxies = {"HTTP", "http://61.152.248.147:80"}
            self.proxies = {"HTTP": "http://116.255.162.107:16816"}
            self.headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"}
    
            # 连接pymysql数据库
            # self.db = pymysql.connect("localhost", "root", "123456", charset="utf8")
            # self.cursor = self.db.cursor()
    
            # 连接mongo数据库
            self.conn = pymongo.MongoClient("localhost", 27017)
            self.db = self.conn["lianjia"]
            self.tab = self.db.lianjiafang
    
        def getHtml(self, url):
            response = requests.get(url, proxies=self.proxies, headers=self.headers)
            response.encoding = "utf-8"
            html = response.text
            self.parse(html)
    
        # 利用正则来匹配数据
        def parse(self, html):
            re_str = '<div class="info clear">.*?data-el="region">(.*?)</a>.*?class="totalPrice"><span>(.*?)</span>.*?<span>(.*?)</span></div>'
            p = re.compile(re_str, re.S)
            # 利用正则匹配,返回的是一个集合列表
            result_list = p.findall(html)
            print(result_list)
            self.saveTomongo(result_list)
    
        # 存入csv
        def saveTocsv(self, result_list):
            for result in result_list:
                print(result)
                with open("lianjia.csv", "a", newline="") as f:
                    writer = csv.writer(f)
                    writer.writerow(result)
    
        # 将数据存入mysql数据库中
        def saveTomysql(self, result_list):
            cd_db = "create database if not exists lianjia charset utf8"
            u_db = "use lianjia"
            c_tab = "create table if not exists lianjiafang(
                                id int primary key auto_increment,
                                name varchar(100),
                                price varchar(100),
                                sq_mPrice varchar(100)
                                )"
            ins = "insert into lianjiafang(name, price, sq_mPrice)
                            values(%s, %s, %s)"
            warnings.filterwarnings("ignore")
            try:
                self.cursor.execute(cd_db)
                self.cursor.execute(u_db)
                self.cursor.execute(c_tab)
            except:
                pass
            # 插入记录
            for result_tuple in result_list:
                L = [result_tuple[0].strip(), int(result_tuple[1].strip())*10000, result_tuple[2].strip()]
                # execute(ins, [列表])
                self.cursor.execute(ins, L)
                self.db.commit()
                print("OK")
    
        # 存入mongo
        def saveTomongo(self, result_list):
            for result_tuple in result_list:
                name = result_tuple[0].strip()
                price = int(result_tuple[1].strip()) * 10000
                sq_mPrice = result_tuple[2].strip()
                d = {"name":name, "star":price, "time":sq_mPrice}
                self.tab.insert(d)
            print("OK")
    
        def workOn(self):
            city = input("请输入你要搜索的城市首拼音:")
            end = int(input("爬取多少页:"))
            for x in range(1, end+1):
                if x == 1:
                    url = self.url % city
                    self.getHtml(url)
                else:
                    url = (self.url % city) + "pg" + str(x) + "/"
                    self.getHtml(url)
            # self.cursor.close()
            # self.db.close()
    
    
    if __name__ == "__main__":
        lianjia = LIANJIA()
        lianjia.workOn()

    得到csv中的结果,mysql和mongo结果就不显示了:

  • 相关阅读:
    触发器基本使用
    查询结果合并用逗号分隔
    查询报表增加小计功能
    sql语句格式化数字(前面补0)
    如何在选择画面中创建下拉列表(drop down list)-as list box
    如何更改函数的函数组(function group)
    ABAP语言中如何定义嵌套内表(nested internal table)
    [REUSE_ALV_GRID_DISPLAY]如何指定单元格颜色
    如何创建嵌套动态内表(Nested dynamic internal table)
    如何根据方法名(method)查找所在类(class)-SE84
  • 原文地址:https://www.cnblogs.com/zengsf/p/10022159.html
Copyright © 2020-2023  润新知