python 读取ini配置文件,csv读取,mongodb

[main.py]

#! /usr/bin/python
# -*- coding: utf-8 -*-


import configparser
import csv
import time
import pymongo
import pymysql

# 加载配置文件
cf = configparser.ConfigParser()
# 加载ini配置
cf.read('static/config.ini', encoding="utf-8")

# MYSQL
mysqlIp = cf.get('mysql', 'server.ip')
mysqlPort = cf.get('mysql', 'server.port')
mysqlAuth = cf.get('mysql', 'server.auth')
mysqlPassword = cf.get('mysql', 'server.password')
mysqlDBName = cf.get('mysql', 'server.dbname')
selectSql = cf.get('mysql', 'select.sql')
# MYSQL连接对象
mysqlConnect = pymysql.connect(host=mysqlIp, port=int(mysqlPort), user=mysqlAuth, passwd=mysqlPassword,
                               database=mysqlDBName)
mysqlCursor = mysqlConnect.cursor()

# MONGO
mongoIp = cf.get('mongoDB', 'server.ip')
mongoPort = cf.get('mongoDB', 'server.port')
mongoAuth = cf.get('mongoDB', 'server.auth')
mongoPassword = cf.get('mongoDB', 'server.password')
mongoDBName = cf.get('mongoDB', 'server.dbname')
mongoDBCollectionName = cf.get('mongoDB', 'server.collectionName')
# mongo 客户端对象,与ADMIN数据库认证
mongoClient = pymongo.MongoClient(host=mongoIp, port=int(mongoPort))
adminDb = mongoClient.admin
adminDb.authenticate(mongoAuth, mongoPassword, mechanism='SCRAM-SHA-1')
# mongo 数据库连接对象
mongoConnect = mongoClient[mongoDBName]
# mongo 数据库集合对象
mongoDBCollection = mongoConnect[mongoDBCollectionName]
# 配置了的CSV信息
csvNames = cf.get('csvConfig', 'csvNames').split(',')
dataHeaders = cf.get('csvConfig', 'dataHeaders').split(',')

def read_csv():
    for index in range(len(csvNames)):
        print('开始读取文件:',csvNames[index])
        count = 1
        with open('static/' + csvNames[index]) as device:
            device_csv = csv.DictReader(device)
            dataList=[]
            allCount = 1
            # 获取表头
            header = next(device_csv)
            for row in device_csv:
                for column in row:
                    if column in dataHeaders:
                        # 采集时间
                        data = {}
                        data['gather_time'] = row['取样时间']
                        data['create_time'] = row['执行时间']
                        data['update_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
                        #mysql查询
                        execSql = selectSql.format(device_code=row['装置id'], tp_name=column)
                        mysqlCursor.execute(execSql)
                        obj = mysqlCursor.fetchone()
                        data['data'] = row[column]
                        if obj:
                            #MYSQL数据库查询
                            data['device_code'] = str(obj[0])
                            data['monitor_type'] = str(obj[1])
                            data['server_code'] = str(obj[2])
                            data['tp_code'] = str(obj[3])
                            data['signal_type'] = str(obj[4])
                        else:
                            data['device_code'] = None
                            data['monitor_type'] = None
                            data['server_code'] = None
                            data['tp_code'] = None
                            data['signal_type'] = None
                        dataList.append(data)
                        print('当前执行到第'+str(allCount)+'轮 第'+str(count)+'条数据')
                        if count == 1000:
                            mongoDBCollection.insert_many(dataList)
                            print('开始插入文件:' + csvNames[index] + '第'+str(allCount)+'轮数据')
                            dataList=[]
                            count = 1
                            allCount = allCount + 1
                        else:
                            count=count+1
    print('数据插入执行完成')
    mysqlConnect.close()


if __name__ == '__main__':
    read_csv()

[config.ini]

;mongoDb连接配置
[mongoDB]
server.ip=192.168.99.101
server.port=27017
server.auth=admin
server.password=123456
;基本数据历史库
server.dbname = ythpt_data
server.collectionName = base_data_history

;mysql连接配置
[mysql]
server.ip=192.168.99.101
server.port=3306
server.auth=root
server.password=123456
server.dbname = ythpt
select.sql = SELECT DISTINCT m.device_code, m.monitor_type, m.server_code, dev.tp_code, dev.signal_type, tp.tp_name FROM monitor_device m JOIN device_point_alarm dev ON m.device_code = dev.device_code  AND dev.del_flag = 0 JOIN tp_code_templet tp ON dev.signal_type = tp.signal_type  AND dev.tp_code = tp.tp_code  AND dev.monitor_type = tp.monitor_type  AND tp.del_flag = 0  WHERE m.del_flag = 0  AND m.device_code = '{device_code}'  AND tp.tp_name = '{tp_name}'


;属性配置
[csvConfig]
csvNames=数据.csv
;装载数据的表头
dataHeaders=二氧化碳,乙烯,乙炔,乙烷,氢气,氧气,甲烷,一氧化碳,总可燃,氮气,水