单独的 python 脚本文件使用 django 自带的 model

django1.9.5&python3.4.4

文件结构

在一个爬虫脚本中将爬取的数据通过django自带的model保存到数据库

 

修改的文件(其余pycharm新建Django项目生成,未修改):

 

# testapp/models.py
from django.db import models
 
class Problem(models.Model):
title = models.CharField(max_length=100, default="")
author = models.CharField(max_length=100, default="")
    def __str__(self):
        return self.title
pass

虚拟欢迎需要安装django

# push_view
import os
import sys
import django

pathname = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, pathname)
sys.path.insert(0, os.path.abspath(os.path.join(pathname, '..')))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xad_unit.settings")

django.setup()

from datetime import datetime
import time
from pv_uv.views import exec_sql
from pv_uv.models import Cpd



class Sql_Tool(object):

    def zy_sql(self, date):
        #sql = 'select ngx_date, path, event_type, count(distinct request_id) as num from xad_app_log where path = "/xad_tracking" and event_type in ("xad_dl_start", "xad_dl_success", "xad_install_start", "xad_install_success", "xad_click_tracking", "xad_impression") and ngx_date="{}"and ad_system="{}" group by ngx_date, path, event_type order by ngx_date, path, event_type limit 1000'.format(
        #     date, 'liyanmobi')

        sql = 'select ngx_date, path, event_type, count(distinct request_id) as num from xad_app_log where path = "/xad_tracking" and event_type in ("xad_dl_start", "xad_dl_success", "xad_install_start", "xad_install_success", "xad_click_tracking", "xad_impression") and ngx_date>="{}" and ad_system="{}" and req_from="1" group by ngx_date, path, event_type order by ngx_date, path, event_type limit 1000'.format(
           date, 'liyanmobi')

        l_list = exec_sql(sql)
        return l_list

    def strint_toDate(self, string):
        return datetime.strptime(string, '%Y%m%d')



if __name__ == '__main__':


    sql_tool = Sql_Tool()
    current_time = time.strftime('%Y%m%d', time.localtime(time.time()))
    #current_time = '20180726'
    l_list = sql_tool.zy_sql(current_time)

    print(l_list)
    #l_list = [{'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_click_tracking', 'num': 302}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_dl_start', 'num': 182}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_dl_success', 'num': 101}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_impression', 'num': 301}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_install_start', 'num': 101}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_install_success', 'num': 42}]
    l2 = l_list
    l_list = list()
    tmp_set = set()

    for i_dict in l2:
        tmp_dict = dict()
        date1 = i_dict.get('ngx_date')
        if date1 not in tmp_set:
            tmp_set.add(date1)
        else:
            continue

        for i_i_dict in l2:
            date2 = i_i_dict.get('ngx_date')

            if date1 == date2:
                tmp_dict['ngx_date'] = i_i_dict.get('ngx_date')
                tmp_dict[str(i_i_dict.get('event_type'))] = i_i_dict.get('num')
        l_list.append(tmp_dict)
        tmp_dict = dict()

    print(l_list)
    for i_dict in l_list:
        cpd = Cpd()
        for key, value in i_dict.items():

            if key == 'ngx_date':
                a = sql_tool.strint_toDate(i_dict.get('ngx_date'))
                cpd.ngx_date = sql_tool.strint_toDate(i_dict.get('ngx_date'))
            if key == 'xad_click_tracking':
                cpd.xad_click_tracking = key
                cpd.xad_click_tracking_nums = value
            if key == 'xad_dl_start':
                cpd.xad_dl_start = key
                cpd.xad_dl_start_nums = value
            if key == 'xad_dl_success':
                cpd.xad_dl_success = key
                cpd.xad_dl_success_nums = value
            if key == 'xad_impression':
                cpd.xad_impression = key
                cpd.xad_impression_nums = value
            if key == 'xad_install_start':
                cpd.xad_install_start = key
                cpd.xad_install_start_nums = value
            if key == 'xad_install_success':
                cpd.xad_install_success = key
                cpd.xad_install_success_nums = value
        cpd.save()