爬取 爱笔智能 招聘职位

爬取爱笔智能招聘职位

http://aibee.com/cn/joinus.aspx 

import requests
from urllib.parse import urlencode
from pyquery import PyQuery as pq
from pymongo import MongoClient
import json


base_url = 'http://aibee.com/cn/joinus.aspx?action=jobinfo&'

headers = {
    'Host': 'aibee.com',
    'Referer': 'http://aibee.com/cn/joinus.aspx',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
} 

client = MongoClient()
db = client['aibee']
collection = db['aibee']
max_id = 50



def get_page(id):  

    formData = {  
            'id': id,
        }  
     

    data = urlencode(formData)
    url = base_url + urlencode(formData)    
    try:
        response = requests.get(url, data=data, headers=headers)
        if response.status_code == 200:

            return response.json()
    except requests.ConnectionError as e:
        print('Error', e.args)


def parse_page(json_1):
    if json_1:
        items = json_1.get('shuzu')
        for item in items:
            if id == 1 :
                continue
            else:
                
                aibee = {}
                aibee['id'] = item.get('id')
                aibee['title'] = item.get('title')
                aibee['zhize'] = pq(item.get('zhize')).text()
                aibee['yaoqiu'] = pq(item.get('yaoqiu')).text()
                aibee['dtt'] = item.get('dtt')
                aibee['emailaddr'] = item.get('emailaddr')
                yield aibee


def write_to_file(content):
    with open('aibee.json','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'
')
        f.close()

def save_to_mongo(result):
    if collection.insert(result):
        print('Saved to Mongo')


if __name__ == '__main__':
    for id in range(1, max_id + 1):
        json_1 = get_page(id)
        #print(json_1)

        results = parse_page(json_1)
        for result in results:
            print(result)
            write_to_file(result)
            save_to_mongo(result)

 或者：

import requests
from urllib.parse import urlencode
from pyquery import PyQuery as pq
from pymongo import MongoClient
import json


url = 'http://aibee.com/cn/joinus.aspx?action=jobinfo'

headers = {
    'Host': 'aibee.com',
    'Referer': 'http://aibee.com/cn/joinus.aspx',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
} 

client = MongoClient()
db = client['aibee']
collection = db['aibee']
max_id = 50



def get_page(id):  

    formData = {  
            'id': id,
        }  
    try:
        r = requests.post(url, data=formData, headers=headers)
        if r.status_code == 200:
            return r.json()
    except requests.ConnectionError as e:
        print('Error', e.args)


def parse_page(json_1):
    if json_1:
        items = json_1.get('shuzu')
        for item in items:
            if id == 1 :
                continue
            else:
                
                aibee = {}
                aibee['id'] = item.get('id')
                aibee['title'] = item.get('title')
                aibee['zhize'] = pq(item.get('zhize')).text()
                aibee['yaoqiu'] = pq(item.get('yaoqiu')).text()
                aibee['dtt'] = item.get('dtt')
                aibee['emailaddr'] = item.get('emailaddr')
                yield aibee


def write_to_file(content):
    with open('aibee.json','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'
')
        f.close()

def save_to_mongo(result):
    if collection.insert(result):
        print('Saved to Mongo')


if __name__ == '__main__':
    for id in range(1, max_id + 1):
        json_1 = get_page(id)
        #print(json_1)

        results = parse_page(json_1)
        for result in results:
            print(result)
            write_to_file(result)
            save_to_mongo(result)