NIPS2020 论文下载 代码

# %% NIPS 2020 论文信息下载
import json
import os
import re

import pandas as pd
import requests
import tqdm
from bs4 import BeautifulSoup


os.chdir(os.path.dirname(os.path.abspath(__file__)))

# %%
PAPER_HASH_PATTERN = re.compile(r'poster_(?P<UID>w+).html')
SESSION_PATTERN = re.compile(r'Orals & Spotlights Track d+:s*(?P<session>[^;]*)')


def cleanup_string(s):
    s = s.strip()
    while '  ' in s:
        s = s.replace('  ', ' ')
    return s


def download_file(download_url, file_name=None):
    if file_name is None:
        file_name = os.path.basename(download_url)
    response = requests.get(download_url, stream=True)
    total = int(response.headers.get('Content-Length'))
    pbar = None
    if total is not None:
        pbar = tqdm.tqdm(desc=f'Downloading from {download_url} to {file_name}',
                         total=total, unit='B', unit_scale=True, unit_divisor=1000)
    with open(file_name, 'wb') as file:
        for chunk in response.iter_content(chunk_size=10240):
            if chunk:
                file.write(chunk)
            if pbar is not None:
                pbar.update(len(chunk))


# %%
# download paper list
if not os.path.exists('papers.json'):
    download_file('https://neurips.cc/virtual/2020/public/papers.json', file_name='papers.json')

# %%
# get oral paper list
oral_papers = set()
response = requests.get('https://neurips.cc/virtual/2020/public/f_orals.html')
soup = BeautifulSoup(response.text, 'html.parser')
for tag in soup.find_all('a', href=PAPER_HASH_PATTERN):
    href = tag['href']
    UID = PAPER_HASH_PATTERN.search(href).group('UID')
    oral_papers.add(UID)

# %%
# process paper list
with open('papers.json', mode='r') as file:
    data = json.load(file)

df = pd.DataFrame(columns=['ID', 'Category', 'Title', 'Authors', 'Keywords', 'Sessions', 'URL', 'Proceedings URL', 'PDF URL', 'UID'])
for i, paper in enumerate(tqdm.tqdm(data)):
    if paper['eventtype'] != 'Poster':
        continue

    UID = paper['UID']
    category = 'Poster'
    sessions = '; '.join(paper['sessions'])
    sessions = '; '.join([match.group('session') for match in SESSION_PATTERN.finditer(sessions)])
    sessions = cleanup_string(sessions)
    if sessions != '':
        category = 'Spotlight'
    if UID in oral_papers:
        category = 'Oral'

    keywords = set()
    for keyword in ('; '.join(paper['keywords'])).split('; '):
        keyword = cleanup_string(keyword)
        if keyword != '':
            keywords.add(keyword)
    keywords = '
'.join(sorted(keywords))

    paper = {
        'ID': paper['id'],
        'Category': category,
        'Title': cleanup_string(paper['title']),
        'Authors': cleanup_string(', '.join(paper['authors'])),
        'Keywords': keywords,
        'Sessions': sessions,
        'URL': f'https://neurips.cc/virtual/2020/public/poster_{UID}.html',
        'Proceedings URL': paper['paper_pdf_url'],
        'PDF URL': f'https://proceedings.neurips.cc/paper/2020/file/{UID}-Paper.pdf',
        'UID': UID
    }
    df.loc[len(df)] = paper

df['Category'] = pd.Categorical(df['Category'], categories=['Oral', 'Spotlight', 'Poster'])
df.sort_values(by=['Category', 'Sessions', 'Keywords'], inplace=True)
df.to_csv('paper_list.csv', index=False)

# %%
# get paper details
all_subject_areas = set()
for i, paper in enumerate(tqdm.tqdm(df.iloc, total=len(df))):
    if paper['Keywords'] == '':
        continue
    areas = set(paper['Keywords'].split('
'))
    all_subject_areas.update(areas)

try:
    all_subject_areas.remove('')
except KeyError:
    pass

df = df.reindex(columns=df.columns.to_list() + sorted(all_subject_areas))
for i, paper in enumerate(df.iloc):
    for area in paper['Keywords'].split('
'):
        if area != '':
            df[area][i] = 'Y'

df.to_csv('NeuraIPS Papers.csv', index=False)