• pandas字符串/列等操作


    # In[1]
    import pandas as pd 
    import numpy as np
    import json
    import os 
    import re 
    
    # In[2]
    # !pwd
    os.chdir('./root/FAQ/')
    
    # In[2]
    with open('./data/all_data.txt', 'r') as f:
        data = f.read().split('[SEP]')
        AQ = pd.DataFrame(data, columns=['question'])
        
    # In[3]
    # 冬奥会类(可以回答),设置为正例(2)
    AQ['question'] = AQ['question'].str.strip()
    AQ['label'] = 2
    
    # In[3]
    AQ          # 冬奥会类, 后面只取一部分
    
    # In[4]
    # 数据集1(因为量少,多弄几个)
    # 体育-非东奥; 非体育类;
    f = open('./data/Negative.json', 'rb')
    line = f.read().decode('utf8', 'ignore')
    f.close()
    with open('./data/Negative.txt', 'w') as f:
        f.write('[' + ','.join(line.split()) + ']')
    
    # In[5]
    # 体育类-非东奥; 非体育类
    NoAQ = pd.read_json('./data/Negative.txt')
    NoAQ['title']
    
    # In[4]
    # 数据集2
    # 体育-非东奥; 非体育类; 
    f = open('./data/Negative02.json', 'rb')
    line2 = f.read().decode('utf8', 'ignore')
    f.close()
    with open('./data/Negative02.txt', 'w') as f:
        f.write('[' + ','.join(line2.split()) + ']')
    
    # In[5]
    NoAQ02 = pd.read_json('./data/Negative02.txt')
    NoAQ02['title']
    
    # 数据集3
    
    
    # In[5]
    NoAQ = NoAQ.append(NoAQ02)
    print(len(NoAQ))
    
    # In[6]
    # 285155
    # 285155
    train_len = len(NoAQ)
    AQ = AQ.iloc[:train_len]
    print('东奥(可回答): ', len(AQ))
    # 285155
    print('体育-非东奥 + 非体育类:', train_len) 
    AQ
    
    # In[7]
    # 筛选体育-非东奥;非体育类;
    # 改列名
    NoAQ = NoAQ.drop(labels=['answer', 'desc', 'url'], axis=1)
    NoAQ.columns = ['question']
    NoAQ
    
    # In[8]
    # 非体育类:0
    NoAQ['label'] = 0
    NoAQ
    
    # In[9]
    # 体育类,非东奥类:1
    sports = ['雪', '赢', '速', '跳', '滑', '冬', '自由', '冰', '剧烈', '开赛', 'vs', '武术', '奥运会', '健身', '跑步', '打球', '强', '壮', '体育', '运动员', '运动', '活动', '训练', '得分', '比赛', '参赛', '赢', '球']
    found = NoAQ['question'].str.contains('|'.join(sports))
    sports_idx = NoAQ['question'][found].index
    print(len(sports_idx))
    NoAQ['label'].iloc[sports_idx] = 1
    
    # In[9]
    NoAQ.loc[NoAQ['label'] == 1]
    NoAQ.loc[NoAQ['label'] == 0]
    # In[10]
    # 整合数据
    AQ = AQ.append(NoAQ)
    # In[11]
    AQ
    # In[11]
    AQ.to_csv('./data/FAQ.csv', sep='	')
    
    # In[12]
    test = pd.read_csv('./data/FAQ.csv', sep='	')
    test
    # %%
    
    
  • 相关阅读:
    深度学习开源框架大礼包
    python3实现多线程
    TNN MatConvertParam参数scale和bias设置
    pytorch加载预训练模型
    linux下遍历文件夹及获取路径下文件名字(字符串操作)
    已解决:在/var/spool/cron/root中执行shell脚本定时任务,ipvsadmsave n > x.save保存的文件为空
    http_build_query
    API接口返回数据结构构建类
    使用vscode+夜神模拟器遇到连接不上的解决方法
    build.gradle
  • 原文地址:https://www.cnblogs.com/douzujun/p/14375660.html
Copyright © 2020-2023  润新知