这次实现了缺省值处理部分内容,主要有将缺省值按中位数,平均值补全。将缺省行或者列去掉。
#缺省值补全 @app.route('/data_clean_supply') def data_clean_supply(): supply_type = request.values.get("supply_type") flag='1' try: global data_clean data_clean =dataclean.data_clean_supply(data_clean,supply_type) # 去重后的数据保存,为以后处理缺省值 except (Exception, BaseException) as e: exstr = traceback.format_exc() print( exstr) flag='0' print(supply_type+"flag:"+flag) return jsonify({"cog":flag}) pass #缺省值去除 @app.route("/data_clean_remove") def data_clean_remove(): action_on = str(request.values.get("action_on")) type_on = str(request.values.get("type_on")) min_num = str(request.values.get("min_num")) flag='1' try: global data_clean dataclean.data_clean_remove(data_clean,action_on,type_on,min_num) except (Exception, BaseException) as e: exstr = traceback.format_exc() print(exstr) flag='0' print( "min_num:"+min_num+",type_on:"+type_on+",action_on:"+action_on+",flag:"+flag) return jsonify({"cog":flag}) pass
#获取要补充的数值 def data_clean_supply_num(temp_col_not_nan,suplly_type): if (suplly_type == "median"): return temp_col_not_nan.median() if(suplly_type == "mean"): return temp_col_not_nan.mean() #补全缺省值 def data_clean_supply(data_clean,suplly_type): for i in range(data_clean.shape[1]):#遍历列数 try: temp_col=data_clean.iloc[:,i]#取一列 temp_col_not_nan=temp_col[temp_col==temp_col].astype('float')#当前一列不为nan,使用布尔索引 mean=data_clean_supply_num(temp_col_not_nan,suplly_type) flag_list=[] num=len(temp_col) for j in range(num): if(temp_col.iloc[j]!=temp_col.iloc[j]): flag_list.append(mean) else: flag_list.append(temp_col.iloc[j]) data_clean.iloc[:,i]=flag_list# 填充均值,必须整列赋值 except: pass #print("不是数字类型") return data_clean pass #去除缺省行或列 def data_clean_remove(data_clean,action_on,type_on,minnum): return data_clean.dropna(axis=action_on, how=type_on, thresh=int(minnum)) pass
有一点要注意由于从数据库中读出的缺省值是‘’而不是nan所以要转为nan
#把‘’转换为nan for j in range(num_1): flag_list=[] for i in range(num_0): if(df.iloc[i][j]==''): flag_list.append(np.nan) else: flag_list.append(df.iloc[i][j]) df.iloc[:,j]=flag_list#必须整列赋值,如果单个赋值则会失败
前台部分:
<blockquote class="layui-elem-quote layui-text"> <h2>缺省值处理</h2> </blockquote> <fieldset class="layui-elem-field layui-field-title" style="margin-top: 30px;"> <legend>缺省值补全</legend> </fieldset> <form class="layui-form" action=""> <div class="layui-form-item"> <label class="layui-form-label">补全方式</label> <div class="layui-input-block"> <select name="interest" lay-filter="aihao" id="supply_type"> <option value=""></option> <option value="median">中位数</option> <option value="mode" selected="">众数</option> <option value="average">平均数</option> </select> <button type="button" class="layui-btn layui-btn-normal" id="supply_submit" onclick="supply_submit">确定</button> </div> </div> </form> <fieldset class="layui-elem-field layui-field-title" style="margin-top: 30px;"> <legend>缺省值去除</legend> </fieldset> <form class="layui-form" action=""> <div class="layui-form-item"> <label class="layui-form-label">作用于</label> <div class="layui-input-block"> <input type="radio" name="action_on" value="index" title="行" checked=""> <input type="radio" name="action_on" value="columns" title="列"> </div> </div> <div class="layui-form-item"> <label class="layui-form-label">非空值最小数</label> <div class="layui-input-block"> <input type="text" id="min_num" lay-verify="title" autocomplete="off" placeholder="非空值小于此数目的行或列将被删除" class="layui-input"> </div> </div> <div class="layui-form-item"> <label class="layui-form-label">方式</label> <div class="layui-input-block"> <input type="radio" name="type_on" value="all" title="all" checked=""> <input type="radio" name="type_on" value="any" title="any"> </div> <button type="button" class="layui-btn layui-btn-normal" id="remove_submit" >确定</button> </div> </form>
<script> $("#supply_submit").click(function() { supply_type = $("#supply_type").find("option:selected").val() $.ajax({ type: "GET", url: "/data_clean_supply?supply_type=" + supply_type, dataType: "json", success: function (data) { if (data.cog == 1) { alert("操作成功") } else { alert("操作失败") } } }) }) $("#remove_submit").click(function() { action_on=$("input[name='action_on']:checked").val(); type_on=$("input[name='type_on']:checked").val(); min_num=$("#min_num").val() $.ajax({ type: "GET", url: "/data_clean_remove?action_on="+action_on+"&type_on="+type_on+"&min_num="+min_num, dataType: "json", success: function (data) { if (data.cog == 1) { alert("操作成功") } else { alert("操作失败") } } }) }) </script>