pandas模块
pandas官方文档链接
pandas基于numpy,可以看成是处理文本或表格数据,pandas中主要有Series数据结构,类似于numpy的一维数组,DataFrame类似于多维表格数据结构
pandas是Python数据分析的核心模块,主要有五大功能
- 支持文件存取操作,支持数据库(sql)、html、json、pickle、csv、Excel、sas、hdf等
- 支持增删改查、切片、高阶函数、分组聚合等操作,以及dict和list互相转换
- 支持多表拼接合并操作
- 支持简单绘图操作
- 支持简单统计分析操作
import pandas as pd
import numpy as np
# Series
import numpy as np
import pandas as pd
arr = np.array([1,2,3,4,np.nan])
print(arr)
[ 1. 2. 3. 4. nan]
s = pd.Series(arr)
print(s)
0 1.0
1 2.0
2 3.0
3 4.0
4 NaN
dtype: float64
# DataFrame
dates = pd.date_range('20000202',periods = 6)
print(dates)
DatetimeIndex(['2000-02-02', '2000-02-03', '2000-02-04', '2000-02-05',
'2000-02-06', '2000-02-07'],
dtype='datetime64[ns]', freq='D')
np.random.seed(1)
arr = 10*np.random.randn(6,4)
print(arr)
[[ 16.24345364 -6.11756414 -5.28171752 -10.72968622]
[ 8.65407629 -23.01538697 17.44811764 -7.61206901]
[ 3.19039096 -2.49370375 14.62107937 -20.60140709]
[ -3.22417204 -3.84054355 11.33769442 -10.99891267]
[ -1.72428208 -8.77858418 0.42213747 5.82815214]
[-11.00619177 11.4472371 9.01590721 5.02494339]]
df = pd.DataFrame(arr,index=dates, columns=['c1','c2','c3','c4'])
df
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
16.243454 |
-6.117564 |
-5.281718 |
-10.729686 |
| 2000-02-03 |
8.654076 |
-23.015387 |
17.448118 |
-7.612069 |
| 2000-02-04 |
3.190391 |
-2.493704 |
14.621079 |
-20.601407 |
| 2000-02-05 |
-3.224172 |
-3.840544 |
11.337694 |
-10.998913 |
| 2000-02-06 |
-1.724282 |
-8.778584 |
0.422137 |
5.828152 |
| 2000-02-07 |
-11.006192 |
11.447237 |
9.015907 |
5.024943 |
# 使用pandas读取字典形式的数据
df2=pd.DataFrame({'a':1,'b':[2,3],'c':np.arange(2),'d':'hello'})
df2
|
a |
b |
c |
d |
| 0 |
1 |
2 |
0 |
hello |
| 1 |
1 |
3 |
1 |
hello |
DataFrame属性
| 属性 |
详解 |
| dtype |
查看数据类型 |
| index |
查看行序列或者索引 |
| columns |
查看各列的标签 |
| values |
查看数据框内的数据，也即不含表头索引的数据 |
| describe |
查看数据每一列的极值，均值，中位数，只可用于数值型数据 |
| transpose |
转置，也可用Ｔ来操作 |
| sort_index |
排序，可按行或列index排序输出 |
| sort_values |
按数据值来排序 |
# 查看数据类型
print(df2.dtypes)
a int64
b int64
c int32
d object
dtype: object
df.index
DatetimeIndex(['2000-02-02', '2000-02-03', '2000-02-04', '2000-02-05',
'2000-02-06', '2000-02-07'],
dtype='datetime64[ns]', freq='D')
df.columns
Index(['c1', 'c2', 'c3', 'c4'], dtype='object')
df.values
array([[ 16.24345364, -6.11756414, -5.28171752, -10.72968622],
[ 8.65407629, -23.01538697, 17.44811764, -7.61206901],
[ 3.19039096, -2.49370375, 14.62107937, -20.60140709],
[ -3.22417204, -3.84054355, 11.33769442, -10.99891267],
[ -1.72428208, -8.77858418, 0.42213747, 5.82815214],
[-11.00619177, 11.4472371 , 9.01590721, 5.02494339]])
df.describe()
|
c1 |
c2 |
c3 |
c4 |
| count |
6.000000 |
6.000000 |
6.000000 |
6.000000 |
| mean |
2.022213 |
-5.466424 |
7.927203 |
-6.514830 |
| std |
9.580084 |
11.107772 |
8.707171 |
10.227641 |
| min |
-11.006192 |
-23.015387 |
-5.281718 |
-20.601407 |
| 25% |
-2.849200 |
-8.113329 |
2.570580 |
-10.931606 |
| 50% |
0.733054 |
-4.979054 |
10.176801 |
-9.170878 |
| 75% |
7.288155 |
-2.830414 |
13.800233 |
1.865690 |
| max |
16.243454 |
11.447237 |
17.448118 |
5.828152 |
df.T
|
2000-02-02 00:00:00 |
2000-02-03 00:00:00 |
2000-02-04 00:00:00 |
2000-02-05 00:00:00 |
2000-02-06 00:00:00 |
2000-02-07 00:00:00 |
| c1 |
16.243454 |
8.654076 |
3.190391 |
-3.224172 |
-1.724282 |
-11.006192 |
| c2 |
-6.117564 |
-23.015387 |
-2.493704 |
-3.840544 |
-8.778584 |
11.447237 |
| c3 |
-5.281718 |
17.448118 |
14.621079 |
11.337694 |
0.422137 |
9.015907 |
| c4 |
-10.729686 |
-7.612069 |
-20.601407 |
-10.998913 |
5.828152 |
5.024943 |
# 按行标签从大到小排序
df.sort_index(axis=0)
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
16.243454 |
-6.117564 |
-5.281718 |
-10.729686 |
| 2000-02-03 |
8.654076 |
-23.015387 |
17.448118 |
-7.612069 |
| 2000-02-04 |
3.190391 |
-2.493704 |
14.621079 |
-20.601407 |
| 2000-02-05 |
-3.224172 |
-3.840544 |
11.337694 |
-10.998913 |
| 2000-02-06 |
-1.724282 |
-8.778584 |
0.422137 |
5.828152 |
| 2000-02-07 |
-11.006192 |
11.447237 |
9.015907 |
5.024943 |
# 按列标签从大到小排列
df2.sort_index(axis =0)
|
a |
b |
c |
d |
| 0 |
1 |
2 |
0 |
hello |
| 1 |
1 |
3 |
1 |
hello |
df2.sort_values(by='a')
|
a |
b |
c |
d |
| 0 |
1 |
2 |
0 |
hello |
| 1 |
1 |
3 |
1 |
hello |
DataFrame取值
df
|
5.1 |
Unnamed: 1 |
1.4 |
0.2 |
| 0 |
4.9 |
3.0 |
1.4 |
0.2 |
| 1 |
4.7 |
3.2 |
NaN |
0.2 |
| 2 |
7.0 |
3.2 |
4.7 |
1.4 |
| 3 |
6.4 |
3.2 |
4.5 |
1.5 |
| 4 |
6.9 |
3.1 |
4.9 |
NaN |
| 5 |
NaN |
NaN |
NaN |
NaN |
df['c3']
2000-02-02 -5.281718
2000-02-03 17.448118
2000-02-04 14.621079
2000-02-05 11.337694
2000-02-06 0.422137
2000-02-07 9.015907
Freq: D, Name: c3, dtype: float64
df[0:3]
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
16.243454 |
-6.117564 |
-5.281718 |
-10.729686 |
| 2000-02-03 |
8.654076 |
-23.015387 |
17.448118 |
-7.612069 |
| 2000-02-04 |
3.190391 |
-2.493704 |
14.621079 |
-20.601407 |
# 通过自定义标签来选择数据
df.loc['2000-02-02':'2000-02-04']
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
16.243454 |
-6.117564 |
-5.281718 |
-10.729686 |
| 2000-02-03 |
8.654076 |
-23.015387 |
17.448118 |
-7.612069 |
| 2000-02-04 |
3.190391 |
-2.493704 |
14.621079 |
-20.601407 |
df.values
array([[ 16.24345364, -6.11756414, -5.28171752, -10.72968622],
[ 8.65407629, -23.01538697, 17.44811764, -7.61206901],
[ 3.19039096, -2.49370375, 14.62107937, -20.60140709],
[ -3.22417204, -3.84054355, 11.33769442, -10.99891267],
[ -1.72428208, -8.77858418, 0.42213747, 5.82815214],
[-11.00619177, 11.4472371 , 9.01590721, 5.02494339]])
df.iloc[2,1]
-2.493703754774101
# 通过行索引选择数据
print(df.iloc[2,1])
-2.493703754774101
df.iloc[1:4,2:4] # 行怎么取 列怎么取
|
c3 |
c4 |
| 2000-02-03 |
17.448118 |
-7.612069 |
| 2000-02-04 |
14.621079 |
-20.601407 |
| 2000-02-05 |
11.337694 |
-10.998913 |
# 使用逻辑替换
df[df['c1']>0]
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
16.243454 |
-6.117564 |
-5.281718 |
-10.729686 |
| 2000-02-03 |
8.654076 |
-23.015387 |
17.448118 |
-7.612069 |
| 2000-02-04 |
3.190391 |
-2.493704 |
14.621079 |
-20.601407 |
# DataFrame值替换
df.iloc[0:3,0:2]
|
c1 |
c2 |
| 2000-02-02 |
16.243454 |
-6.117564 |
| 2000-02-03 |
8.654076 |
-23.015387 |
| 2000-02-04 |
3.190391 |
-2.493704 |
df[df['c1']>0] =100
df # 原地修改
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
100.000000 |
100.000000 |
100.000000 |
100.000000 |
| 2000-02-03 |
100.000000 |
100.000000 |
100.000000 |
100.000000 |
| 2000-02-04 |
100.000000 |
100.000000 |
100.000000 |
100.000000 |
| 2000-02-05 |
-3.224172 |
-3.840544 |
11.337694 |
-10.998913 |
| 2000-02-06 |
-1.724282 |
-8.778584 |
0.422137 |
5.828152 |
| 2000-02-07 |
-11.006192 |
11.447237 |
9.015907 |
5.024943 |
df
|
c1 |
c2 |
c3 |
c4 |
| 2000-02-02 |
100.000000 |
100.000000 |
100.000000 |
100.000000 |
| 2000-02-03 |
100.000000 |
100.000000 |
100.000000 |
100.000000 |
| 2000-02-04 |
100.000000 |
100.000000 |
100.000000 |
100.000000 |
| 2000-02-05 |
-3.224172 |
-3.840544 |
11.337694 |
-10.998913 |
| 2000-02-06 |
-1.724282 |
-8.778584 |
0.422137 |
5.828152 |
| 2000-02-07 |
-11.006192 |
11.447237 |
9.015907 |
5.024943 |
读取CSV文件
import pandas as pd
from io import StringIO
test_data = '''
5.1,,1.4,0.2
4.9,3.0,1.4,0.2
4.7,3.2,,0.2
7.0,3.2,4.7,1.4
6.4,3.2,4.5,1.5
6.9,3.1,4.9,
,,,
'''
# df = pd.read_csv('C:/Users/test_data.csv')
test_data = StringIO(test_data)
df = pd.read_csv(test_data)
df = pd.read_excel(test_data)
df.columns = ['c1', 'c2', 'c3', 'c4']
df
# 通过在isnull()方法后使用sum()方法即可获得该数据集某个特征含有多少个缺失值
print(df.isnull().sum())
df
# axis = 0 删除有NaN值得行
df dropa(axis=1)
# 删除axis = 1 删除有NaN值的列
df.dropa(axis = 1)
# 删除全为NaN值的行或列
df.dropa(how = all)
导入导出数据
df = pd.read_csv(filename)读取文件, 使用df.to_csv(filename)来保存文件
合并数据
df1 = pd.DataFrame(np.zeros(3,4))
df1
df2 = pd.DataFrame(np.ones(3,4))
df2