统计人口性别和身高

自动生成实验数据:

自动生成1000个样例数据

from pyspark import SparkContext,SparkConf

import random

def getRandomGender():

rand = random.randint(0,2)+1

if rand % 2 ==0:

return "M"

else:

return "F"

sc = SparkContext('local','createDataFile')

outputFile = "file:///usr/local/spark/mycode/exercise/peopleage1.txt"

Arrays=[]

for i in range(0,1000):

height = random.randint(0,230)

if height < 50:

height = height +50

gender = getRandomGender()

if (height <100 and gender =="M"):

height= height+100

if (height <100 and gender =="F"):

height = height+50

Arrays.append(str(i) +" "+gender+" "+str(height))

rdd = sc.parallelize(Arrays)

rdd.saveAsTextFile(outputFile)

 

 

 

统计人口与身高的代码:

from pyspark import SparkContext

sc = SparkContext('local','CountPeopleData')

#读入数据

RDD = sc.textFile("file:///usr/local/spark/mycode/exercise/people.txt")

#按空格切割，并且通过filter过滤，得到全是M的新的RDD，然后wordcount

totalMen = RDD.flatMap(lambda line : line.split(" ")).filter(lambda a : a == "M").map(lambda a : (a,1)).reduceByKey(lambda a,b : (a+b))

totalWomen = RDD.flatMap(lambda line: line.split(" ")).filter( lambda a: a =="F").map(lambda a: (a,1)).reduceByKey(lambda a,b :(a+b))

 

#保留性别与身高部分

totaPeopleHeightinfo = RDD.map(lambda line : (line.split(" ")[1] + " "+line.split(" ")[2]))

 

#过滤，留下全是M 的数据，然后从小到大排序，取第一个，为最小

totalMenLowHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "M").map(lambda a: (a,1)).sortByKey().first()

#过滤，留下全是F的数据，然后设置ascending=False，大到小排序，取第一个，为最大

totalMenBestHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "M").map(lambda a : (a,1)).sortByKey(ascending=False).first()

#过滤，留下全是M 的数据，然后从小到大排序，取第一个，为最小

totalWoMenLowHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "F").map(lambda a: (a,1)).sortByKey().first()

#过滤，留下全是F的数据，然后设置ascending=False，大到小排序，取第一个，为最大

totalWoMenBestHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "F").map(lambda a : (a,1)).sortByKey(ascending=False).first()

print("男生总数:",totalMen.collect())

print("女生总数:",totalWomen.collect())

print("男生最低身高为:",totalMenLowHieghtinfo)

print("男生最高身高为:",totalMenBestHieghtinfo)

print("女生最低身高为:",totalWoMenLowHieghtinfo)

print("女生最高身高为:",totalWoMenBestHieghtinfo)