线性回归的Spark实现 [Linear Regression / Machine Learning / Spark]

1- 问题提出

---

2- 线性回归

---

3- 理论推导

---

4- Python/Spark实现

# -*- coding: utf-8 -*-
from pyspark import SparkContext


theta = [0, 0]
alpha = 0.001

sc = SparkContext('local')

def func_theta_x(x):
    return sum([i * j for i, j in zip(theta, x)])

def cost(x):
    thx = func_theta_x(x)
    return thx - x[-1]

def partial_theta(x):
    dif = cost(x)
    return [dif * i for i in x[:-1]]

rdd = sc.textFile('/home/freyr/linearRegression.txt')
        .map(lambda line: map(float, line.strip().split('	')))

maxiter = 400
iter = 0
while True:
    parTheta = rdd.map(partial_theta)
                  .reduce(lambda x, y: [i + j for i, j in zip(x, y)])

    for i in range(2):
        theta[i] = theta[i] - alpha * parTheta[i]

    iter += 1

    if iter <= maxiter:
        if sum(map(abs, parTheta)) <= 0.01:
            print 'I get it!!!'
            print 'Iter = %s' % iter
            print 'Theta = %s' % theta
            break
    else:
        print 'Failed...'
        break

PS: 1. linearRegression.txt

---