机器学习: Python with Recurrent Neural Network

之前我们介绍了Recurrent neural network (RNN) 的原理:

http://blog.csdn.net/matrix_space/article/details/53374040

http://blog.csdn.net/matrix_space/article/details/53376870　　

这里，我们构建一个简单的RNN网络，激励函数我们用sigmoid 函数，利用这个网络，我们来测试二进制数的运算。网络重复模块的表达式是：

ht=σ(Wh⋅ht−1+Wi⋅Xt)

ot=σ(Wo⋅ht)

e=12(yt−ot)2

import copy, numpy as np

np.random.seed(0)

# compute sigmoid nonlinearity
# 定义sigmoid 函数
def sigmoid (x):
    output = 1 / (1+np.exp(-x))
    return output

# convert output to sigmoid function to its derivative
# 定义sigmoid 函数的导数
def sigmoid_output_to_derivative(output):
    return  output*(1-output)

# training dataset generation
# 生成训练集
int2binary = {}
binary_dim = 8
max_number = pow (2, binary_dim)
binary = np.unpackbits(np.array([range(max_number)], dtype=np.uint8).T, axis=1)
# np.unpackbits 是将一个uint8的数组元素都转换成０－１二进制形式，这里max_number 是256,　
# binary 里面一共存了0-255 共 256 个二进制数


for i in range(max_number):
    int2binary[i]=binary[i]

# input parameters
alpha = 0.1
input_dim = 2
hidden_dim = 16
output_dim = 1

# weight 的初始化
synapse_0 = 2 * np.random.random((input_dim, hidden_dim))-1
synapse_1 = 2 * np.random.random((hidden_dim, output_dim))-1
synapse_h = 2 * np.random.random((hidden_dim, hidden_dim)) -1

synapse_0_update = np.zeros_like(synapse_0)
synapse_1_update = np.zeros_like(synapse_1)
synapse_h_update = np.zeros_like(synapse_h)

for j in range(10000):
    # 生成一个0-128之间的随机数
    # 获取这个数的二进制序列
    a_int = np.random.randint(max_number/2)
    a = int2binary [a_int]　　　

    b_int = np.random.randint(max_number/2)
    b = int2binary [b_int]

    c_int = a_int + b_int
    c = int2binary [c_int]

    d = np.zeros_like(c)

    overallError = 0

    layer_2_deltas = list ()
    layer_1_values = list ()
    layer_1_values.append(np.zeros(hidden_dim))

    # moving along the positions in the binary encoding
    for position in range(binary_dim):

        # generate input and output
        X = np.array([[a[binary_dim-position-1], b[binary_dim-position-1]]])
        y = np.array([[c[binary_dim-position-1]]]).T

　　　　　# 计算重复模块的隐含层的输入和输出
        layer_1 = sigmoid(np.dot(X, synapse_0) + np.dot(layer_1_values[-1], synapse_h))

        layer_2 = sigmoid(np.dot(layer_1, synapse_1))

　　　　　# BP
        layer_2_error = y-layer_2
        layer_2_deltas.append((layer_2_error)*sigmoid_output_to_derivative(layer_2))
        overallError += np.abs(layer_2_error[0])

        d[binary_dim-position-1] = np.round(layer_2[0][0])

        layer_1_values.append(copy.deepcopy(layer_1))

    future_layer_1_delta = np.zeros(hidden_dim)

    for position in range (binary_dim):

        X = np.array([[a[position], b[position]]])
        layer_1 = layer_1_values [-position-1]
        pre_layer_1 = layer_1_values[-position-2]

        layer_2_delta = layer_2_deltas[-position-1]

        layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) + layer_2_delta.dot(
            synapse_1.T)) * sigmoid_output_to_derivative(layer_1)

        # weight update
        synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)
        synapse_h_update += np.atleast_2d(pre_layer_1).T.dot(layer_1_delta)
        synapse_0_update += X.T.dot(layer_1_delta)

        future_layer_1_delta = layer_1_delta

    synapse_0 += synapse_0_update * alpha
    synapse_1 += synapse_1_update * alpha
    synapse_h += synapse_h_update * alpha

    synapse_0_update *= 0
    synapse_1_update *= 0
    synapse_h_update *= 0

    # print out progress
    if (j % 500 == 0):
        print ("Error: ", str(overallError))
        print ("Pred:", str(d))
        print ("True:", str(c))
        out = 0
        for index, x in enumerate(reversed(d)):
            out += x*pow(2, index)
        print (str(a_int) + "+" + str(b_int) + "=" + str(out))
        print ("---------------")

运行结果：

('Error: ', '[ 3.45638663]')
('Pred:', '[0 0 0 0 0 0 0 1]')
('True:', '[0 1 0 0 0 1 0 1]')
9+60=1
---------------
('Error: ', '[ 4.02253884]')
('Pred:', '[0 1 1 0 1 0 1 1]')
('True:', '[1 0 0 0 0 0 0 1]')
112+17=107
---------------
('Error: ', '[ 3.63389116]')
('Pred:', '[1 1 1 1 1 1 1 1]')
('True:', '[0 0 1 1 1 1 1 1]')
28+35=255
---------------
('Error: ', '[ 3.99234598]')
('Pred:', '[1 1 0 1 1 0 1 0]')
('True:', '[1 0 1 1 0 0 1 1]')
78+101=218
---------------
('Error: ', '[ 3.91366595]')
('Pred:', '[0 1 0 0 1 0 0 0]')
('True:', '[1 0 1 0 0 0 0 0]')
116+44=72
---------------
('Error: ', '[ 3.65154804]')
('Pred:', '[1 1 0 1 1 0 1 0]')
('True:', '[1 1 0 1 1 1 1 0]')
122+100=218
---------------
('Error: ', '[ 3.72191702]')
('Pred:', '[1 1 0 1 1 1 1 1]')
('True:', '[0 1 0 0 1 1 0 1]')
4+73=223
---------------
('Error: ', '[ 3.35048888]')
('Pred:', '[1 0 0 1 1 0 0 1]')
('True:', '[1 0 0 1 0 0 0 1]')
76+69=153
---------------
('Error: ', '[ 3.5852713]')
('Pred:', '[0 0 0 0 1 0 0 0]')
('True:', '[0 1 0 1 0 0 1 0]')
71+11=8
---------------
('Error: ', '[ 2.43239777]')
('Pred:', '[0 1 1 0 1 0 1 1]')
('True:', '[0 1 1 0 1 0 1 1]')
72+35=107
---------------
('Error: ', '[ 2.53352328]')
('Pred:', '[1 0 1 0 0 0 1 0]')
('True:', '[1 1 0 0 0 0 1 0]')
81+113=162
---------------
('Error: ', '[ 1.87382863]')
('Pred:', '[0 1 1 0 0 0 1 0]')
('True:', '[0 1 1 0 0 0 1 0]')
21+77=98
---------------
('Error: ', '[ 0.57691441]')
('Pred:', '[0 1 0 1 0 0 0 1]')
('True:', '[0 1 0 1 0 0 0 1]')
81+0=81
---------------
('Error: ', '[ 0.75100965]')
('Pred:', '[0 0 1 1 1 1 0 0]')
('True:', '[0 0 1 1 1 1 0 0]')
49+11=60
---------------
('Error: ', '[ 1.42589952]')
('Pred:', '[1 0 0 0 0 0 0 1]')
('True:', '[1 0 0 0 0 0 0 1]')
4+125=129
---------------
('Error: ', '[ 0.6594703]')
('Pred:', '[0 1 1 0 1 1 0 0]')
('True:', '[0 1 1 0 1 1 0 0]')
80+28=108
---------------
('Error: ', '[ 0.47477457]')
('Pred:', '[0 0 1 1 1 0 0 0]')
('True:', '[0 0 1 1 1 0 0 0]')
39+17=56
---------------
('Error: ', '[ 0.7200904]')
('Pred:', '[1 0 1 0 1 0 0 0]')
('True:', '[1 0 1 0 1 0 0 0]')
123+45=168
---------------
('Error: ', '[ 0.21595037]')
('Pred:', '[0 0 0 0 1 1 1 0]')
('True:', '[0 0 0 0 1 1 1 0]')
11+3=14
---------------
('Error: ', '[ 0.52112049]')
('Pred:', '[1 0 1 0 1 0 1 1]')
('True:', '[1 0 1 0 1 0 1 1]')
71+100=171
---------------

参考来源：

https://github.com/llSourcell/recurrent_neural_net_demo