Sigmoid函数导数图像如下，函数最大值约为0.25
根据BP算法，每次更新的权值会是多层sigmoid prime的乘积，随着层数增多会越来越小。
ReLu函数
f(x)=max(0,x)，导数在x>0时为1，x<0时为0。
使用ReLu可以一定程度减小运算量，避免层数增加的问题。
下面代码中，仅需把sigmoid替换成relu。
1 # coding:utf8 2 import cPickle 3 import numpy as np 4 5 class Network(object): 6 def __init__(self, sizes): 7 self.num_layers = len(sizes) 8 self.sizes = sizes 9 self.biases = [np.random.randn(y, 1) for y in sizes[1:]] # L(n-1)->L(n) 10 # self.weights = [np.random.randn(y, x) 11 self.weights = [np.random.randn(y, x)/np.sqrt(x) # improved weight initializer 12 for x, y in zip(sizes[:-1], sizes[1:])] 13 14 def feedforward(self, a): 15 for b_, w_ in zip(self.biases[:-1], self.weights[:-1]): 16 # a = self.sigmoid(np.dot(w_, a)+b_) 17 a = self.relu(np.dot(w_, a)+b_) 18 # a=self.sigmoid(np.dot(self.weights[-1], a)+self.biases[-1]) 19 a=self.relu(np.dot(self.weights[-1], a)+self.biases[-1]) 20 # a=self.softmax(np.dot(self.weights[-1], a)+self.biases[-1]) # add for softmax 21 return a 22 23 def SGD(self, training_data, test_data,epochs, mini_batch_size, eta=1.0, lambda_=0.1): 24 n_test = len(test_data) 25 n = len(training_data) 26 cx=range(epochs) 27 for j in cx: 28 self.cost = 0.0 29 np.random.shuffle(training_data) # shuffle 30 for k in xrange(0, n, mini_batch_size): 31 mini_batch = training_data[k:k+mini_batch_size] 32 self.update_mini_batch(mini_batch, eta,n) 33 self.cost+=0.5*lambda_*sum(np.linalg.norm(w_)**2 for w_ in self.weights) 34 print "Epoch {0}: cost={1} {2} / {3}, {4} / {5}".format(j, self.cost/n, 35 self.evaluate(training_data,1), n,self.evaluate(test_data), n_test) 36 37 def update_mini_batch(self, mini_batch, eta, n, lambda_=0.1): 38 for x, y in mini_batch: 39 delta_b, delta_w = self.backprop(x, y) 40 for i in range(len(self.weights)): # L2 regularization 41 self.weights[i]-=eta/len(mini_batch)*(delta_w[i] +lambda_/n*self.weights[i]) 42 self.biases -= eta/len(mini_batch)*delta_b 43 a=self.feedforward(x) 44 # cost=np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a))) cost entropy 45 cost=0.5*np.sum((a-y)**2) 46 self.cost += cost 47 48 def backprop(self, x, y): 49 b=np.zeros_like(self.biases) 50 w=np.zeros_like(self.weights) 51 a_ = x 52 a = [x] 53 for b_, w_ in zip(self.biases, self.weights): 54 # a_ = self.sigmoid(np.dot(w_, a_)+b_) 55 a_ = self.relu(np.dot(w_, a_)+b_) 56 a.append(a_) 57 for l in xrange(1, self.num_layers): 58 if l==1: 59 # delta= self.sigmoid_prime(a[-1])*(a[-1]-y) # O(k)=a[-1], t(k)=y 60 delta= a[-1]-y # cross-entropy 61 # delta=self.softmax(np.dot(w_, a[-2])+b_) -y # add for softmax 62 else: 63 #sp = self.sigmoid_prime(a[-l]) # O(j)=a[-l] 64 sp=self.relu_prime(a[-l]) 65 delta = np.dot(self.weights[-l+1].T, delta) * sp 66 b[-l] = delta 67 w[-l] = np.dot(delta, a[-l-1].T) 68 return (b, w) 69 70 def evaluate(self, test_data, train=0): 71 test_results = [(np.argmax(self.feedforward(x)), y) 72 for (x, y) in test_data] 73 if train: 74 return sum(int(x == np.argmax(y)) for (x, y) in test_results) 75 else: 76 return sum(int(x == y) for (x, y) in test_results) 77 78 def sigmoid(self,z): 79 return 1.0/(1.0+np.exp(-z)) 80 81 def sigmoid_prime(self,z): 82 return z*(1-z) 83 84 def softmax(self,a): 85 m = np.exp(a) 86 return m / np.sum(m) 87 88 def relu(self,z): 89 return np.maximum(z, 0.0) 90 91 def relu_prime(self,z): 92 z[z>0]=1 # <numpy.core._internal._ctypes> 93 return z 94 95 def get_label(i): 96 c=np.zeros((10,1)) 97 c[i]=1 98 return c 99 100 if __name__ == '__main__': 101 def get_data(data): 102 return [np.reshape(x, (784,1)) for x in data[0]] 103 104 f = open('mnist.pkl', 'rb') 105 training_data, validation_data, test_data = cPickle.load(f) 106 training_inputs = get_data(training_data) 107 training_label=[get_label(y_) for y_ in training_data[1]] 108 data = zip(training_inputs,training_label) 109 test_inputs = training_inputs = get_data(test_data) 110 test = zip(test_inputs,test_data[1]) 111 net = Network([784, 30, 10]) 112 net.SGD(data[:5000],test[:5000],epochs=30,mini_batch_size=10, eta=0.1, lambda_=1.0)