马上注册，结交更多好友，享用更多功能，让你轻松玩转社区。
您需要 登录 才可以下载或查看，没有账号？立即注册
x
引言
我们构建了一个可以执行前向传播、反向传播以及精度丈量等辅助任务的模型。通过编写相当多的代码并在一些较大的代码块中进行修改，我们实现了这些功能。此时，将模型自己转化为一个对象的做法开始显得更有意义，特殊是当我们盼望生存和加载这个对象以用于未来的预测任务时。别的，我们还可以利用这个对象减少一些常见代码行，使得与当前代码库的协作更加便捷，同时也更容易构建新的模型。为了完成模型对象的转换，我们将利用我们最近工作的模型，即利用正弦数据的回归模型：
- from nnfs.datasets import sine_data
- X, y = sine_data()
- # Model class
- class Model:
- def __init__(self):
- # Create a list of network objects
- self.layers = []
-
- # Add objects to the model
- def add(self, layer):
- self.layers.append(layer)
- # Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(1, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 1))
- model.add(Activation_Linear())
- >>>
- [<__main__.Layer_Dense object at 0x000001D1EB2A2900>,
- <__main__.Activation_ReLU object at 0x000001D1EB2A2180>,
- <__main__.Layer_Dense object at 0x000001D1EB2A3F20>,
- <__main__.Activation_ReLU object at 0x000001D1EB2B9220>,
- <__main__.Layer_Dense object at 0x000001D1EB2BB800>,
- <__main__.Activation_Linear object at 0x000001D1EB2BBA40>]
- # Set loss and optimizer
- def set(self, *, loss, optimizer):
- self.loss = loss
- self.optimizer = optimizer
现在，我们可以将一个调用此方法的语句添加到我们新创建的模型对象中，并传递loss和optimizer对象：
- # Create datasetX, y = sine_data()# Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(1, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 1))
- model.add(Activation_Linear())
- # Set loss and optimizer objectsmodel.set( loss=Loss_MeanSquaredError(), optimizer=Optimizer_Adam(learning_rate=0.005, decay=1e-3), )
- # Train the model
- def train(self, X, y, *, epochs=1, print_every=1):
- # Main training loop
- for epoch in range(1, epochs+1):
- # Temporary
- pass
- # Create datasetX, y = sine_data()# Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(1, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 1))
- model.add(Activation_Linear())
- # Set loss and optimizer objectsmodel.set( loss=Loss_MeanSquaredError(), optimizer=Optimizer_Adam(learning_rate=0.005, decay=1e-3), )model.train(X, y, epochs=10000, print_every=100)
- # Input "layer"
- class Layer_Input:
- # Forward pass
- def forward(self, inputs):
- self.output = inputs
- # Finalize the model
- def finalize(self):
- # Create and set the input layer
- self.input_layer = Layer_Input()
- # Count all the objects
- layer_count = len(self.layers)
- # Iterate the objects
- for i in range(layer_count):
- # If it's the first layer,
- # the previous layer object is the input layer
- if i == 0:
- self.layers[i].prev = self.input_layer
- self.layers[i].next = self.layers[i+1]
- # All layers except for the first and the last
- elif i < layer_count - 1:
- self.layers[i].prev = self.layers[i-1]
- self.layers[i].next = self.layers[i+1]
- # The last layer - the next object is the loss
- else:
- self.layers[i].prev = self.layers[i-1]
- self.layers[i].next = self.loss
现在，我们已经为模型对象执行前向传播所需的层信息准备就绪，让我们添加一个forward方法。我们将同时在练习时和之后仅进行预测（也称为模型推理）时利用这个forward方法。以下是在Model类中继续添加的代码：
- # Forward pass
- class Model:
- ...
- # Performs forward pass
- def forward(self, X):
- # Call forward method on the input layer
- # this will set the output property that
- # the first layer in "prev" object is expecting
- self.input_layer.forward(X)
- # Call forward method of every object in a chain
- # Pass output of the previous object as a parameter
- for layer in self.layers:
- layer.forward(layer.prev.output)
- # "layer" is now the last object from the list,
- # return its output
- return layer.output
这就是一次前向传播。现在，让我们将这个前向传播方法调用添加到 Model 类的 train 方法中：
- # Forward pass
- class Model:
- ...
- # Train the model
- def train(self, X, y, *, epochs=1, print_every=1):
- # Main training loop
- for epoch in range(1, epochs+1):
- # Perform the forward pass
- output = self.forward(X)
- # Temporary
- print(output)
- sys.exit()
- # Model class
- class Model:
- def __init__(self):
- # Create a list of network objects
- self.layers = []
-
- # Add objects to the model
- def add(self, layer):
- self.layers.append(layer)
- # Set loss and optimizer def set(self, *, loss, optimizer): self.loss = loss self.optimizer = optimizer # Train the model def train(self, X, y, *, epochs=1, print_every=1): # Main training loop for epoch in range(1, epochs+1): # Perform the forward pass output = self.forward(X) # Temporary print(output) sys.exit() # Finalize the model def finalize(self): # Create and set the input layer self.input_layer = Layer_Input() # Count all the objects layer_count = len(self.layers) # Iterate the objects for i in range(layer_count): # If it's the first layer, # the previous layer object is the input layer if i == 0: self.layers[i].prev = self.input_layer self.layers[i].next = self.layers[i+1] # All layers except for the first and the last elif i < layer_count - 1: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.layers[i+1] # The last layer - the next object is the loss else: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.loss # Performs forward pass def forward(self, X): # Call forward method on the input layer # this will set the output property that # the first layer in "prev" object is expecting self.input_layer.forward(X) # Call forward method of every object in a chain # Pass output of the previous object as a parameter for layer in self.layers: layer.forward(layer.prev.output) # "layer" is now the last object from the list, # return its output return layer.output
- # Create datasetX, y = sine_data()# Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(1, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 64))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 1))
- model.add(Activation_Linear())
- # Set loss and optimizer objectsmodel.set( loss=Loss_MeanSquaredError(), optimizer=Optimizer_Adam(learning_rate=0.005, decay=1e-3), )# Finalize the modelmodel.finalize()model.train(X, y, epochs=10000, print_every=100)
- >>>
- [[ 0.00000000e+00]
- [-1.13209149e-08]
- [-2.26418297e-08]
- ...
- [-1.12869511e-05]
- [-1.12982725e-05]
- [-1.13095930e-05]]
- # 如果层包含一个名为“weights”的属性，
- # 那么它是一个可训练层 -
- # 将其添加到可训练层列表中
- # 我们不需要检查偏置 -
- # 检查权重已经足够了
- if hasattr(self.layers[i], 'weights'):
- self.trainable_layers.append(self.layers[i])
- # Finalize the model
- def finalize(self):
- # Create and set the input layer
- self.input_layer = Layer_Input()
- # Count all the objects
- layer_count = len(self.layers)
- # Initialize a list containing trainable layers:
- self.trainable_layers = []
- # Iterate the objects
- for i in range(layer_count):
- # If it's the first layer,
- # the previous layer object is the input layer
- if i == 0:
- self.layers[i].prev = self.input_layer
- self.layers[i].next = self.layers[i+1]
- # All layers except for the first and the last
- elif i < layer_count - 1:
- self.layers[i].prev = self.layers[i-1]
- self.layers[i].next = self.layers[i+1]
- # The last layer - the next object is the loss
- # Also let's save aside the reference to the last object
- # whose output is the model's output
- else:
- self.layers[i].prev = self.layers[i-1]
- self.layers[i].next = self.loss
- self.output_layer_activation = self.layers[i]
-
- # 如果层包含一个名为“weights”的属性，
- # 那么它是一个可训练层 -
- # 将其添加到可训练层列表中
- # 我们不需要检查偏置 -
- # 检查权重已经足够了
- if hasattr(self.layers[i], 'weights'):
- self.trainable_layers.append(self.layers[i])
- # Common loss class
- class Loss:
- ...
- # Calculates the data and regularization losses
- # given model output and ground truth values
- def calculate(self, output, y):
- # Calculate sample losses
- sample_losses = self.forward(output, y)
- # Calculate mean loss
- data_loss = np.mean(sample_losses)
- # Return the data and regularization losses
- return data_loss, self.regularization_loss()
-
- # Set/remember trainable layers
- def remember_trainable_layers(self, trainable_layers):
- self.trainable_layers = trainable_layers
- # Common loss class
- class Loss:
- ...
- # Regularization loss calculation
- def regularization_loss(self):
- # 0 by default
- regularization_loss = 0
- # Calculate regularization loss
- # iterate all trainable layers
- for layer in self.trainable_layers:
- # L1 regularization - weights
- # calculate only when factor greater than 0
- if layer.weight_regularizer_l1 > 0:
- regularization_loss += layer.weight_regularizer_l1 * np.sum(np.abs(layer.weights))
- # L2 regularization - weights
- if layer.weight_regularizer_l2 > 0:
- regularization_loss += layer.weight_regularizer_l2 * np.sum(layer.weights * layer.weights)
- # L1 regularization - biases
- # calculate only when factor greater than 0
- if layer.bias_regularizer_l1 > 0:
- regularization_loss += layer.bias_regularizer_l1 * np.sum(np.abs(layer.biases))
- # L2 regularization - biases
- if layer.bias_regularizer_l2 > 0:
- regularization_loss += layer.bias_regularizer_l2 * np.sum(layer.biases * layer.biases)
- return regularization_loss
- # Softmax activation
- class Activation_Softmax:
- ...
- # Calculate predictions for outputs
- def predictions(self, outputs):
- return np.argmax(outputs, axis=1)
- # Sigmoid activation
- class Activation_Sigmoid:
- ...
- # Calculate predictions for outputs
- def predictions(self, outputs):
- return (outputs > 0.5) * 1
- # Linear activation
- class Activation_Linear:
- ...
- # Calculate predictions for outputs
- def predictions(self, outputs):
- return outputs
- # ReLU activation
- class Activation_ReLU:
- ...
- # Calculate predictions for outputs
- def predictions(self, outputs):
- return outputs
- # Model class
- class Model:
- ...
- # Finalize the model
- def finalize(self):
- ...
- # The last layer - the next object is the loss
- # Also let's save aside the reference to the last object
- # whose output is the model's output
- else:
- self.layers[i].prev = self.layers[i-1]
- self.layers[i].next = self.loss
- self.output_layer_activation = self.layers[i]
首先，我们会编写一个通用的 Accuracy 类，该类目前只包含一个方法 calculate，用于返回根据比较效果计算的准确率。我们已经在代码中添加了对 self.compare 方法的调用，但这个方法目前还不存在。我们将在继承自 Accuracy 类的其他类中创建该方法。现在只必要知道这个方法会返回一个由 True 和 False 值组成的列表，指示预测是否与真实值匹配。接下来，我们计算这些值的均匀值（True 被视为1，False 被视为0），并将其作为准确率返回。代码如下：
- # Common accuracy class
- class Accuracy:
- # Calculates an accuracy
- # given predictions and ground truth values
- def calculate(self, predictions, y):
- # Get comparison results
- comparisons = self.compare(predictions, y)
- # Calculate an accuracy
- accuracy = np.mean(comparisons)
- # Return accuracy
- return accuracy
对于回归模型，init 方法将计算准确率的精度（与我们之前为回归模型编写并在练习循环之前运行的内容雷同）。compare 方法将包含我们在练习循环中实际实现的比较代码，利用 self.precision。必要留意的是，初始化时不会重新计算精度，除非通过将 reinit 参数设置为 True 强制重新计算。这种筹划允许多种用例，包罗独立设置 self.precision、在必要时调用 init（例如，在模型创建过程中从外部调用），甚至多次调用 init（这将在后续某些情况下非常有效）：
- # Accuracy calculation for regression model
- class Accuracy_Regression(Accuracy):
- def __init__(self):
- # Create precision property
- self.precision = None
- # Calculates precision value
- # based on passed in ground truth
- def init(self, y, reinit=False):
- if self.precision is None or reinit:
- self.precision = np.std(y) / 250
- # Compares predictions to the ground truth values
- def compare(self, predictions, y):
- return np.absolute(predictions - y) < self.precision
- # Model class
- class Model:
- ...
- # Set loss, optimizer and accuracy
- def set(self, *, loss, optimizer, accuracy):
- self.loss = loss
- self.optimizer = optimizer
- self.accuracy = accuracy
- # Model class
- class Model:
- ...
- # Train the model
- def train(self, X, y, *, epochs=1, print_every=1):
- # Initialize accuracy object
- self.accuracy.init(y)
- # Main training loop
- for epoch in range(1, epochs+1):
- # Perform the forward pass
- output = self.forward(X)
- # Calculate loss
- data_loss, regularization_loss = self.loss.calculate(output, y)
- loss = data_loss + regularization_loss
- # Get predictions and calculate an accuracy
- predictions = self.output_layer_activation.predictions(output)
- accuracy = self.accuracy.calculate(predictions, y)
- # Model class
- class Model:
- def __init__(self):
- # Create a list of network objects
- self.layers = []
-
- # Add objects to the model
- def add(self, layer):
- self.layers.append(layer)
- # Set loss, optimizer and accuracy def set(self, *, loss, optimizer, accuracy): self.loss = loss self.optimizer = optimizer self.accuracy = accuracy # Finalize the model def finalize(self): # Create and set the input layer self.input_layer = Layer_Input() # Count all the objects layer_count = len(self.layers) # Initialize a list containing trainable layers: self.trainable_layers = [] # Iterate the objects for i in range(layer_count): # If it's the first layer, # the previous layer object is the input layer if i == 0: self.layers[i].prev = self.input_layer self.layers[i].next = self.layers[i+1] # All layers except for the first and the last elif i < layer_count - 1: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.layers[i+1] # The last layer - the next object is the loss # Also let's save aside the reference to the last object # whose output is the model's output else: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.loss self.output_layer_activation = self.layers[i] # 如果层包含一个名为“weights”的属性， # 那么它是一个可练习层 - # 将其添加到可练习层列表中 # 我们不必要查抄偏置 - # 查抄权重已经足够了 if hasattr(self.layers[i], 'weights'): self.trainable_layers.append(self.layers[i]) # Update loss object with trainable layers self.loss.remember_trainable_layers(self.trainable_layers) # Train the model def train(self, X, y, *, epochs=1, print_every=1): # Initialize accuracy object self.accuracy.init(y) # Main training loop for epoch in range(1, epochs+1): # Perform the forward pass output = self.forward(X) # Calculate loss data_loss, regularization_loss = self.loss.calculate(output, y) loss = data_loss + regularization_loss # Get predictions and calculate an accuracy predictions = self.output_layer_activation.predictions(output) accuracy = self.accuracy.calculate(predictions, y) # Performs forward pass def forward(self, X): # Call forward method on the input layer # this will set the output property that # the first layer in "prev" object is expecting self.input_layer.forward(X) # Call forward method of every object in a chain # Pass output of the previous object as a parameter for layer in self.layers: layer.forward(layer.prev.output) # "layer" is now the last object from the list, # return its output return layer.output
- # Common loss class
- class Loss:
- # Regularization loss calculation
- def regularization_loss(self):
- # 0 by default
- regularization_loss = 0
- # Calculate regularization loss
- # iterate all trainable layers
- for layer in self.trainable_layers:
- # L1 regularization - weights
- # calculate only when factor greater than 0
- if layer.weight_regularizer_l1 > 0:
- regularization_loss += layer.weight_regularizer_l1 * np.sum(np.abs(layer.weights))
- # L2 regularization - weights
- if layer.weight_regularizer_l2 > 0:
- regularization_loss += layer.weight_regularizer_l2 * np.sum(layer.weights * layer.weights)
- # L1 regularization - biases
- # calculate only when factor greater than 0
- if layer.bias_regularizer_l1 > 0:
- regularization_loss += layer.bias_regularizer_l1 * np.sum(np.abs(layer.biases))
- # L2 regularization - biases
- if layer.bias_regularizer_l2 > 0:
- regularization_loss += layer.bias_regularizer_l2 * np.sum(layer.biases * layer.biases)
- return regularization_loss
- # Set/remember trainable layers
- def remember_trainable_layers(self, trainable_layers):
- self.trainable_layers = trainable_layers
- # Calculates the data and regularization losses
- # given model output and ground truth values
- def calculate(self, output, y):
- # Calculate sample losses
- sample_losses = self.forward(output, y)
- # Calculate mean loss
- data_loss = np.mean(sample_losses)
- # Return the data and regularization losses
- return data_loss, self.regularization_loss()
- # Model class
- class Model:
- ...
- # Performs backward pass
- def backward(self, output, y):
- # First call backward method on the loss
- # this will set dinputs property that the last
- # layer will try to access shortly
- self.loss.backward(output, y)
- # Call backward method going through all the objects
- # in reversed order passing dinputs as a parameter
- for layer in reversed(self.layers):
- layer.backward(layer.next.dinputs)
- # Perform backward pass
- self.backward(output, y)
- # Optimize (update parameters)
- self.optimizer.pre_update_params()
- for layer in self.trainable_layers:
- self.optimizer.update_params(layer)
- self.optimizer.post_update_params()
- # Print a summary
- if not epoch % print_every:
- print(f'epoch: {epoch}, ' +
- f'acc: {accuracy:.3f}, ' +
- f'loss: {loss:.3f} (' +
- f'data_loss: {data_loss:.3f}, ' +
- f'reg_loss: {regularization_loss:.3f}), ' +
- f'lr: {self.optimizer.current_learning_rate}')
- # Model class
- class Model:
- ...
- # Train the model
- def train(self, X, y, *, epochs=1, print_every=1):
- # Initialize accuracy object
- self.accuracy.init(y)
- # Main training loop
- for epoch in range(1, epochs+1):
- # Perform the forward pass
- output = self.forward(X)
- # Calculate loss
- data_loss, regularization_loss = self.loss.calculate(output, y)
- loss = data_loss + regularization_loss
- # Get predictions and calculate an accuracy
- predictions = self.output_layer_activation.predictions(output)
- accuracy = self.accuracy.calculate(predictions, y)
- # Perform backward pass self.backward(output, y) # Optimize (update parameters) self.optimizer.pre_update_params() for layer in self.trainable_layers: self.optimizer.update_params(layer) self.optimizer.post_update_params() # Print a summary if not epoch % print_every: print(f'epoch: {epoch}, ' + f'acc: {accuracy:.3f}, ' + f'loss: {loss:.3f} (' + f'data_loss: {data_loss:.3f}, ' + f'reg_loss: {regularization_loss:.3f}), ' + f'lr: {self.optimizer.current_learning_rate}')
- >>>
- epoch: 100, acc: 0.006, loss: 0.085 (data_loss: 0.085, reg_loss: 0.000), lr: 0.004549590536851684
- epoch: 200, acc: 0.032, loss: 0.035 (data_loss: 0.035, reg_loss: 0.000), lr: 0.004170141784820684
- ...
- epoch: 9900, acc: 0.934, loss: 0.000 (data_loss: 0.000, reg_loss: 0.000), lr: 0.00045875768419121016
- epoch: 10000, acc: 0.970, loss: 0.000 (data_loss: 0.000, reg_loss: 0.000), lr: 0.00045458678061641964
- # Accuracy calculation for classification model
- class Accuracy_Categorical(Accuracy):
- # No initialization is needed
- def init(self, y):
- pass
- # Compares predictions to the ground truth values
- def compare(self, predictions, y):
- if len(y.shape) == 2:
- y = np.argmax(y, axis=1)
- return predictions == y
- # Common loss class
- class Loss:
- ...
- # Calculates the data and regularization losses
- # given model output and ground truth values
- def calculate(self, output, y, *, include_regularization=False):
- # Calculate sample losses
- sample_losses = self.forward(output, y)
- # Calculate mean loss
- data_loss = np.mean(sample_losses)
- # If just data loss - return it
- if not include_regularization:
- return data_loss
- # Return the data and regularization losses
- return data_loss, self.regularization_loss()
- # Calculate loss
- data_loss, regularization_loss = self.loss.calculate(output, y, include_regularization=True)
- # Model class
- class Model:
- ...
- # Train the model
- def train(self, X, y, *, epochs=1, print_every=1, validation_data=None):
- ...
- # If there is the validation data
- if validation_data is not None:
- # For better readability
- X_val, y_val = validation_data
- # Perform the forward pass
- output = self.forward(X_val)
- # Calculate the loss
- loss = self.loss.calculate(output, y_val)
- # Get predictions and calculate an accuracy
- predictions = self.output_layer_activation.predictions(output)
- accuracy = self.accuracy.calculate(predictions, y_val)
- # Print a summary
- print(f'validation, ' +
- f'acc: {accuracy:.3f}, ' +
- f'loss: {loss:.3f}')
- # Create train and test dataset
- X, y = spiral_data(samples=100, classes=2)
- X_test, y_test = spiral_data(samples=100, classes=2)
- # Reshape labels to be a list of lists
- # Inner list contains one output (either 0 or 1)
- # per each output neuron, 1 in this case
- y = y.reshape(-1, 1)
- y_test = y_test.reshape(-1, 1)
- # Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(2, 64, weight_regularizer_l2=5e-4, bias_regularizer_l2=5e-4))
- model.add(Activation_ReLU())
- model.add(Layer_Dense(64, 1))
- model.add(Activation_Sigmoid())
- # Set loss, optimizer and accuracy objects
- model.set(
- loss=Loss_BinaryCrossentropy(),
- optimizer=Optimizer_Adam(decay=5e-7),
- accuracy=Accuracy_Categorical()
- )
- # Finalize the model
- model.finalize()
- # Train the model
- model.train(X, y, validation_data=(X_test, y_test), epochs=10000, print_every=100)
- >>>
- epoch: 100, acc: 0.625, loss: 0.675 (data_loss: 0.674, reg_loss: 0.001), lr: 0.0009999505024501287
- epoch: 200, acc: 0.630, loss: 0.669 (data_loss: 0.668, reg_loss: 0.001), lr: 0.0009999005098992651
- ...
- epoch: 9900, acc: 0.905, loss: 0.312 (data_loss: 0.276, reg_loss: 0.037), lr: 0.0009950748768967994
- epoch: 10000, acc: 0.905, loss: 0.312 (data_loss: 0.275, reg_loss: 0.036), lr: 0.0009950253706593885
- validation, acc: 0.775, loss: 0.423
- # Forward pass
- def forward(self, inputs, training):
- # If not in the training mode - return values
- if not training:
- self.output = inputs.copy()
- return
- # Dropout
- class Layer_Dropout:
- ...
- # Forward pass
- def forward(self, inputs, training):
- # Save input values
- self.inputs = inputs
- # If not in the training mode - return values
- if not training:
- self.output = inputs.copy()
- return
- # Generate and save scaled mask
- self.binary_mask = np.random.binomial(1, self.rate, size=inputs.shape) / self.rate
- # Apply mask to output values
- self.output = inputs * self.binary_mask
- # Model class
- class Model:
- ...
- # Performs forward pass
- def forward(self, X, training):
- # Call forward method on the input layer
- # this will set the output property that
- # the first layer in "prev" object is expecting
- self.input_layer.forward(X, training)
- # Call forward method of every object in a chain
- # Pass output of the previous object as a parameter
- for layer in self.layers:
- layer.forward(layer.prev.output, training)
- # "layer" is now the last object from the list,
- # return its output
- return layer.output
- # Perform the forward pass
- output = self.forward(X, training=True)
- # Perform the forward pass
- output = self.forward(X_val, training=False)
- # Model class
- class Model:
- ...
- # Train the model
- def train(self, X, y, *, epochs=1, print_every=1, validation_data=None):
- # Initialize accuracy object
- self.accuracy.init(y)
- # Main training loop
- for epoch in range(1, epochs+1):
- # Perform the forward pass
- output = self.forward(X, training=True)
- # Calculate loss
- data_loss, regularization_loss = self.loss.calculate(output, y, include_regularization=True)
- loss = data_loss + regularization_loss
- # Get predictions and calculate an accuracy
- predictions = self.output_layer_activation.predictions(output)
- accuracy = self.accuracy.calculate(predictions, y)
- # Perform backward pass
- self.backward(output, y)
- # Optimize (update parameters)
- self.optimizer.pre_update_params()
- for layer in self.trainable_layers:
- self.optimizer.update_params(layer)
- self.optimizer.post_update_params()
- # Print a summary
- if not epoch % print_every:
- print(f'epoch: {epoch}, ' +
- f'acc: {accuracy:.3f}, ' +
- f'loss: {loss:.3f} (' +
- f'data_loss: {data_loss:.3f}, ' +
- f'reg_loss: {regularization_loss:.3f}), ' +
- f'lr: {self.optimizer.current_learning_rate}')
- # If there is the validation data
- if validation_data is not None:
- # For better readability
- X_val, y_val = validation_data
- # Perform the forward pass
- output = self.forward(X_val, training=False)
- # Calculate the loss
- loss = self.loss.calculate(output, y_val)
- # Get predictions and calculate an accuracy
- predictions = self.output_layer_activation.predictions(output)
- accuracy = self.accuracy.calculate(predictions, y_val)
- # Print a summary
- print(f'validation, ' +
- f'acc: {accuracy:.3f}, ' +
- f'loss: {loss:.3f}')
由于结合对象仅优化了后向传播的部分，我们决定让前向传播保持稳定，仍然利用单独的Softmax激活和Categorical Cross-Entropy丧失对象，只处置惩罚后向传播部分。
首先，我们必要主动确定当前模型是否是一个分类器，以及它是否利用了Softmax激活和Categorical Cross-Entropy丧失。这可以通过查抄末了一层对象的类名（这是一个激活函数对象）以及丧失函数对象的类名来实现。我们将在finalize方法的末尾添加此查抄：
- # If output activation is Softmax and
- # loss function is Categorical Cross-Entropy
- # create an object of combined activation
- # and loss function containing
- # faster gradient calculation
- if isinstance(self.layers[-1], Activation_Softmax) and isinstance(self.loss, Loss_CategoricalCrossentropy):
- # Create an object of combined activation
- # and loss functions
- self.softmax_classifier_output = Activation_Softmax_Loss_CategoricalCrossentropy()
我们还必要在Model类的构造函数中，将此属性初始化为None值：
- # Softmax classifier's output object
- self.softmax_classifier_output = None
首先，我们调用组合对象的backward方法；然后，由于我们不会调用激活函数对象（即层列表中的末了一个对象）的backward方法，因此必要用在激活/丧失对象中计算出的梯度来设置该对象的dinputs属性。末了，我们可以对除末了一层以外的所有层进行迭代并执行它们的反向传播操作：
- # If softmax classifier
- if self.softmax_classifier_output is not None:
- # First call backward method
- # on the combined activation/loss
- # this will set dinputs property
- self.softmax_classifier_output.backward(output, y)
- # Since we'll not call backward method of the last layer
- # which is Softmax activation
- # as we used combined activation/loss
- # object, let's set dinputs in this object
- self.layers[-1].dinputs = self.softmax_classifier_output.dinputs
- # Call backward method going through
- # all the objects but last
- # in reversed order passing dinputs as a parameter
- for layer in reversed(self.layers[:-1]):
- layer.backward(layer.next.dinputs)
- return
- # Model classclass Model: def __init__(self): # Create a list of network objects self.layers = [] # Softmax classifier's output object
- self.softmax_classifier_output = None
- # Add objects to the model def add(self, layer): self.layers.append(layer) # Set loss, optimizer and accuracy def set(self, *, loss, optimizer, accuracy): self.loss = loss self.optimizer = optimizer self.accuracy = accuracy # Finalize the model def finalize(self): # Create and set the input layer self.input_layer = Layer_Input() # Count all the objects layer_count = len(self.layers) # Initialize a list containing trainable layers: self.trainable_layers = [] # Iterate the objects for i in range(layer_count): # If it's the first layer, # the previous layer object is the input layer if i == 0: self.layers[i].prev = self.input_layer self.layers[i].next = self.layers[i+1] # All layers except for the first and the last elif i < layer_count - 1: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.layers[i+1] # The last layer - the next object is the loss # Also let's save aside the reference to the last object # whose output is the model's output else: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.loss self.output_layer_activation = self.layers[i] # If layer contains an attribute called "weights", # it's a trainable layer - # add it to the list of trainable layers # We don't need to check for biases - # checking for weights is enough if hasattr(self.layers[i], 'weights'): self.trainable_layers.append(self.layers[i]) # Update loss object with trainable layers self.loss.remember_trainable_layers(self.trainable_layers) # If output activation is Softmax and
- # loss function is Categorical Cross-Entropy
- # create an object of combined activation
- # and loss function containing
- # faster gradient calculation
- if isinstance(self.layers[-1], Activation_Softmax) and isinstance(self.loss, Loss_CategoricalCrossentropy):
- # Create an object of combined activation
- # and loss functions
- self.softmax_classifier_output = Activation_Softmax_Loss_CategoricalCrossentropy()
- # Train the model def train(self, X, y, *, epochs=1, print_every=1, validation_data=None): # Initialize accuracy object self.accuracy.init(y) # Main training loop for epoch in range(1, epochs+1): # Perform the forward pass output = self.forward(X, training=True) # Calculate loss data_loss, regularization_loss = self.loss.calculate(output, y, include_regularization=True) loss = data_loss + regularization_loss # Get predictions and calculate an accuracy predictions = self.output_layer_activation.predictions(output) accuracy = self.accuracy.calculate(predictions, y) # Perform backward pass self.backward(output, y) # Optimize (update parameters) self.optimizer.pre_update_params() for layer in self.trainable_layers: self.optimizer.update_params(layer) self.optimizer.post_update_params() # Print a summary if not epoch % print_every: print(f'epoch: {epoch}, ' + f'acc: {accuracy:.3f}, ' + f'loss: {loss:.3f} (' + f'data_loss: {data_loss:.3f}, ' + f'reg_loss: {regularization_loss:.3f}), ' + f'lr: {self.optimizer.current_learning_rate}') # If there is the validation data if validation_data is not None: # For better readability X_val, y_val = validation_data # Perform the forward pass output = self.forward(X_val, training=False) # Calculate the loss loss = self.loss.calculate(output, y_val) # Get predictions and calculate an accuracy predictions = self.output_layer_activation.predictions(output) accuracy = self.accuracy.calculate(predictions, y_val) # Print a summary print(f'validation, ' + f'acc: {accuracy:.3f}, ' + f'loss: {loss:.3f}') # Performs forward pass def forward(self, X, training): # Call forward method on the input layer # this will set the output property that # the first layer in "prev" object is expecting self.input_layer.forward(X, training) # Call forward method of every object in a chain # Pass output of the previous object as a parameter for layer in self.layers: layer.forward(layer.prev.output, training) # "layer" is now the last object from the list, # return its output return layer.output # Performs backward pass def backward(self, output, y): # If softmax classifier if self.softmax_classifier_output is not None: # First call backward method # on the combined activation/loss # this will set dinputs property self.softmax_classifier_output.backward(output, y) # Since we'll not call backward method of the last layer # which is Softmax activation # as we used combined activation/loss # object, let's set dinputs in this object self.layers[-1].dinputs = self.softmax_classifier_output.dinputs # Call backward method going through # all the objects but last # in reversed order passing dinputs as a parameter for layer in reversed(self.layers[:-1]): layer.backward(layer.next.dinputs) return # First call backward method on the loss # this will set dinputs property that the last # layer will try to access shortly self.loss.backward(output, y) # Call backward method going through all the objects # in reversed order passing dinputs as a parameter for layer in reversed(self.layers): layer.backward(layer.next.dinputs)
- # Softmax classifier - combined Softmax activation
- # and cross-entropy loss for faster backward step
- class Activation_Softmax_Loss_CategoricalCrossentropy():
- ...
- # Backward pass
- def backward(self, dvalues, y_true):
- # Number of samples
- samples = len(dvalues)
- # Copy so we can safely modify
- self.dinputs = dvalues.copy()
- # Calculate gradient
- self.dinputs[range(samples), y_true] -= 1
- # Normalize gradient
- self.dinputs = self.dinputs / samples
- # Create dataset
- X, y = spiral_data(samples=1000, classes=3)
- X_test, y_test = spiral_data(samples=100, classes=3)
- # Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(2, 512, weight_regularizer_l2=5e-4, bias_regularizer_l2=5e-4))
- model.add(Activation_ReLU())
- model.add(Layer_Dropout(0.1))
- model.add(Layer_Dense(512, 3))
- model.add(Activation_Softmax())
- # Set loss, optimizer and accuracy objects
- model.set(
- loss=Loss_CategoricalCrossentropy(),
- optimizer=Optimizer_Adam(learning_rate=0.05, decay=5e-5),
- accuracy=Accuracy_Categorical()
- )
- # Finalize the model
- model.finalize()
- # Train the model
- model.train(X, y, validation_data=(X_test, y_test), epochs=10000, print_every=100)
- >>>
- epoch: 100, acc: 0.716, loss: 0.726 (data_loss: 0.666, reg_loss: 0.060), lr:
- 0.04975371909050202
- epoch: 200, acc: 0.787, loss: 0.615 (data_loss: 0.538, reg_loss: 0.077), lr:
- 0.049507401356502806
- ...
- epoch: 9900, acc: 0.861, loss: 0.436 (data_loss: 0.389, reg_loss: 0.046),
- lr: 0.0334459346466437
- epoch: 10000, acc: 0.880, loss: 0.394 (data_loss: 0.347, reg_loss: 0.047),
- lr: 0.03333444448148271
- validation, acc: 0.867, loss: 0.379
到目前为止的完整代码：
- import numpy as npimport nnfsfrom nnfs.datasets import sine_data, spiral_dataimport sysnnfs.init()# Dense layerclass Layer_Dense: # Layer initialization def __init__(self, n_inputs, n_neurons, weight_regularizer_l1=0, weight_regularizer_l2=0, bias_regularizer_l1=0, bias_regularizer_l2=0): # Initialize weights and biases # self.weights = 0.01 * np.random.randn(n_inputs, n_neurons) self.weights = 0.1 * np.random.randn(n_inputs, n_neurons) self.biases = np.zeros((1, n_neurons)) # Set regularization strength self.weight_regularizer_l1 = weight_regularizer_l1 self.weight_regularizer_l2 = weight_regularizer_l2 self.bias_regularizer_l1 = bias_regularizer_l1 self.bias_regularizer_l2 = bias_regularizer_l2 # Forward pass def forward(self, inputs, training): # Remember input values self.inputs = inputs # Calculate output values from inputs, weights and biases self.output = np.dot(inputs, self.weights) + self.biases # Backward pass def backward(self, dvalues): # Gradients on parameters self.dweights = np.dot(self.inputs.T, dvalues) self.dbiases = np.sum(dvalues, axis=0, keepdims=True) # Gradients on regularization # L1 on weights if self.weight_regularizer_l1 > 0: dL1 = np.ones_like(self.weights) dL1[self.weights < 0] = -1 self.dweights += self.weight_regularizer_l1 * dL1 # L2 on weights if self.weight_regularizer_l2 > 0: self.dweights += 2 * self.weight_regularizer_l2 * self.weights # L1 on biases if self.bias_regularizer_l1 > 0: dL1 = np.ones_like(self.biases) dL1[self.biases < 0] = -1 self.dbiases += self.bias_regularizer_l1 * dL1 # L2 on biases if self.bias_regularizer_l2 > 0: self.dbiases += 2 * self.bias_regularizer_l2 * self.biases # Gradient on values self.dinputs = np.dot(dvalues, self.weights.T) # Dropoutclass Layer_Dropout: # Init def __init__(self, rate): # Store rate, we invert it as for example for dropout # of 0.1 we need success rate of 0.9 self.rate = 1 - rate # Forward pass def forward(self, inputs, training): # Save input values self.inputs = inputs # If not in the training mode - return values if not training: self.output = inputs.copy() return # Generate and save scaled mask self.binary_mask = np.random.binomial(1, self.rate, size=inputs.shape) / self.rate # Apply mask to output values self.output = inputs * self.binary_mask # Backward pass def backward(self, dvalues): # Gradient on values self.dinputs = dvalues * self.binary_mask # Input "layer"class Layer_Input: # Forward pass def forward(self, inputs, training): self.output = inputs # ReLU activationclass Activation_ReLU: # Forward pass def forward(self, inputs, training): # Remember input values self.inputs = inputs # Calculate output values from inputs self.output = np.maximum(0, inputs) # Backward pass def backward(self, dvalues): # Since we need to modify original variable, # let's make a copy of values first self.dinputs = dvalues.copy() # Zero gradient where input values were negative self.dinputs[self.inputs <= 0] = 0 # Calculate predictions for outputs def predictions(self, outputs): return outputs # Softmax activationclass Activation_Softmax: # Forward pass def forward(self, inputs, training): # Remember input values self.inputs = inputs # Get unnormalized probabilities exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True)) # Normalize them for each sample probabilities = exp_values / np.sum(exp_values, axis=1, keepdims=True) self.output = probabilities # Backward pass def backward(self, dvalues): # Create uninitialized array self.dinputs = np.empty_like(dvalues) # Enumerate outputs and gradients for index, (single_output, single_dvalues) in enumerate(zip(self.output, dvalues)): # Flatten output array single_output = single_output.reshape(-1, 1) # Calculate Jacobian matrix of the output and jacobian_matrix = np.diagflat(single_output) - np.dot(single_output, single_output.T) # Calculate sample-wise gradient # and add it to the array of sample gradients self.dinputs[index] = np.dot(jacobian_matrix, single_dvalues) # Calculate predictions for outputs def predictions(self, outputs): return np.argmax(outputs, axis=1) # Sigmoid activationclass Activation_Sigmoid: # Forward pass def forward(self, inputs, training): # Save input and calculate/save output # of the sigmoid function self.inputs = inputs self.output = 1 / (1 + np.exp(-inputs)) # Backward pass def backward(self, dvalues): # Derivative - calculates from output of the sigmoid function self.dinputs = dvalues * (1 - self.output) * self.output # Calculate predictions for outputs def predictions(self, outputs): return (outputs > 0.5) * 1 # Linear activationclass Activation_Linear: # Forward pass def forward(self, inputs, training): # Just remember values self.inputs = inputs self.output = inputs # Backward pass def backward(self, dvalues): # derivative is 1, 1 * dvalues = dvalues - the chain rule self.dinputs = dvalues.copy() # Calculate predictions for outputs def predictions(self, outputs): return outputs # SGD optimizerclass Optimizer_SGD: # Initialize optimizer - set settings, # learning rate of 1. is default for this optimizer def __init__(self, learning_rate=1., decay=0., momentum=0.): self.learning_rate = learning_rate self.current_learning_rate = learning_rate self.decay = decay self.iterations = 0 self.momentum = momentum # Call once before any parameter updates def pre_update_params(self): if self.decay: self.current_learning_rate = self.learning_rate * (1. / (1. + self.decay * self.iterations)) # Update parameters def update_params(self, layer): # If we use momentum if self.momentum: # If layer does not contain momentum arrays, create them # filled with zeros if not hasattr(layer, 'weight_momentums'): layer.weight_momentums = np.zeros_like(layer.weights) # If there is no momentum array for weights # The array doesn't exist for biases yet either. layer.bias_momentums = np.zeros_like(layer.biases) # Build weight updates with momentum - take previous # updates multiplied by retain factor and update with # current gradients weight_updates = self.momentum * layer.weight_momentums - self.current_learning_rate * layer.dweights layer.weight_momentums = weight_updates # Build bias updates bias_updates = self.momentum * layer.bias_momentums - self.current_learning_rate * layer.dbiases layer.bias_momentums = bias_updates # Vanilla SGD updates (as before momentum update) else: weight_updates = -self.current_learning_rate * layer.dweights bias_updates = -self.current_learning_rate * layer.dbiases # Update weights and biases using either # vanilla or momentum updates layer.weights += weight_updates layer.biases += bias_updates # Call once after any parameter updates def post_update_params(self): self.iterations += 1 # Adagrad optimizerclass Optimizer_Adagrad: # Initialize optimizer - set settings def __init__(self, learning_rate=1., decay=0., epsilon=1e-7): self.learning_rate = learning_rate self.current_learning_rate = learning_rate self.decay = decay self.iterations = 0 self.epsilon = epsilon # Call once before any parameter updates def pre_update_params(self): if self.decay: self.current_learning_rate = self.learning_rate * (1. / (1. + self.decay * self.iterations)) # Update parameters def update_params(self, layer): # If layer does not contain cache arrays, # create them filled with zeros if not hasattr(layer, 'weight_cache'): layer.weight_cache = np.zeros_like(layer.weights) layer.bias_cache = np.zeros_like(layer.biases) # Update cache with squared current gradients layer.weight_cache += layer.dweights**2 layer.bias_cache += layer.dbiases**2 # Vanilla SGD parameter update + normalization # with square rooted cache layer.weights += -self.current_learning_rate * layer.dweights / (np.sqrt(layer.weight_cache) + self.epsilon) layer.biases += -self.current_learning_rate * layer.dbiases / (np.sqrt(layer.bias_cache) + self.epsilon) # Call once after any parameter updates def post_update_params(self): self.iterations += 1 # RMSprop optimizerclass Optimizer_RMSprop: # Initialize optimizer - set settings def __init__(self, learning_rate=0.001, decay=0., epsilon=1e-7, rho=0.9): self.learning_rate = learning_rate self.current_learning_rate = learning_rate self.decay = decay self.iterations = 0 self.epsilon = epsilon self.rho = rho # Call once before any parameter updates def pre_update_params(self): if self.decay: self.current_learning_rate = self.learning_rate * (1. / (1. + self.decay * self.iterations)) # Update parameters def update_params(self, layer): # If layer does not contain cache arrays, # create them filled with zeros if not hasattr(layer, 'weight_cache'): layer.weight_cache = np.zeros_like(layer.weights) layer.bias_cache = np.zeros_like(layer.biases) # Update cache with squared current gradients layer.weight_cache = self.rho * layer.weight_cache + (1 - self.rho) * layer.dweights**2 layer.bias_cache = self.rho * layer.bias_cache + (1 - self.rho) * layer.dbiases**2 # Vanilla SGD parameter update + normalization # with square rooted cache layer.weights += -self.current_learning_rate * layer.dweights / (np.sqrt(layer.weight_cache) + self.epsilon) layer.biases += -self.current_learning_rate * layer.dbiases / (np.sqrt(layer.bias_cache) + self.epsilon) # Call once after any parameter updates def post_update_params(self): self.iterations += 1 # Adam optimizerclass Optimizer_Adam: # Initialize optimizer - set settings def __init__(self, learning_rate=0.001, decay=0., epsilon=1e-7, beta_1=0.9, beta_2=0.999): self.learning_rate = learning_rate self.current_learning_rate = learning_rate self.decay = decay self.iterations = 0 self.epsilon = epsilon self.beta_1 = beta_1 self.beta_2 = beta_2 # Call once before any parameter updates def pre_update_params(self): if self.decay: self.current_learning_rate = self.learning_rate * (1. / (1. + self.decay * self.iterations)) # Update parameters def update_params(self, layer): # If layer does not contain cache arrays, # create them filled with zeros if not hasattr(layer, 'weight_cache'): layer.weight_momentums = np.zeros_like(layer.weights) layer.weight_cache = np.zeros_like(layer.weights) layer.bias_momentums = np.zeros_like(layer.biases) layer.bias_cache = np.zeros_like(layer.biases) # Update momentum with current gradients layer.weight_momentums = self.beta_1 * layer.weight_momentums + (1 - self.beta_1) * layer.dweights layer.bias_momentums = self.beta_1 * layer.bias_momentums + (1 - self.beta_1) * layer.dbiases # Get corrected momentum # self.iteration is 0 at first pass # and we need to start with 1 here weight_momentums_corrected = layer.weight_momentums / (1 - self.beta_1 ** (self.iterations + 1)) bias_momentums_corrected = layer.bias_momentums / (1 - self.beta_1 ** (self.iterations + 1)) # Update cache with squared current gradients layer.weight_cache = self.beta_2 * layer.weight_cache + (1 - self.beta_2) * layer.dweights**2 layer.bias_cache = self.beta_2 * layer.bias_cache + (1 - self.beta_2) * layer.dbiases**2 # Get corrected cache weight_cache_corrected = layer.weight_cache / (1 - self.beta_2 ** (self.iterations + 1)) bias_cache_corrected = layer.bias_cache / (1 - self.beta_2 ** (self.iterations + 1)) # Vanilla SGD parameter update + normalization # with square rooted cache layer.weights += -self.current_learning_rate * weight_momentums_corrected / (np.sqrt(weight_cache_corrected) + self.epsilon) layer.biases += -self.current_learning_rate * bias_momentums_corrected / (np.sqrt(bias_cache_corrected) + self.epsilon) # Call once after any parameter updates def post_update_params(self): self.iterations += 1 # Common loss classclass Loss: # Regularization loss calculation def regularization_loss(self): # 0 by default regularization_loss = 0 # Calculate regularization loss # iterate all trainable layers for layer in self.trainable_layers: # L1 regularization - weights # calculate only when factor greater than 0 if layer.weight_regularizer_l1 > 0: regularization_loss += layer.weight_regularizer_l1 * np.sum(np.abs(layer.weights)) # L2 regularization - weights if layer.weight_regularizer_l2 > 0: regularization_loss += layer.weight_regularizer_l2 * np.sum(layer.weights * layer.weights) # L1 regularization - biases # calculate only when factor greater than 0 if layer.bias_regularizer_l1 > 0: regularization_loss += layer.bias_regularizer_l1 * np.sum(np.abs(layer.biases)) # L2 regularization - biases if layer.bias_regularizer_l2 > 0: regularization_loss += layer.bias_regularizer_l2 * np.sum(layer.biases * layer.biases) return regularization_loss # Set/remember trainable layers def remember_trainable_layers(self, trainable_layers): self.trainable_layers = trainable_layers # Calculates the data and regularization losses # given model output and ground truth values def calculate(self, output, y, *, include_regularization=False): # Calculate sample losses sample_losses = self.forward(output, y) # Calculate mean loss data_loss = np.mean(sample_losses) # If just data loss - return it if not include_regularization: return data_loss # Return the data and regularization losses return data_loss, self.regularization_loss() # Cross-entropy lossclass Loss_CategoricalCrossentropy(Loss): # Forward pass def forward(self, y_pred, y_true): # Number of samples in a batch samples = len(y_pred) # Clip data to prevent division by 0 # Clip both sides to not drag mean towards any value y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7) # Probabilities for target values - # only if categorical labels if len(y_true.shape) == 1: correct_confidences = y_pred_clipped[ range(samples), y_true ] # Mask values - only for one-hot encoded labels elif len(y_true.shape) == 2: correct_confidences = np.sum(y_pred_clipped * y_true, axis=1) # Losses negative_log_likelihoods = -np.log(correct_confidences) return negative_log_likelihoods # Backward pass def backward(self, dvalues, y_true): # Number of samples samples = len(dvalues) # Number of labels in every sample # We'll use the first sample to count them labels = len(dvalues[0]) # If labels are sparse, turn them into one-hot vector if len(y_true.shape) == 1: y_true = np.eye(labels)[y_true] # Calculate gradient self.dinputs = -y_true / dvalues # Normalize gradient self.dinputs = self.dinputs / samples # Softmax classifier - combined Softmax activation# and cross-entropy loss for faster backward stepclass Activation_Softmax_Loss_CategoricalCrossentropy(): # # Creates activation and loss function objects # def __init__(self): # self.activation = Activation_Softmax() # self.loss = Loss_CategoricalCrossentropy() # # Forward pass # def forward(self, inputs, y_true): # # Output layer's activation function # self.activation.forward(inputs) # # Set the output # self.output = self.activation.output # # Calculate and return loss value # return self.loss.calculate(self.output, y_true) # Backward pass def backward(self, dvalues, y_true): # Number of samples samples = len(dvalues) # If labels are one-hot encoded, # turn them into discrete values if len(y_true.shape) == 2: y_true = np.argmax(y_true, axis=1) # Copy so we can safely modify self.dinputs = dvalues.copy() # Calculate gradient self.dinputs[range(samples), y_true] -= 1 # Normalize gradient self.dinputs = self.dinputs / samples # Binary cross-entropy lossclass Loss_BinaryCrossentropy(Loss): # Forward pass def forward(self, y_pred, y_true): # Clip data to prevent division by 0 # Clip both sides to not drag mean towards any value y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7) # Calculate sample-wise loss sample_losses = -(y_true * np.log(y_pred_clipped) + (1 - y_true) * np.log(1 - y_pred_clipped)) sample_losses = np.mean(sample_losses, axis=-1) # Return losses return sample_losses # Backward pass def backward(self, dvalues, y_true): # Number of samples samples = len(dvalues) # Number of outputs in every sample # We'll use the first sample to count them outputs = len(dvalues[0]) # Clip data to prevent division by 0 # Clip both sides to not drag mean towards any value clipped_dvalues = np.clip(dvalues, 1e-7, 1 - 1e-7) # Calculate gradient self.dinputs = -(y_true / clipped_dvalues - (1 - y_true) / (1 - clipped_dvalues)) / outputs # Normalize gradient self.dinputs = self.dinputs / samples # Mean Squared Error lossclass Loss_MeanSquaredError(Loss): # L2 loss # Forward pass def forward(self, y_pred, y_true): # Calculate loss sample_losses = np.mean((y_true - y_pred)**2, axis=-1) # Return losses return sample_losses # Backward pass def backward(self, dvalues, y_true): # Number of samples samples = len(dvalues) # Number of outputs in every sample # We'll use the first sample to count them outputs = len(dvalues[0]) # Gradient on values self.dinputs = -2 * (y_true - dvalues) / outputs # Normalize gradient self.dinputs = self.dinputs / samples # Mean Absolute Error lossclass Loss_MeanAbsoluteError(Loss): # L1 loss def forward(self, y_pred, y_true): # Calculate loss sample_losses = np.mean(np.abs(y_true - y_pred), axis=-1) # Return losses return sample_losses # Backward pass def backward(self, dvalues, y_true): # Number of samples samples = len(dvalues) # Number of outputs in every sample # We'll use the first sample to count them outputs = len(dvalues[0]) # Calculate gradient self.dinputs = np.sign(y_true - dvalues) / outputs # Normalize gradient self.dinputs = self.dinputs / samples # Common accuracy class
- class Accuracy:
- # Calculates an accuracy
- # given predictions and ground truth values
- def calculate(self, predictions, y):
- # Get comparison results
- comparisons = self.compare(predictions, y)
- # Calculate an accuracy
- accuracy = np.mean(comparisons)
- # Return accuracy
- return accuracy
- # Accuracy calculation for classification model
- class Accuracy_Categorical(Accuracy):
- # No initialization is needed
- def init(self, y):
- pass
- # Compares predictions to the ground truth values
- def compare(self, predictions, y):
- if len(y.shape) == 2:
- y = np.argmax(y, axis=1)
- return predictions == y
- # Accuracy calculation for regression model
- class Accuracy_Regression(Accuracy):
- def __init__(self):
- # Create precision property
- self.precision = None
- # Calculates precision value
- # based on passed in ground truth
- def init(self, y, reinit=False):
- if self.precision is None or reinit:
- self.precision = np.std(y) / 250
- # Compares predictions to the ground truth values
- def compare(self, predictions, y):
- return np.absolute(predictions - y) < self.precision
- # Model classclass Model: def __init__(self): # Create a list of network objects self.layers = [] # Softmax classifier's output object
- self.softmax_classifier_output = None
- # Add objects to the model def add(self, layer): self.layers.append(layer) # Set loss, optimizer and accuracy def set(self, *, loss, optimizer, accuracy): self.loss = loss self.optimizer = optimizer self.accuracy = accuracy # Finalize the model def finalize(self): # Create and set the input layer self.input_layer = Layer_Input() # Count all the objects layer_count = len(self.layers) # Initialize a list containing trainable layers: self.trainable_layers = [] # Iterate the objects for i in range(layer_count): # If it's the first layer, # the previous layer object is the input layer if i == 0: self.layers[i].prev = self.input_layer self.layers[i].next = self.layers[i+1] # All layers except for the first and the last elif i < layer_count - 1: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.layers[i+1] # The last layer - the next object is the loss # Also let's save aside the reference to the last object # whose output is the model's output else: self.layers[i].prev = self.layers[i-1] self.layers[i].next = self.loss self.output_layer_activation = self.layers[i] # If layer contains an attribute called "weights", # it's a trainable layer - # add it to the list of trainable layers # We don't need to check for biases - # checking for weights is enough if hasattr(self.layers[i], 'weights'): self.trainable_layers.append(self.layers[i]) # Update loss object with trainable layers self.loss.remember_trainable_layers(self.trainable_layers) # If output activation is Softmax and
- # loss function is Categorical Cross-Entropy
- # create an object of combined activation
- # and loss function containing
- # faster gradient calculation
- if isinstance(self.layers[-1], Activation_Softmax) and isinstance(self.loss, Loss_CategoricalCrossentropy):
- # Create an object of combined activation
- # and loss functions
- self.softmax_classifier_output = Activation_Softmax_Loss_CategoricalCrossentropy()
- # Train the model def train(self, X, y, *, epochs=1, print_every=1, validation_data=None): # Initialize accuracy object self.accuracy.init(y) # Main training loop for epoch in range(1, epochs+1): # Perform the forward pass output = self.forward(X, training=True) # Calculate loss data_loss, regularization_loss = self.loss.calculate(output, y, include_regularization=True) loss = data_loss + regularization_loss # Get predictions and calculate an accuracy predictions = self.output_layer_activation.predictions(output) accuracy = self.accuracy.calculate(predictions, y) # Perform backward pass self.backward(output, y) # Optimize (update parameters) self.optimizer.pre_update_params() for layer in self.trainable_layers: self.optimizer.update_params(layer) self.optimizer.post_update_params() # Print a summary if not epoch % print_every: print(f'epoch: {epoch}, ' + f'acc: {accuracy:.3f}, ' + f'loss: {loss:.3f} (' + f'data_loss: {data_loss:.3f}, ' + f'reg_loss: {regularization_loss:.3f}), ' + f'lr: {self.optimizer.current_learning_rate}') # If there is the validation data if validation_data is not None: # For better readability X_val, y_val = validation_data # Perform the forward pass output = self.forward(X_val, training=False) # Calculate the loss loss = self.loss.calculate(output, y_val) # Get predictions and calculate an accuracy predictions = self.output_layer_activation.predictions(output) accuracy = self.accuracy.calculate(predictions, y_val) # Print a summary print(f'validation, ' + f'acc: {accuracy:.3f}, ' + f'loss: {loss:.3f}') # Performs forward pass def forward(self, X, training): # Call forward method on the input layer # this will set the output property that # the first layer in "prev" object is expecting self.input_layer.forward(X, training) # Call forward method of every object in a chain # Pass output of the previous object as a parameter for layer in self.layers: layer.forward(layer.prev.output, training) # "layer" is now the last object from the list, # return its output return layer.output # Performs backward pass def backward(self, output, y): # If softmax classifier if self.softmax_classifier_output is not None: # First call backward method # on the combined activation/loss # this will set dinputs property self.softmax_classifier_output.backward(output, y) # Since we'll not call backward method of the last layer # which is Softmax activation # as we used combined activation/loss # object, let's set dinputs in this object self.layers[-1].dinputs = self.softmax_classifier_output.dinputs # Call backward method going through # all the objects but last # in reversed order passing dinputs as a parameter for layer in reversed(self.layers[:-1]): layer.backward(layer.next.dinputs) return # First call backward method on the loss # this will set dinputs property that the last # layer will try to access shortly self.loss.backward(output, y) # Call backward method going through all the objects # in reversed order passing dinputs as a parameter for layer in reversed(self.layers): layer.backward(layer.next.dinputs) # # Create dataset# X, y = sine_data()# # Instantiate the model# model = Model()# # Add layers# model.add(Layer_Dense(1, 64))# model.add(Activation_ReLU())# model.add(Layer_Dense(64, 64))# model.add(Activation_ReLU())# model.add(Layer_Dense(64, 1))# model.add(Activation_Linear())# # Set loss and optimizer objects# model.set(# loss=Loss_MeanSquaredError(),# optimizer=Optimizer_Adam(learning_rate=0.005, decay=1e-3),# accuracy=Accuracy_Regression()# )# # Finalize the model# model.finalize()# model.train(X, y, epochs=10000, print_every=100)########################################################################################## # Create train and test dataset# X, y = spiral_data(samples=100, classes=2)# X_test, y_test = spiral_data(samples=100, classes=2)# # Reshape labels to be a list of lists# # Inner list contains one output (either 0 or 1)# # per each output neuron, 1 in this case# y = y.reshape(-1, 1)# y_test = y_test.reshape(-1, 1)# # Instantiate the model# model = Model()# # Add layers# model.add(Layer_Dense(2, 64, weight_regularizer_l2=5e-4, bias_regularizer_l2=5e-4))# model.add(Activation_ReLU())# model.add(Layer_Dense(64, 1))# model.add(Activation_Sigmoid())# # Set loss, optimizer and accuracy objects# model.set(# loss=Loss_BinaryCrossentropy(),# optimizer=Optimizer_Adam(decay=5e-7),# accuracy=Accuracy_Categorical()# )# # Finalize the model# model.finalize()# # Train the model# model.train(X, y, validation_data=(X_test, y_test), epochs=10000, print_every=100)########################################################################################## Create dataset
- X, y = spiral_data(samples=1000, classes=3)
- X_test, y_test = spiral_data(samples=100, classes=3)
- # Instantiate the model
- model = Model()
- # Add layers
- model.add(Layer_Dense(2, 512, weight_regularizer_l2=5e-4, bias_regularizer_l2=5e-4))
- model.add(Activation_ReLU())
- model.add(Layer_Dropout(0.1))
- model.add(Layer_Dense(512, 3))
- model.add(Activation_Softmax())
- # Set loss, optimizer and accuracy objects
- model.set(
- loss=Loss_CategoricalCrossentropy(),
- optimizer=Optimizer_Adam(learning_rate=0.05, decay=5e-5),
- accuracy=Accuracy_Categorical()
- )
- # Finalize the model
- model.finalize()
- # Train the model
- model.train(X, y, validation_data=(X_test, y_test), epochs=10000, print_every=100)
本章的章节代码、更多资源和勘误表：https://nnfs.io/ch18
免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。 |
