动手撸神经网络的代码，是大家常常遇到的问题。在设计自己的网络时，需要考虑网络大小，隐藏层层数，激活函数和参数初始化方法。最笨拙的方法就是固定下来，发生变化就要手动调整一次。这里介绍一种可以自动生生网络的方法，每次改变只需要调整一些参数，网络就会自动改变，大大提升了生产代码的效率。
参考链接：
1、Pytorch之搭建神经网络的四种方法
2、Pytorch–1.使用Pytorch搭建一个简易的神经网络
3、十分钟掌握Pytorch搭建神经网络的流程
4、使用pytorch搭建神经网络
5、PyTorch使用教程-PyTorch构建神经网络(下)
6、Pytorch 默认参数初始化
7、Pytorch中常用的四种优化器SGD、Momentum、RMSProp、Adam
# -*- coding: utf-8 -*- """ Created on 2021.06.18 @author: LXA """ import torch import torch.nn as tn import torch.nn.functional as tnf from torch.nn.parameter import Parameter import numpy as np class my_actFunc(tn.Module): def __init__(self, actName='linear'): super(my_actFunc, self).__init__() self.actName = actName def forward(self, x_input): if str.lower(self.actName) == 'relu': out_x = tnf.relu(x_input) elif str.lower(self.actName) == 'leaky_relu': out_x = tnf.leaky_relu(x_input) elif str.lower(self.actName) == 'tanh': out_x = torch.tanh(x_input) elif str.lower(self.actName) == 'srelu': out_x = tnf.relu(x_input)*tnf.relu(1-x_input) elif str.lower(self.actName) == 'elu': out_x = tnf.elu(x_input) elif str.lower(self.actName) == 'sin': out_x = torch.sin(x_input) elif str.lower(self.actName) == 'sigmoid': out_x = tnf.sigmoid(x_input) else: out_x = x_input return out_x # ----------------dense net(constructing NN and initializing weights and bias )------------ class Pure_DenseNet(tn.Module): """ Args: indim: the dimension for input data outdim: the dimension for output hidden_units: the number of units for hidden layer, a list or a tuple name2Model: the name of using DNN type, DNN , ScaleDNN or FourierDNN actName2in: the name of activation function for input layer actName: the name of activation function for hidden layer actName2out: the name of activation function for output layer scope2W: the namespace of weight scope2B: the namespace of bias """ def __init__(self, indim=1, outdim=1, hidden_units=None, name2Model='DNN', actName2in='tanh', actName='tanh', actName2out='linear', scope2W='Weight', scope2B='Bias', type2float='float32', to_gpu=False, gpu_no=0): super(Pure_DenseNet, self).__init__() self.indim = indim self.outdim = outdim self.hidden_units = hidden_units self.name2Model = name2Model self.actFunc_in = my_actFunc(actName=actName2in) self.actFunc = my_actFunc(actName=actName) self.actFunc_out = my_actFunc(actName=actName2out) self.dense_layers = tn.ModuleList() input_layer = tn.Linear(in_features=indim, out_features=hidden_units[0]) tn.init.xavier_normal_(input_layer.weight) tn.init.uniform_(input_layer.bias, -1, 1) self.dense_layers.append(input_layer) for i_layer in range(len(hidden_units)-1): hidden_layer = tn.Linear(in_features=hidden_units[i_layer], out_features=hidden_units[i_layer+1]) tn.init.xavier_normal_(hidden_layer.weight) tn.init.uniform_(hidden_layer.bias, -1, 1) self.dense_layers.append(hidden_layer) out_layer = tn.Linear(in_features=hidden_units[-1], out_features=outdim) tn.init.xavier_normal_(out_layer.weight) tn.init.uniform_(out_layer.bias, -1, 1) self.dense_layers.append(out_layer) def get_regular_sum2WB(self, regular_model='L2'): regular_w = 0 regular_b = 0 if regular_model == 'L1': for layer in self.dense_layers: regular_w = regular_w + torch.sum(torch.abs(layer.weight)) regular_b = regular_b + torch.sum(torch.abs(layer.bias)) elif regular_model == 'L2': for layer in self.dense_layers: regular_w = regular_w + torch.sum(torch.mul(layer.weight, layer.weight)) regular_b = regular_b + torch.sum(torch.mul(layer.bias, layer.bias)) return regular_w, regular_b def forward(self, inputs, scale=None, training=None, mask=None): # ------ dealing with the input data --------------- dense_in = self.dense_layers[0] H = dense_in(inputs) H = self.actFunc_in(H) # ---resnet(one-step skip connection for two consecutive layers if have equal neurons）--- hidden_record = self.hidden_units[0] for i_layer in range(0, len(self.hidden_units)-1): H_pre = H dense_layer = self.dense_layers[i_layer+1] H = dense_layer(H) H = self.actFunc(H) if self.hidden_units[i_layer + 1] == hidden_record: H = H + H_pre hidden_record = self.hidden_units[i_layer + 1] dense_out = self.dense_layers[-1] H = dense_out(H) H = self.actFunc_out(H) return H class DNN_test(tn.Module): def __init__(self, dim_in=2, dim_out=1, hidden_layers=None, name2Model='DNN', actName_in='tanh', actName_hidden='tanh', actName_out='linear', use_gpu=False, no2gpu=0): super(DNN_test, self).__init__() self.name2Model = name2Model self.dim_in = dim_in self.dim_out = dim_out if name2Model == 'DNN': self.DNN = Pure_DenseNet(indim=dim_in, outdim=dim_out, hidden_units=hidden_layers, name2Model=name2Model, actName2in=actName_in, actName=actName_hidden, actName2out=actName_out) def forward(self, x_input, freq=None): out = self.DNN(x_input, scale=freq) return out def get_sum2wB(self): if self.name2Model == 'DNN' or self.name2Model == 'Scale_DNN' or self.name2Model == 'Fourier_DNN': sum2WB = self.DNN.get_regular_sum2WB() return sum2WB def cal_l2loss(self, x_input=None, freq=None, y_input=None): out = self.DNN(x_input, scale=freq) squre_loss = torch.mul(y_input - out, y_input - out) loss = torch.mean(squre_loss, dim=0) return loss, out def test_DNN(): batch_size = 10 dim_in = 2 dim_out = 1 hidden_list = (10, 20, 10, 10, 20) freq = np.array([1, 2, 3, 4], dtype=np.float32) model_name = 'DNN' init_lr = 0.01 max_it = 10000 with_gpu = True model = DNN_test(dim_in=dim_in, dim_out=dim_out, hidden_layers=hidden_list, name2Model=model_name, actName_in='tanh', actName_hidden='tanh', use_gpu=with_gpu, no2gpu=0) if with_gpu: model = model.cuda(device='cuda:'+str(0)) params2Net = model.DNN.parameters() # 定义优化方法，并给定初始学习率 # optimizer = torch.optim.SGD(params2Net, lr=init_lr) # SGD # optimizer = torch.optim.SGD(params2Net, lr=init_lr, momentum=0.8) # momentum # optimizer = torch.optim.RMSprop(params2Net, lr=init_lr, alpha=0.95) # RMSProp optimizer = torch.optim.Adam(params2Net, lr=init_lr) # Adam # 定义更新学习率的方法 # scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99) # scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 1/(epoch+1)) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 10, gamma=0.995) arr2epoch = [] arr2loss = [] arr2lr = [] for i_epoch in range(max_it): x = np.random.rand(batch_size, dim_in) x = x.astype(dtype=np.float32) torch_x = torch.from_numpy(x) y = np.reshape(np.sin(x[:, 0] * x[:, 0] + x[:, 1] * x[:, 1]), newshape=(-1, 1)) torch_y = torch.from_numpy(y) if with_gpu: torch_x = torch_x.cuda(device='cuda:'+str(0)) torch_y = torch_y.cuda(device='cuda:' + str(0)) loss, prediction = model.cal_l2loss(x_input=torch_x, freq=freq, y_input=torch_y) sum2wb = model.get_sum2wB() optimizer.zero_grad() # 求导前先清零, 只要在下一次求导前清零即可 loss.backward() # 求偏导 optimizer.step() # 更新参数 scheduler.step() if i_epoch % 100 == 0: print('i_epoch --- loss:', i_epoch, loss.item()) # print("第%d个epoch的学习率：%f" % (i_epoch, optimizer.param_groups[0]['lr'])) arr2loss.append(loss.item()) arr2lr.append(optimizer.param_groups[0]['lr']) plt.figure() ax = plt.gca() plt.plot(arr2loss, 'b-.', label='loss') plt.xlabel('epoch/100', fontsize=14) plt.ylabel('loss', fontsize=14) plt.legend(fontsize=18) ax.set_yscale('log') plt.show() # plt.cla() # plt.plot(x[:, 0], x[:, 1], y, 'b*') # plt.show() if __name__ == "__main__": test_DNN()