yolo darknet的启示（残差相加放在leakrelu之后）

之前的训练cifar10的lenet网络架构修改后的网络架构看到没残差后的relu被我注释掉了我们再看前版本残差块residualExt2再看新版本residualExt2改了什么class residualExt2 :public Layer {//改进成先降维再升维202607101844public:residualExt2(cudnnHandle_t cudnn_, int batch_, int c, int h, int w) : cudnn(cudnn_), batch(batch_), _c(c), _h(h), _w(w) {layers.emplace_back(std::make_sharedConv2D(cudnn, batch, _c, _c , _h, _w, 1, 1));//c3,6*12*12-16*8*8layers.emplace_back(std::make_sharedBN(cudnn, batch, _c , _h, _w));layers.emplace_back(std::make_sharedLeakyRL(cudnn, batch, _c , _h, _w)); //c3,6*12*12-16*8*8layers.emplace_back(std::make_sharedConv2D(cudnn, batch, _c , _c, _h, _w, 3, 1, 1));//尝试残差此处要记住输入X//layers.emplace_back(std::make_sharedConv2D(cudnn, batch, _c, _c/2, _h, _w, 1, 1));//c3,6*12*12-16*8*8//layers.emplace_back(std::make_sharedBN(cudnn, batch, _c/2, _h, _w));//layers.emplace_back(std::make_sharedLeakyRL(cudnn, batch, _c /2, _h, _w)); //c3,6*12*12-16*8*8//layers.emplace_back(std::make_sharedConv2D(cudnn, batch, _c/2, _c, _h, _w, 3, 1, 1));layers.emplace_back(std::make_sharedBN(cudnn, batch, _c, _h, _w));layers.emplace_back(std::make_sharedLeakyRL(cudnn, batch, _c, _h, _w));//20260710收到darknet的启发1506cudaMalloc(output, batch * _c * _h * _w * sizeof(float));//输出32*32*32-----------------------显然输入也是32*32*32cudaMalloc(input2, batch * _c * _h * _w * sizeof(float));cudaMalloc(d_residual, batch * _c * _h * _w * sizeof(float));// cudaMalloc(output, batch * 10 * sizeof(float));//这里的10代表10个类所以不能用cudaMalloc(grad_input, batch * _c * _h * _w * sizeof(float));//反向和梯度计算不管}void forward(float* input_)override {input input_;input2 input_;for (const auto l : layers) {l-forward(input);input l-get_output();}int NN batch * _c * _h * _w;residual_forward_kernel (NN 255) / 256, 256 (output, input, input2, NN);error_handling(cudaGetLastError());//cudaMemcpy(input2, inputTemp, sizeof(float)*batch * 10, cudaMemcpyDeviceToDevice);}void forward2(float* input_)override {input input_;input2 input_; //batch 1;for (const auto l : layers) {l-forward2(input);input l-get_output();}int NN batch * _c * _h * _w;//int NN batch * 32 * 32 * 32;residual_forward_kernel (NN 255) / 256, 256 (output, input, input2, NN);// error_handling(cudaGetLastError());// cudaMemcpy(input2, inputTemp, sizeof(float)*batch * 10, cudaMemcpyDeviceToDevice);/*const float alpha 1.0f, beta 0.0f;forward(input);*/}/*void forward2(float* inputtest)override {input inputtest;const float alpha 1.0f, beta 0.0f;forward(input);}*/void backward(float* grad_output)override {//梯度来自残差块后的relu当前只有一个残差块float* grad grad_output;//要记住这个梯度即备份一个float* grad备用 grad_output;for (int i layers.size() - 1; i 0; i--) {layers[i]-backward(grad);grad layers[i]-get_grad_input();}//float* d_residual grad备用*X输入数据;//input2 input_;// float* d_residual grad备用*input2;//input2 input_;int NN batch * _c * _h * _w;/*for (int i 0; i NN; i){d_residual[i] grad备用[i]*input2[i];}*/int threads 256;int blocks (NN threads - 1) / threads;//mulext blocks, threads (NN, batch, _c, _h, _w, input2, _c, grad备用);//// mul blocks, threads (grad备用, input2, d_residual, NN);//c为输出d_residual//error_handling(cudaGetLastError());//residual_backprop_kernel blocks, threads (grad, grad_input, grad备用, NN);//error_handling(cudaGetLastError());//// cudaMemcpy(grad_input, grad, sizeof(float)*batch * 32 * 32 * 32, cudaMemcpyDeviceToDevice);//使用yolo 的残差试一试看两个bn有什么情况mul blocks, threads (grad备用, input2, d_residual, NN);//c为输出d_residualerror_handling(cudaGetLastError());shortcut_gpu(batch, _w, _h, _c, d_residual, _w, _h, _c, grad);//虚线l.out_c12,l.c16,在这里是实线l.out_c16,l.c16cudaMemcpy(grad_input, grad, sizeof(float) * NN, cudaMemcpyDeviceToDevice);error_handling(cudaGetLastError());//仍然是第二个bn层方差均值为零}int getname() override { return 3; }float* get_output() override { return output; }float* get_grad_input() override { return grad_input; }void update(float lr) {for (const auto l : layers) {l-update(lr);}}~residualExt2() {cudaFree(output);cudaFree(grad_input);}private:// cublasHandle_t cublas;int _c, _h, _w;cudnnHandle_t cudnn;int batch;float* input, * output, * grad_input;float* input2;float* d_residual;public:std::vectorstd::shared_ptrLayer layers;};上面红色加粗的是新增及修改过的mul函数修改如下__global__ voidmul(float* a, float* b, float* c, int N) {int index blockIdx.x * blockDim.x threadIdx.x;if (index N) {//c[index] a[index] * (b[index] 0 ? 1 : 0.1);//之前最优版本leakyrelu放在add之后c[index] a[index] * b[index] ;//借鉴darknetleakyrelu放在add之前了202607101514// c[index] a[index] * (1 - b[index]* b[index]);//tanhx方式}}