import torch import torch.nn as nn import numpy as np import math import os from torch.nn import functional as F import torch.nn.init as init import logging class HSwish(nn.Module): def forward(self, x): out = x * F.relu6(x + 3, inplace=True) / 6 return out class ConvBNACT(nn.Module): def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, groups=1, act=None): super().__init__() self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, groups=groups, bias=False) self.bn = nn.BatchNorm2d(out_channels) if act == 'relu': self.act = nn.ReLU(inplace=True) elif act == 'hard_swish': self.act = HSwish() elif act is None: self.act = None def forward(self, x): x = self.conv(x) x = self.bn(x) if self.act is not None: x = self.act(x) return x class ConvBNACTWithPool(nn.Module): def __init__(self, in_channels, out_channels, kernel_size, groups=1, act=None): super().__init__() # self.pool = nn.AvgPool2d(kernel_size=2, stride=2, padding=0, ceil_mode=True) self.pool = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=1, padding=(kernel_size - 1) // 2, groups=groups, bias=False) self.bn = nn.BatchNorm2d(out_channels) if act is None: self.act = None else: self.act = nn.ReLU(inplace=True) def forward(self, x): x = self.pool(x) x = self.conv(x) x = self.bn(x) if self.act is not None: x = self.act(x) return x class ShortCut(nn.Module): def __init__(self, in_channels, out_channels, stride, name, if_first=False): super().__init__() assert name is not None, 'shortcut must have name' self.name = name if in_channels != out_channels or stride != 1: if if_first: self.conv = ConvBNACT(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride, padding=0, groups=1, act=None) else: self.conv = ConvBNACTWithPool(in_channels=in_channels, out_channels=out_channels, kernel_size=1, groups=1, act=None) elif if_first: self.conv = ConvBNACT(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride, padding=0, groups=1, act=None) else: self.conv = None def forward(self, x): if self.conv is not None: x = self.conv(x) return x class BottleneckBlock(nn.Module): def __init__(self, in_channels, out_channels, stride, if_first, name): super().__init__() assert name is not None, 'bottleneck must have name' self.name = name self.conv0 = ConvBNACT(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=1, padding=0, groups=1, act='relu') self.conv1 = ConvBNACT(in_channels=out_channels, out_channels=out_channels, kernel_size=3, stride=stride, padding=1, groups=1, act='relu') self.conv2 = ConvBNACT(in_channels=out_channels, out_channels=out_channels * 4, kernel_size=1, stride=1, padding=0, groups=1, act=None) self.shortcut = ShortCut(in_channels=in_channels, out_channels=out_channels * 4, stride=stride, if_first=if_first, name=f'{name}_branch1') self.relu = nn.ReLU(inplace=True) self.output_channels = out_channels * 4 def forward(self, x): y = self.conv0(x) y = self.conv1(y) y = self.conv2(y) y = y + self.shortcut(x) return self.relu(y) class BasicBlock(nn.Module): def __init__(self, in_channels, out_channels, stride, if_first, name): super().__init__() assert name is not None, 'block must have name' self.name = name self.conv0 = ConvBNACT(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=stride, padding=1, groups=1, act='relu') self.conv1 = ConvBNACT(in_channels=out_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1, groups=1, act=None) self.shortcut = ShortCut(in_channels=in_channels, out_channels=out_channels, stride=stride, name=f'{name}_branch1', if_first=if_first, ) self.relu = nn.ReLU(inplace=True) self.output_channels = out_channels def forward(self, x): y = self.conv0(x) y = self.conv1(y) y = y + self.shortcut(x) return self.relu(y) class ResNet(nn.Module): def __init__(self, in_channels, layers, out_indices=[0, 1, 2, 3], pretrained=True, **kwargs): """ the Resnet backbone network for detection module. Args: params(dict): the super parameters for network build """ super().__init__() supported_layers = { 18: {'depth': [2, 2, 2, 2], 'block_class': BasicBlock}, 34: {'depth': [3, 4, 6, 3], 'block_class': BasicBlock}, 50: {'depth': [3, 4, 6, 3], 'block_class': BottleneckBlock}, 101: {'depth': [3, 4, 23, 3], 'block_class': BottleneckBlock}, 152: {'depth': [3, 8, 36, 3], 'block_class': BottleneckBlock}, 200: {'depth': [3, 12, 48, 3], 'block_class': BottleneckBlock} } assert layers in supported_layers, \ "supported layers are {} but input layer is {}".format(supported_layers, layers) depth = supported_layers[layers]['depth'] block_class = supported_layers[layers]['block_class'] self.use_supervised = kwargs.get('use_supervised', False) self.out_indices = out_indices num_filters = [64, 128, 256, 512] self.conv1 = nn.Sequential( ConvBNACT(in_channels=in_channels, out_channels=32, kernel_size=3, stride=2, padding=1, act='relu'), ConvBNACT(in_channels=32, out_channels=32, kernel_size=3, stride=1, padding=1, act='relu'), ConvBNACT(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1, act='relu') ) self.pool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.stages = nn.ModuleList() self.out_channels = [] tmp_channels = [] in_ch = 64 for block_index in range(len(depth)): block_list = [] for i in range(depth[block_index]): if layers >= 50: if layers in [101, 152, 200] and block_index == 2: if i == 0: conv_name = "res" + str(block_index + 2) + "a" else: conv_name = "res" + str(block_index + 2) + "b" + str(i) else: conv_name = "res" + str(block_index + 2) + chr(97 + i) else: conv_name = f'res{str(block_index + 2)}{chr(97 + i)}' block_list.append(block_class(in_channels=in_ch, out_channels=num_filters[block_index], stride=2 if i == 0 and block_index != 0 else 1, if_first=block_index == i == 0, name=conv_name)) in_ch = block_list[-1].output_channels tmp_channels.append(in_ch) self.stages.append(nn.Sequential(*block_list)) for idx, ch in enumerate(tmp_channels): if idx in self.out_indices: self.out_channels.append(ch) if pretrained: ckpt_path = f'./weights/resnet{layers}_vd.pth' logger = logging.getLogger('torchocr') if os.path.exists(ckpt_path): logger.info('load imagenet weights') self.load_state_dict(torch.load(ckpt_path)) else: logger.info(f'{ckpt_path} not exists') if self.use_supervised: ckpt_path = f'./weights/res_supervised_140w_387e.pth' logger = logging.getLogger('torchocr') if os.path.exists(ckpt_path): logger.info('load supervised weights') self.load_state_dict(torch.load(ckpt_path)) else: logger.info(f'{ckpt_path} not exists') def forward(self, x): x = self.conv1(x) x = self.pool1(x) out = [] for idx, stage in enumerate(self.stages): x = stage(x) if idx in self.out_indices: out.append(x) return out def weights_init(m): if isinstance(m, nn.Conv2d): init.kaiming_normal_(m.weight.data) if m.bias is not None: init.normal_(m.bias.data) elif isinstance(m, nn.ConvTranspose2d): init.kaiming_normal_(m.weight.data) if m.bias is not None: init.normal_(m.bias.data) elif isinstance(m, nn.BatchNorm2d): init.normal_(m.weight.data, mean=1, std=0.02) init.constant_(m.bias.data, 0) class DB_fpn(nn.Module): def __init__(self, in_channels, out_channels=256, **kwargs): """ :param in_channels: 基础网络输出的维度 :param kwargs: """ super().__init__() inplace = True self.out_channels = out_channels # reduce layers self.in2_conv = nn.Conv2d(in_channels[0], self.out_channels, kernel_size=1, bias=False) self.in3_conv = nn.Conv2d(in_channels[1], self.out_channels, kernel_size=1, bias=False) self.in4_conv = nn.Conv2d(in_channels[2], self.out_channels, kernel_size=1, bias=False) self.in5_conv = nn.Conv2d(in_channels[3], self.out_channels, kernel_size=1, bias=False) # Smooth layers self.p5_conv = nn.Conv2d(self.out_channels, self.out_channels // 4, kernel_size=3, padding=1, bias=False) self.p4_conv = nn.Conv2d(self.out_channels, self.out_channels // 4, kernel_size=3, padding=1, bias=False) self.p3_conv = nn.Conv2d(self.out_channels, self.out_channels // 4, kernel_size=3, padding=1, bias=False) self.p2_conv = nn.Conv2d(self.out_channels, self.out_channels // 4, kernel_size=3, padding=1, bias=False) self.in2_conv.apply(weights_init) self.in3_conv.apply(weights_init) self.in4_conv.apply(weights_init) self.in5_conv.apply(weights_init) self.p5_conv.apply(weights_init) self.p4_conv.apply(weights_init) self.p3_conv.apply(weights_init) self.p2_conv.apply(weights_init) def _interpolate_add(self, x, y): return F.interpolate(x, scale_factor=2) + y def _interpolate_cat(self, p2, p3, p4, p5): p3 = F.interpolate(p3, scale_factor=2) p4 = F.interpolate(p4, scale_factor=4) p5 = F.interpolate(p5, scale_factor=8) return torch.cat([p5, p4, p3, p2], dim=1) def forward(self, x): c2, c3, c4, c5 = x in5 = self.in5_conv(c5) in4 = self.in4_conv(c4) in3 = self.in3_conv(c3) in2 = self.in2_conv(c2) out4 = self._interpolate_add(in5, in4) out3 = self._interpolate_add(out4, in3) out2 = self._interpolate_add(out3, in2) p5 = self.p5_conv(in5) p4 = self.p4_conv(out4) p3 = self.p3_conv(out3) p2 = self.p2_conv(out2) x = self._interpolate_cat(p2, p3, p4, p5) return x class Head(nn.Module): def __init__(self, in_channels): super().__init__() self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels // 4, kernel_size=3, padding=1, bias=False) # self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels // 4, kernel_size=5, padding=2, # bias=False) self.conv_bn1 = nn.BatchNorm2d(in_channels // 4) self.relu = nn.ReLU(inplace=True) self.conv2 = nn.ConvTranspose2d(in_channels=in_channels // 4, out_channels=in_channels // 4, kernel_size=2, stride=2) self.conv_bn2 = nn.BatchNorm2d(in_channels // 4) self.conv3 = nn.ConvTranspose2d(in_channels=in_channels // 4, out_channels=1, kernel_size=2, stride=2) def forward(self, x): x = self.conv1(x) x = self.conv_bn1(x) x = self.relu(x) x = self.conv2(x) x = self.conv_bn2(x) x = self.relu(x) x = self.conv3(x) x = torch.sigmoid(x) return x class DBHead(nn.Module): """ Differentiable Binarization (DB) for text detection: see https://arxiv.org/abs/1911.08947 args: params(dict): super parameters for build DB network """ def __init__(self, in_channels, k=50): super().__init__() self.k = k self.binarize = Head(in_channels) self.thresh = Head(in_channels) self.binarize.apply(weights_init) self.thresh.apply(weights_init) def step_function(self, x, y): return torch.reciprocal(1 + torch.exp(-self.k * (x - y))) def forward(self, x): shrink_maps = self.binarize(x) if not self.training: return shrink_maps threshold_maps = self.thresh(x) binary_maps = self.step_function(shrink_maps, threshold_maps) y = torch.cat((shrink_maps, threshold_maps, binary_maps), dim=1) return y class DB_ResNet_18(nn.Module): def __init__(self, ): super().__init__() self.backbone = ResNet(in_channels=3,layers=18,pretrained=False) self.neck = DB_fpn(in_channels=self.backbone.out_channels,out_channels=256) self.head = DBHead(self.neck.out_channels) def forward(self, x): x = self.backbone(x) x = self.neck(x) x = self.head(x) return x