BIDI_EXTRACTION_AI/BiddingKG/dl/channel/channel_bert.py

# coding: UTF-8
import torch
import torch.nn as nn
import torch.nn.functional as F
import time
import re
import os
import transformers
from transformers import ElectraTokenizer
import numpy as np
import json

# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device("cpu") # 线上用CPU

class PositionalEncoding(nn.Module):

    def __init__(self,dim_hid):
        super(PositionalEncoding,self).__init__()
        base_array = np.array([np.power(10000,2*(hid_j//2)/dim_hid) for hid_j in range(dim_hid)])
        self.base_tensor = torch.from_numpy(base_array).to(torch.float32).to(device) #[1,D]

    def forward(self,x):
        # x(B,N,d)
        B,N,d = x.shape
        pos = torch.arange(N).unsqueeze(-1).to(torch.float32).to(device) #[N,1]
        pos = pos/self.base_tensor
        pos = pos.unsqueeze(0)
        pos[:,:,0::2] = torch.sin(pos[:,:,0::2])
        pos[:,:,1::2] = torch.cos(pos[:,:,1::2])
        return x+pos

class ScaledDotProductAttention(nn.Module):
    ''' Scaled Dot-Product Attention '''

    def __init__(self, temperature, attn_dropout=0.1):
        super().__init__()
        self.temperature = temperature
        self.dropout = nn.Dropout(attn_dropout)

    def forward(self, q, k, v, mask=None):

        # print(q.shape,k.shape)
        attn = torch.matmul(q / self.temperature, k.transpose(2, 3))
        if mask is not None:
            attn = attn.masked_fill(mask == 0, -1e9)
        # t1 = time.time()
        attn = self.dropout(torch.softmax(attn, dim=-1))
        # print('cost',time.time()-t1) # 主要时间花费
        output = torch.matmul(attn, v)

        return output, attn


class MultiHeadAttention(nn.Module):
    ''' Multi-Head Attention module '''

    def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):
        super().__init__()

        self.n_head = n_head
        self.d_k = d_k
        self.d_v = d_v

        self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False)
        self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False)
        self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False)
        self.fc = nn.Linear(n_head * d_v, d_model, bias=False)

        self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5)

        self.dropout = nn.Dropout(dropout)
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

        self.rotaryEmbedding = RotaryEmbedding(d_k)

    def forward(self, q, k, v, mask=None):

        d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
        sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1)

        residual = q


        # Pass through the pre-attention projection: b x lq x (n*dv)
        # Separate different heads: b x lq x n x dv
        q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
        k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
        v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)


        # Transpose for attention dot product: b x n x lq x dv
        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)

        # RoPE embed
        target_tensor = torch.zeros((q.size(0), q.size(2)))
        position_ids = torch.arange(q.size(2), dtype=torch.long).unsqueeze(0).expand_as(target_tensor)
        _cos, _sin = self.rotaryEmbedding(q, position_ids)
        q, k = apply_rotary_pos_emb(q, k, _cos, _sin)

        if mask is not None:
            # mask = mask.unsqueeze(1)   # For head axis broadcasting.
            mask = mask.unsqueeze(1).unsqueeze(2)   # For head axis broadcasting.

        q, attn = self.attention(q, k, v, mask=mask)

        #q (sz_b,n_head,N=len_q,d_k)
        #k (sz_b,n_head,N=len_k,d_k)
        #v (sz_b,n_head,N=len_v,d_v)

        # Transpose to move the head dimension back: b x lq x n x dv
        # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv)
        q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1)

        #q (sz_b,len_q,n_head,N * d_k)
        q = self.dropout(self.fc(q))
        q += residual

        q = self.layer_norm(q)

        return q, attn


class PositionwiseFeedForward(nn.Module):
    ''' A two-feed-forward-layer module '''

    def __init__(self, d_in, d_hid, dropout=0.1):
        super().__init__()
        self.w_1 = nn.Linear(d_in, d_hid) # position-wise
        self.w_2 = nn.Linear(d_hid, d_in) # position-wise
        self.layer_norm = nn.LayerNorm(d_in, eps=1e-6)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):

        residual = x
        x = self.w_2(torch.relu(self.w_1(x)))
        x = self.dropout(x)
        x += residual
        x = self.layer_norm(x)

        return x


class RotaryEmbedding(nn.Module):
    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
        super().__init__()

        self.dim = dim # it is set to the head_dim
        self.max_position_embeddings = max_position_embeddings
        self.base = base

        # Calculate the theta according to the formula theta_i = base^(2i/dim) where i = 0, 1, 2, ..., dim // 2
        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim))
        self.register_buffer("inv_freq", tensor=inv_freq, persistent=False)

    @torch.no_grad()
    def forward(self, x, position_ids, seq_len=None):
        # x: [bs, num_attention_heads, seq_len, head_size]
        self.inv_freq = self.inv_freq.to(device)
        position_ids = position_ids.to(device)
        # Copy the inv_freq tensor for batch in the sequence
        # inv_freq_expanded: [Batch_Size, Head_Dim // 2, 1]
        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
        # position_ids_expanded: [Batch_Size, 1, Seq_Len]
        position_ids_expanded = position_ids[:, None, :].float()

        # Multiply each theta by the position (which is the argument of the sin and cos functions)
        # freqs: [Batch_Size, Head_Dim // 2, 1] @ [Batch_Size, 1, Seq_Len] --> [Batch_Size, Seq_Len, Head_Dim // 2]
        freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
        # emb: [Batch_Size, Seq_Len, Head_Dim]
        emb = torch.cat((freqs, freqs), dim=-1)
        # cos, sin: [Batch_Size, Seq_Len, Head_Dim]
        cos = emb.cos()
        sin = emb.sin()
        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)


def rotate_half(x):
    # Build the [-x2, x1, -x4, x3, ...] tensor for the sin part of the positional encoding.
    x1 = x[..., : x.shape[-1] // 2] # Takes the first half of the last dimension
    x2 = x[..., x.shape[-1] // 2 :] # Takes the second half of the last dimension
    return torch.cat((-x2, x1), dim=-1)


def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
    cos = cos.unsqueeze(unsqueeze_dim) # Add the head dimension
    sin = sin.unsqueeze(unsqueeze_dim) # Add the head dimension
    # Apply the formula (34) of the Rotary Positional Encoding paper.
    q_embed = (q * cos) + (rotate_half(q) * sin)
    k_embed = (k * cos) + (rotate_half(k) * sin)
    return q_embed, k_embed



class EncoderLayer(nn.Module):
    ''' Compose with two layers '''

    def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):
        super(EncoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)

    def forward(self, enc_input, slf_attn_mask=None):
        enc_output, enc_slf_attn = self.slf_attn(
            enc_input, enc_input, enc_input, mask=slf_attn_mask)
        enc_output = self.pos_ffn(enc_output)
        return enc_output, enc_slf_attn


class Encoder(nn.Module):
    ''' A encoder model with self attention mechanism. '''

    def __init__(
            self, n_src_vocab, d_word_vec, n_layers, n_head, d_k, d_v,
            d_model, d_inner, pad_idx, dropout=0.1, n_position=200, scale_emb=False,embedding=None):

        super().__init__()

        if embedding is None:
            self.src_word_emb = nn.Embedding(n_src_vocab, d_word_vec, padding_idx=pad_idx)
        else:
            self.src_word_emb = nn.Embedding(n_src_vocab, d_word_vec,padding_idx=pad_idx,
                                             # _weight=torch.from_numpy(embedding ,))
                                             # _weight=torch.tensor(embedding ,dtype=torch.float64).to(device))
                                             _weight=torch.tensor(embedding))
        self.position_enc = PositionalEncoding(d_word_vec)
        self.dropout = nn.Dropout(p=dropout)
        self.layer_stack = nn.ModuleList([
            EncoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout)
            for _ in range(n_layers)])
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
        self.scale_emb = scale_emb
        self.d_model = d_model

    def forward(self, src_seq, src_mask, return_attns=False):

        enc_slf_attn_list = []

        # -- Forward
        enc_output = self.src_word_emb(src_seq)
        if self.scale_emb:
            enc_output *= self.d_model ** 0.5

        # enc_output = self.dropout(self.position_enc(enc_output))
        enc_output = self.dropout(enc_output)

        enc_output = self.layer_norm(enc_output)

        for enc_layer in self.layer_stack:
            enc_output, enc_slf_attn = enc_layer(enc_output, slf_attn_mask=src_mask)
            enc_slf_attn_list += [enc_slf_attn] if return_attns else []

        if return_attns:
            return enc_output, enc_slf_attn_list
        return enc_output

class bidiBert(nn.Module):

    def __init__(self, n_src_vocab, d_word_vec, n_layers, n_head, d_k, d_v,
                 d_model, d_inner, pad_idx,n_class,embedding = None):
        super(bidiBert, self).__init__()

        self.encoder = Encoder(n_src_vocab, d_word_vec, n_layers, n_head, d_k, d_v,
                               d_model, d_inner, pad_idx,embedding = embedding)
        # self.src_word_emb = nn.Embedding(n_src_vocab, d_word_vec, padding_idx=pad_idx,
        #                                  # _weight=torch.from_numpy(embedding ,))
        #                                  # _weight=torch.tensor(embedding ,dtype=torch.float64).to(device))
        #                                  _weight=torch.tensor(embedding))
        # self.encoder =  nn.LSTM(128, 256, 2,
        #                     bidirectional=True, batch_first=True, dropout=0.3)

        self.dropout = nn.Dropout(p=0.1)
        self.pooler = nn.Linear(d_inner,d_inner)
        self.liner = nn.Linear(d_inner,n_class)
        # self.liner = nn.Linear(256*4,n_class)
        self.avg_pool1 = nn.AdaptiveAvgPool1d(1) # Max Pooling: nn.AdaptiveMaxPool1d(1)
        self.avg_pool2 = nn.AdaptiveAvgPool1d(1)

    def forward(self, input_title,input_doctext):
        # out = self.encoder(inputs, attention_mask)

        # input_title = self.src_word_emb(input_title)
        # input_title,_ = self.encoder(input_title)
        # input_title = self.encoder(src_seq=input_title[0], src_mask=input_title[1])
        # input_title = input_title[:, 1, :]
        # input_title = torch.mean(input_title,dim=-2)
        # input_title = self.avg_pool1(input_title.transpose(1, 2)).squeeze(-1)
        # input_title = torch.tanh(self.pooler(input_title[:,0]))

        # input_doctext = self.src_word_emb(input_doctext)
        # input_doctext,_ = self.encoder(input_doctext)
        input_doctext = self.encoder(src_seq=input_doctext[0], src_mask=input_doctext[1])
        # input_doctext = input_doctext[:, 1, :]
        # input_doctext = torch.mean(input_doctext,dim=-2)
        # input_doctext = self.avg_pool2(input_doctext.transpose(1, 2)).squeeze(-1)
        input_doctext = self.pooler(input_doctext[:,0])
        input_doctext = self.dropout(input_doctext)
        input_doctext = torch.tanh(input_doctext)

        # print('size:',input_title.size(),input_doctext.size())
        # out = torch.cat((input_title, input_doctext), dim=-1)
        out = input_doctext
        # bs, n, m = out.size()
        # out = out.view(bs,n*m)
        out = self.liner(out)
        out = F.softmax(out, dim=-1)
        return out


phone = re.compile('1[3-9][0-9][-—－―]?\d{4}[-—－―]?\d{4}|'
                       '\+86.?1[3-9]\d{9}|'
                       # '0[^0]\d{1,2}[-—－―][1-9]\d{6,7}/[1-9]\d{6,10}|'
                       '0[1-9]\d{1,2}[-—－―][2-9]\d{6}\d?[-—－―]\d{1,4}|'
                       '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?(?=1[3-9]\d{9})|'
                       '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?(?=0[1-9]\d{1,2}[-—－―]?[2-9]\d{6}\d?)|'
                       '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?(?=[2-9]\d{6,7})|'
                       '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?|'
                       '[\（|\(]0[1-9]\d{1,2}[\）|\)]-?[2-9]\d{6}\d?-?\d{,4}|'
                       '400\d{7}转\d{1,4}|'
                       '[2-9]\d{6,7}')
def text_process(text):
    text = text.strip()
    text = re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]',"",text) # 非法字符
    text = re.sub("extractJson:|fullTextSeg:","",text)

    # text = re.sub("[?？]{1,}", "", text)
    text = re.sub("[?？]{2,}", "", text)
    text = re.sub(r'(http[s]?://|www\.)(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', "", text) # 网站
    text = re.sub(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', "", text)# 邮箱
    text = re.sub('[0-9a-zA-Z@#*%=?&~()_|<>/.（）｛｝【】{}\[\]\-]{6,}', "", text) # 编号

    text = re.sub(phone,"",text) # 号码

    text = re.sub(r'\b\d[-.\s]?\d{3}[-.\s]?\d{4}\b', "", text) # 座机
    text = re.sub('0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?', "", text) # 座机
    text = re.sub('1(3[0-9]|4[01456879]|5[0-35-9]|6[2567]|7[0-8]|8[0-9]|9[0-35-9])\d{8}', "", text)# 手机号
    text = re.sub('&?nbsp;?|&?ensp;?|&?emsp;?', "", text)
    text = re.sub('\\\\n|\\\\r|\\\\t', "", text)
    # text = re.sub("\s+", "", text)
    text = re.sub("\s+", " ", text)

    # 优化部分未识别表达
    text = re.sub("中止", "终止", text)
    text = re.sub("遴选", "招标", text)
    text = re.sub("合同段", "", text)

    return text

label2class_dict = {
        0: 51, 1:52 , 2:101,
         3:102,  4:103, 5:105,
         6:114,  7:118,  8:119,
         9:120,  10:121,  11:122
}

def channel_predict(title,text,min_text_len=None):
    if globals().get("channel_pytorch_model") is None or globals().get("channel_tokenizer") is None:
        # config
        config = {
            # 'n_src_vocab': len(vocab),
            'd_word_vec': 128,
            'n_layers': 3,
            'n_head': 3,
            'd_k': 128,
            'd_v': 128,
            'd_model': 128,
            'd_inner': 128,
            'pad_idx': 0
        }
        # n_src_vocab = config['n_src_vocab']
        d_word_vec = config['d_word_vec']
        n_layers = config['n_layers']
        n_head = config['n_head']
        d_k = config['d_k']
        d_v = config['d_v']
        d_model = config['d_model']
        d_inner = config['d_inner']
        pad_idx = config['pad_idx']
        n_class = 12

        # tokenizer
        base_model_name = os.path.abspath(os.path.dirname(__file__)) + "/pytorch_model/tokenizer"
        tokenizer = ElectraTokenizer.from_pretrained(base_model_name)
        n_src_vocab = len(tokenizer.get_vocab())

        # 实例化模型
        model_path = os.path.abspath(os.path.dirname(__file__)) + '/pytorch_model/channel.pth'
        model = bidiBert(n_src_vocab, d_word_vec, n_layers, n_head, d_k, d_v,
                     d_model, d_inner, pad_idx, n_class, embedding=None)
        model.to(device)

        model_state = torch.load(model_path, map_location=device)
        model_state_dict = model.state_dict()
        pretrained_state_dict = model_state
        # missing_keys = set(model_state_dict.keys()) - set(pretrained_state_dict.keys())
        unexpected_keys = set(pretrained_state_dict.keys()) - set(model_state_dict.keys())

        add_kv = []
        for k, v in model_state.items():
            if k in unexpected_keys:
                # model_state[k.replace("module.","")] = v
                add_kv.append([k.replace("module.", ""), v])
        for i in add_kv:
            model_state[i[0]] = i[1]
        for k in list(unexpected_keys):
            del model_state[k]
        model.load_state_dict(model_state)

        # 将模型设置为评估模式
        model.eval()
        globals()["channel_pytorch_model"] = model
        globals()["channel_tokenizer"] = tokenizer
    else:
        model = globals().get("channel_pytorch_model")
        tokenizer = globals().get("channel_tokenizer")


    # process text
    if title in text:
        text = text.replace(title, '', 1)
        text = text.lstrip("，")
        text = text.lstrip("。")
    if "##attachment##" in text:
        main_text,attachment_text = text.split("##attachment##",maxsplit=1)
        # print('main_text',main_text)
        if len(main_text)>=500: # 正文有足够的内容时不需要使用附件预测
            text = main_text
    text = re.sub("##attachment##。?","",text)
    text = text_process(text)

    if min_text_len is None:
        min_text_len = 200
    if len(text)<=min_text_len:
        # 正文内容过短时，不预测
        return
    # elif len(text)<=150:
    #     # 正文内容过短时，重复正文
    #     text = text * 2
    text = text[:2000]
    title = text_process(title)
    title = title[:100]
    text = "公告标题：" + title + "。" + "公告内容：" + text
    text = text[:2000]
    # print('predict text:',text)

    # to torch data
    text = [text]
    text_max_len = 2000
    # text = [tokenizer.encode_plus(
    #     _t,
    #     add_special_tokens=True,  # 添加特殊标记，如[CLS]和[SEP]
    #     max_length=text_max_len,  # 设置最大长度
    #     padding='max_length',  # 填充到最大长度
    #     truncation=True,  # 截断超过最大长度的文本
    #     return_attention_mask=True,  # 返回attention_mask
    #     return_tensors='pt'  # 返回PyTorch张量
    # ) for _t in text]
    # text = [torch.LongTensor(np.array([_t['input_ids'].numpy()[0] for _t in text])).to(device),
    #      torch.LongTensor(np.array([_t['attention_mask'].numpy()[0] for _t in text])).to(device)]
    text = [tokenizer.encode_plus(
        _t,
        add_special_tokens=True,  # 添加特殊标记，如[CLS]和[SEP]
        max_length=text_max_len,  # 设置最大长度
        padding='max_length',  # 填充到最大长度
        truncation=True,  # 截断超过最大长度的文本
        return_attention_mask=True,  # 返回attention_mask
        # return_tensors='pt'  # 返回PyTorch张量
        return_tensors=None  #不返回PyTorch张量
    ) for _t in text]
    text = [torch.LongTensor(np.array([_t['input_ids'] for _t in text])).to(device),
            torch.LongTensor(np.array([_t['attention_mask'] for _t in text])).to(device)]
    # predict
    with torch.no_grad():
        outputs = model(None, text)
        predic = torch.max(outputs.data, 1)[1].cpu().numpy()
        pred_prob = torch.max(outputs.data, 1)[0].cpu().numpy()
        # print('pred_prob',pred_prob)
        if pred_prob>0.5:
            pred_label = predic[0]
            pred_class = label2class_dict[pred_label]
        else:
            return
    # print('check rule before',pred_class)
    # check rule
    if pred_class==101 and re.search("((资格|资质)(审查|预审|后审|审核)|资审)结果(公告|公示)?|(资质|资格)(预审|后审)公示|资审及业绩公示",title): # 纠正部分‘资审结果’模型错误识别为中标
        pred_class = 105
    elif pred_class==122 and re.search("验收服务",title):
        pred_class = None
    # elif pred_class==118 and re.search("重新招标",title): #重新招标类公告，因之前公告的废标原因而错识别为废标公告
    #     pred_class = 52


    return pred_class


class_dict = {51: '公告变更',
       52: '招标公告',
       101: '中标信息',
       102: '招标预告',
       103: '招标答疑',
       104: '招标文件',
       105: '资审结果',
       106: '法律法规',
       107: '新闻资讯',
       108: '拟建项目',
       109: '展会推广',
       110: '企业名录',
       111: '企业资质',
       112: '全国工程',
       113: '业主采购',
       114: '采购意向',
       115: '拍卖出让',
       116: '土地矿产',
       117: '产权交易',
       118: '废标公告',
       119: '候选人公示',
       120: '合同公告',
       121: '开标记录',
       122: '验收合同'
              }

tenderee_type = ['公告变更','招标公告','招标预告','招标答疑','资审结果','采购意向']
win_type = ['中标信息','废标公告','候选人公示','合同公告','开标记录','验收合同']


def is_contain_winner(extract_json):
    if re.search('win_tenderer', extract_json):
        return True
    else:
        return False

def merge_channel(list_articles,channel_dic,original_docchannel,page_time,prem={},web_source_no="",is_finnal=False):

    def merge_rule(title,text,docchannel,pred_channel,channel_dic,original_docchannel):
        front_text_len = len(text)//3 if len(text)>300 else 100
        front_text = text[:front_text_len]
        pred_channel = class_dict[pred_channel]
        # print('pred_channel',pred_channel,'docchannel',docchannel,'original_docchannel',original_docchannel)
        if pred_channel == docchannel:
            channel_dic['docchannel']['use_original_docchannel'] = 0
        else:
            if pred_channel in ['采购意向','招标预告'] and docchannel in ['采购意向','招标预告']:
                merge_res = '采购意向' if re.search("意向|意愿",title) or re.search("意向|意愿",front_text) else "招标预告"
                channel_dic['docchannel']['docchannel'] = merge_res
                channel_dic['docchannel']['use_original_docchannel'] = 0
            elif pred_channel in ['公告变更','招标答疑'] and docchannel in ['公告变更','招标答疑']:
                channel_dic['docchannel']['docchannel'] = docchannel
                channel_dic['docchannel']['use_original_docchannel'] = 0
            elif pred_channel=='公告变更' and docchannel in ['中标信息','废标公告','候选人公示','合同公告']: #中标类的变更还是中标类公告
                channel_dic['docchannel']['docchannel'] = docchannel
                channel_dic['docchannel']['use_original_docchannel'] = 0
            elif docchannel=='公告变更' and pred_channel in ['中标信息','废标公告','候选人公示','合同公告']:
                channel_dic['docchannel']['docchannel'] = pred_channel
                channel_dic['docchannel']['use_original_docchannel'] = 0
            elif docchannel in ['中标信息','候选人公示'] and pred_channel in ['中标信息','候选人公示']:
                if re.search('候选人(变更)?公[告示]|评标(结果)?(公[告示]|报告)|评审结果', title):
                    channel_dic['docchannel']['docchannel'] = '候选人公示'
                    channel_dic['docchannel']['use_original_docchannel'] = 0
                else:
                    if original_docchannel in [101,119]:
                        channel_dic['docchannel']['docchannel'] = class_dict.get(original_docchannel, '原始类别')
                        channel_dic['docchannel']['use_original_docchannel'] = 1
                    else:
                        channel_dic['docchannel']['docchannel'] = pred_channel
                        channel_dic['docchannel']['use_original_docchannel'] = 0
            elif docchannel in ['中标信息','候选人公示'] and pred_channel=='开标记录':
                re_text_len = max(500,len(text)//3)
                re_text = text[:re_text_len]
                if re.search('开标记录|截标信息|开标安排|开标数据表|开标信息|开标情况|开标一览表|开标结果',re_text):
                    channel_dic['docchannel']['docchannel'] = '开标记录'
                    channel_dic['docchannel']['use_original_docchannel'] = 0
            elif pred_channel=='招标答疑' and docchannel in ['招标预告','招标公告']: #招标答疑公告修正
                if re.search("招标答疑|澄清.?修改|答疑.?澄清|澄清.?答疑|(澄清|答疑)(公告|公示)",title+text[:100]):
                    channel_dic['docchannel']['docchannel'] = '招标答疑'
                    channel_dic['docchannel']['use_original_docchannel'] = 0
            elif docchannel=='招标答疑' and pred_channel in ['招标预告','招标公告']:
                if re.search("招标答疑|澄清.?修改|答疑.?澄清|澄清.?答疑|(澄清|答疑)(公告|公示)",title+text[:100]):
                    channel_dic['docchannel']['docchannel'] = '招标答疑'
                    channel_dic['docchannel']['use_original_docchannel'] = 0

            # 以上规则都没用上，使用和original_docchannel相同的预测结果
            if 'use_original_docchannel' not in channel_dic['docchannel']:
                if docchannel==class_dict.get(original_docchannel, '原始类别') or pred_channel==class_dict.get(original_docchannel, '原始类别'):
                    # 其中一个结果和original_docchannel相同
                    channel_dic['docchannel']['docchannel'] = class_dict.get(original_docchannel, '原始类别')
                    channel_dic['docchannel']['use_original_docchannel'] = 0

            if 'use_original_docchannel' not in channel_dic['docchannel']:
                original_type = class_dict.get(original_docchannel, '原始类别')
                if pred_channel in tenderee_type and docchannel in tenderee_type and original_type not in tenderee_type:
                    # pred_channel和docchannel都是同一（招标/中标）类型时，original_docchannel不一致时不使用原网类型
                    channel_dic['docchannel']['use_original_docchannel'] = 0
                elif pred_channel in win_type and docchannel in win_type and original_type not in win_type:
                    # pred_channel和docchannel都是同一（招标/中标）类型时，original_docchannel不一致时不使用原网类型
                    channel_dic['docchannel']['use_original_docchannel'] = 0
                else:
                    channel_dic = {'docchannel': {'doctype': '采招数据',
                                                  'docchannel': original_type,
                                                  'life_docchannel': original_type}}
                    channel_dic['docchannel']['use_original_docchannel'] = 1

        return channel_dic


    article = list_articles[0]
    title = article.title
    text = article.content

    doctype = channel_dic['docchannel']['doctype']
    docchannel = channel_dic['docchannel']['docchannel']
    # print('doctype',doctype,'docchannel',docchannel,'original_docchannel',original_docchannel)
    compare_type = ['公告变更','招标公告','中标信息','招标预告','招标答疑','资审结果','采购意向','废标公告','候选人公示',
                      '合同公告','开标记录','验收合同']
    prem_json = json.dumps(prem, ensure_ascii=False)
    contain_winner = is_contain_winner(prem_json)
    # 仅比较部分数据
    special_pattern = re.compile("单一来源|直接采购|单源直采|直采")
    if doctype=='采招数据' and docchannel in compare_type:
        if not re.search(special_pattern,title) and not re.search(special_pattern,text[:100]):
            pred = channel_predict(title, text)
            # print(text[:2000], '\n pred_res1', pred)
            if pred is not None and original_docchannel: # 无original_docchannel时不进行对比校正
                channel_dic = merge_rule(title,text,docchannel,pred,channel_dic,original_docchannel)
    elif doctype=='采招数据' and docchannel=="":
        if is_finnal:
            pred = channel_predict(title, text,min_text_len=100)
        else:
            pred = channel_predict(title, text)
        # print(text[:2000], '\n pred_res2', pred)
        if pred is not None:
            pred = class_dict[pred]
            channel_dic['docchannel']['docchannel'] = pred
            channel_dic['docchannel']['life_docchannel'] = pred
            channel_dic['docchannel']['use_original_docchannel'] = 0
    # print('channel_dic1',channel_dic)
    if is_finnal:
        # 公告分类后验规则
        if doctype=="采招数据":
            if contain_winner and original_docchannel in [101,119,120] and \
                    channel_dic['docchannel']['docchannel'] in ['招标公告','招标预告','采购意向']:
                if class_dict.get(original_docchannel):
                    channel_dic['docchannel']['docchannel'] = class_dict.get(original_docchannel)
                    channel_dic['docchannel']['use_original_docchannel'] = 1
            elif contain_winner and docchannel in ['中标信息','候选人公示','合同公告'] and \
                    channel_dic['docchannel']['docchannel'] in ['招标公告','招标预告','采购意向']:
                channel_dic['docchannel']['docchannel'] = docchannel
                channel_dic['docchannel']['use_original_docchannel'] = 0
            elif not contain_winner and ((channel_dic['docchannel'].get('docchannel')=='' and original_docchannel in [101,119]) or
                channel_dic['docchannel'].get('docchannel') in ['中标信息','候选人公示']) and re.search("开标.?记录|开标.?安排|开标.?时间|开标.?(地址|地点)|开标一览表|开标日程|评审专家公示|评委名单公示|开标数据表",title+text[:150]) and len(text)<=300:
                # 识别公告内容较短的开标安排(时间、地点)，例：798134361,790800148
                channel_dic['docchannel']['docchannel'] = "开标记录"
                channel_dic['docchannel']['use_original_docchannel'] = 0
            elif channel_dic['docchannel']['docchannel'] in ['招标公告','招标预告'] and re.search("第[1-9一二三]次(变更|更正|更改|修改)|((变更|更正|更改|修改)(事项)?|延期|暂停)(招标|采购)?的?(公告|公示|通知)|变更$|更正$",title):
                # 对标题关键词明显的招标公告进行修正
                channel_dic['docchannel']['docchannel'] = "公告变更"
                channel_dic['docchannel']['use_original_docchannel'] = 0
            # elif not contain_winner and original_docchannel in [52] and channel_dic['docchannel'].get('docchannel') in ['招标公告'] and \
            #         re.search("开标.?记录|开标.?安排|开标.?时间|开标.?(地址|地点)|开标一览表|开标日程|评审专家公示|评委名单公示|开标数据表",title+text[:150]) and len(text)<=300:
            #     # 识别公告内容较短的开标安排(时间、地点)，例：799590061
            #     channel_dic['docchannel']['docchannel'] = "开标记录"
            #     channel_dic['docchannel']['use_original_docchannel'] = 0
            # elif not contain_winner and original_docchannel in [52,102,114] and \
            #         channel_dic['docchannel']['docchannel'] in ['中标信息','合同公告']:
            #     channel_dic['docchannel']['docchannel'] = class_dict.get(original_docchannel)
            #     channel_dic['docchannel']['use_original_docchannel'] = 0
            # elif not contain_winner and docchannel in ['招标公告','招标预告','采购意向'] and \
            #         channel_dic['docchannel']['docchannel'] in ['中标信息','合同公告']:
            #     channel_dic['docchannel']['docchannel'] = docchannel

    # '招标预告'类 规则纠正，规则排除部分站源
    if channel_dic['docchannel']['doctype']=='采招数据' and channel_dic['docchannel']['docchannel'] in ["招标公告","公告变更"] and web_source_no not in ['DX000027-1']:
        if "##attachment##" in text:
            main_text, attachment_text = text.split("##attachment##", maxsplit=1)
        else:
            main_text = text
        main_text = text_process(main_text)
        # if re.search("采购实施月份|采购月份|预计(招标|采购|发标|发包)(时间|月份)|招标公告预计发布时间",main_text[:max(500,len(main_text)//2)]):
        if re.search("采购实施月份|采购月份|(计划|预计|预期)(招标|采购|发标|发包)(时间|月份)|(招标公告|资格预审公告|招标公告[（\(]资格预审公告[）\)])预计发布时间|预计(招标公告|资格预审公告|招标公告[（\(]资格预审公告[）\)])发布时间",main_text):
            front_text_len = len(main_text) // 3 if len(main_text) > 300 else 100
            front_text = main_text[:front_text_len]
            if re.search("意向|意愿",title) or re.search("意向|意愿",front_text):
                channel_dic['docchannel']['docchannel'] = "采购意向"
            else:
                channel_dic['docchannel']['docchannel'] = "招标预告"
            channel_dic['docchannel']['use_original_docchannel'] = 0
    # '招标预告'类规则纠正，有开标/截标时间的改为'招标公告'
    if channel_dic['docchannel']['doctype']=='采招数据' and channel_dic['docchannel']['docchannel']=="招标预告":
        time_bidopen = prem.get("time_bidopen","")
        time_bidclose = prem.get("time_bidclose","")
        time_getFileEnd = prem.get("time_getFileEnd","")
        time_registrationEnd = prem.get("time_registrationEnd","")
        for _time in [time_bidopen,time_bidclose,time_getFileEnd,time_registrationEnd]:
            if _time and page_time and _time[:10]>=page_time:
                # print("change docchannel by time",_time,page_time)
                channel_dic['docchannel']['docchannel'] = "招标公告"
                channel_dic['docchannel']['use_original_docchannel'] = 0
                break

    # print("original channel_dic",channel_dic)
    # docchannel预测为空时，补充原网类别
    if channel_dic['docchannel'].get('docchannel') == '' and is_finnal is True:
        if channel_dic['docchannel'].get('doctype') in ("采招数据","拍卖出让","产权交易","土地矿产"):
            if channel_dic['docchannel'].get('life_docchannel') and channel_dic['docchannel']['life_docchannel'] not in ("拍卖出让","产权交易","土地矿产","原始类别"):
                channel_dic['docchannel']['docchannel'] = channel_dic['docchannel']['life_docchannel']
                channel_dic['docchannel']['use_original_docchannel'] = 1
    # print('channel_dic2',channel_dic)
    return channel_dic


if __name__ == '__main__':
    title = '关于【2024年四好农村路大中村药红路、空坦路延伸段设计服务】无效项目的公示'
    text = '''关于【2024年四好农村路大中村药红路、空坦路延伸段设计服务】无效项目的公示 点击查看招标公告 关于【2024年四好农村路大中村药红路、空坦路延伸段设计服务】无效项目的公示 项目名称 2024年四好农村路大中村药红路、空坦路延伸段设计服务， 采购人 重庆市巴南区人民政府莲花街道办事处， 选取方式 直接选取， 是否重新发布招标公告 是 ，无效类型 项目取消， 无效原因 资质设置错误，附件已盖章上传 ，无效时间 2024-10-21 ，公示附件 大中村设计变更.jpg'''
    pred_class = channel_predict(title,text)
    print(pred_class)
    # pred_class2 = channel_predict(title,text)
    # print(pred_class2)

    pass