TABLE_DETECTION/table_line.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 9 23:11:51 2020
table line detect
@author: chineseocr
"""
import copy

import tensorflow as tf
import tensorflow.keras as K
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, concatenate, Conv2D, MaxPooling2D, BatchNormalization, UpSampling2D
from tensorflow.keras.layers import LeakyReLU
from config import tableModeLinePath
from utils import letterbox_image, get_table_line, adjust_lines, line_to_line, draw_boxes
import numpy as np
import cv2
import time
from utils import draw_lines


def dice_coef(y_true, y_pred, smooth=1e-5):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)


def dice_coef_loss():
    def dice_coef_loss_fixed(y_true, y_pred):
        return -dice_coef(y_true, y_pred)
    return dice_coef_loss_fixed


def focal_loss(gamma=3., alpha=.5):
    # 3 0.85 2000e          acc-0.6 p-0.99 r-0.99 val_acc-0.56 val_p-0.86 val_r-0.95
    # 2 0.85 double_gpu     acc-
    # 3 0.25 gpu 50e        acc-0.5 p-0.99 r-0.99 val_acc-0.45 val_p-0.96 val_r-0.88
    # 2 0.25 gpu            acc-
    # 3 0.5  double_gpu     acc-0.6 p-0.99 r-0.99 val_acc-0.60 val_p-0.93 val_r-0.93
    def focal_loss_fixed(y_true, y_pred):
        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
        return -K.backend.sum(alpha * K.backend.pow(1. - pt_1, gamma)
                              * K.backend.log(K.backend.epsilon()+pt_1))-K.backend.sum((1-alpha) * K.backend.pow( pt_0, gamma) * K.backend.log(1. - pt_0 + K.backend.epsilon()))
    return focal_loss_fixed


def table_net_large(input_shape=(1152, 896, 3), num_classes=1):
    inputs = Input(shape=input_shape)
    # 512
    use_bias = False
    down0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(inputs)
    down0a = BatchNormalization()(down0a)
    down0a = LeakyReLU(alpha=0.1)(down0a)
    down0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(down0a)
    down0a = BatchNormalization()(down0a)
    down0a = LeakyReLU(alpha=0.1)(down0a)
    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)

    # 256
    down0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(down0a_pool)
    down0 = BatchNormalization()(down0)
    
    down0 = LeakyReLU(alpha=0.1)(down0)
    down0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(down0)
    down0 = BatchNormalization()(down0)
    down0 = LeakyReLU(alpha=0.1)(down0)
    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
    # 128

    down1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(down0_pool)
    down1 = BatchNormalization()(down1)
    down1 = LeakyReLU(alpha=0.1)(down1)
    down1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(down1)
    down1 = BatchNormalization()(down1)
    down1 = LeakyReLU(alpha=0.1)(down1)
    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
    # 64

    down2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(down1_pool)
    down2 = BatchNormalization()(down2)
    down2 = LeakyReLU(alpha=0.1)(down2)
    down2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(down2)
    down2 = BatchNormalization()(down2)
    down2 = LeakyReLU(alpha=0.1)(down2)
    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
    # 32

    down3 = Conv2D(256, (3, 3), padding='same', use_bias=use_bias)(down2_pool)
    down3 = BatchNormalization()(down3)
    down3 = LeakyReLU(alpha=0.1)(down3)
    down3 = Conv2D(256, (3, 3), padding='same', use_bias=use_bias)(down3)
    down3 = BatchNormalization()(down3)
    down3 = LeakyReLU(alpha=0.1)(down3)
    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
    # 16

    down4 = Conv2D(512, (3, 3), padding='same', use_bias=use_bias)(down3_pool)
    down4 = BatchNormalization()(down4)
    down4 = LeakyReLU(alpha=0.1)(down4)
    down4 = Conv2D(512, (3, 3), padding='same', use_bias=use_bias)(down4)
    down4 = BatchNormalization()(down4)
    down4 = LeakyReLU(alpha=0.1)(down4)
    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)
    # 8

    center = Conv2D(1024, (3, 3), padding='same', use_bias=use_bias)(down4_pool)
    center = BatchNormalization()(center)
    center = LeakyReLU(alpha=0.1)(center)
    center = Conv2D(1024, (3, 3), padding='same', use_bias=use_bias)(center)
    center = BatchNormalization()(center)
    center = LeakyReLU(alpha=0.1)(center)
    # center

    up4 = UpSampling2D((2, 2))(center)
    up4 = concatenate([down4, up4], axis=3)
    up4 = Conv2D(512, (3, 3), padding='same', use_bias=use_bias)(up4)
    up4 = BatchNormalization()(up4)
    up4 = LeakyReLU(alpha=0.1)(up4)
    up4 = Conv2D(512, (3, 3), padding='same', use_bias=use_bias)(up4)
    up4 = BatchNormalization()(up4)
    up4 = LeakyReLU(alpha=0.1)(up4)
    up4 = Conv2D(512, (3, 3), padding='same', use_bias=use_bias)(up4)
    up4 = BatchNormalization()(up4)
    up4 = LeakyReLU(alpha=0.1)(up4)
    # 16

    up3 = UpSampling2D((2, 2))(up4)
    up3 = concatenate([down3, up3], axis=3)
    up3 = Conv2D(256, (3, 3), padding='same', use_bias=use_bias)(up3)
    up3 = BatchNormalization()(up3)
    up3 = LeakyReLU(alpha=0.1)(up3)
    up3 = Conv2D(256, (3, 3), padding='same', use_bias=use_bias)(up3)
    up3 = BatchNormalization()(up3)
    up3 = LeakyReLU(alpha=0.1)(up3)
    up3 = Conv2D(256, (3, 3), padding='same', use_bias=use_bias)(up3)
    up3 = BatchNormalization()(up3)
    up3 = LeakyReLU(alpha=0.1)(up3)
    # 32

    up2 = UpSampling2D((2, 2))(up3)
    up2 = concatenate([down2, up2], axis=3)
    up2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(up2)
    up2 = BatchNormalization()(up2)
    up2 = LeakyReLU(alpha=0.1)(up2)
    up2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(up2)
    up2 = BatchNormalization()(up2)
    up2 = LeakyReLU(alpha=0.1)(up2)
    up2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(up2)
    up2 = BatchNormalization()(up2)
    up2 = LeakyReLU(alpha=0.1)(up2)
    # 64

    up1 = UpSampling2D((2, 2))(up2)
    up1 = concatenate([down1, up1], axis=3)
    up1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up1)
    up1 = BatchNormalization()(up1)
    up1 = LeakyReLU(alpha=0.1)(up1)
    up1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up1)
    up1 = BatchNormalization()(up1)
    up1 = LeakyReLU(alpha=0.1)(up1)
    up1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up1)
    up1 = BatchNormalization()(up1)
    up1 = LeakyReLU(alpha=0.1)(up1)
    # 128

    up0 = UpSampling2D((2, 2))(up1)
    up0 = concatenate([down0, up0], axis=3)
    up0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up0)
    up0 = BatchNormalization()(up0)
    up0 = LeakyReLU(alpha=0.1)(up0)
    up0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up0)
    up0 = BatchNormalization()(up0)
    up0 = LeakyReLU(alpha=0.1)(up0)
    up0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up0)
    up0 = BatchNormalization()(up0)
    up0 = LeakyReLU(alpha=0.1)(up0)
    # 256

    up0a = UpSampling2D((2, 2))(up0)
    up0a = concatenate([down0a, up0a], axis=3)
    up0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up0a)
    up0a = BatchNormalization()(up0a)
    up0a = LeakyReLU(alpha=0.1)(up0a)
    up0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up0a)
    up0a = BatchNormalization()(up0a)
    up0a = LeakyReLU(alpha=0.1)(up0a)
    up0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up0a)
    up0a = BatchNormalization()(up0a)
    up0a = LeakyReLU(alpha=0.1)(up0a)
    # 512
    classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0a)
    model = Model(inputs=inputs, outputs=classify)
    return model


def table_net(input_shape=(1152, 896, 3), num_classes=1):
    inputs = Input(shape=input_shape)
    # 512
    use_bias = False
    down0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(inputs)
    down0a = BatchNormalization()(down0a)
    down0a = LeakyReLU(alpha=0.1)(down0a)
    down0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(down0a)
    down0a = BatchNormalization()(down0a)
    down0a = LeakyReLU(alpha=0.1)(down0a)
    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)

    # 256
    down0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(down0a_pool)
    down0 = BatchNormalization()(down0)

    down0 = LeakyReLU(alpha=0.1)(down0)
    down0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(down0)
    down0 = BatchNormalization()(down0)
    down0 = LeakyReLU(alpha=0.1)(down0)
    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
    # 128

    down1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(down0_pool)
    down1 = BatchNormalization()(down1)
    down1 = LeakyReLU(alpha=0.1)(down1)
    down1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(down1)
    down1 = BatchNormalization()(down1)
    down1 = LeakyReLU(alpha=0.1)(down1)
    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
    # 64

    down2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(down1_pool)
    down2 = BatchNormalization()(down2)
    down2 = LeakyReLU(alpha=0.1)(down2)
    down2 = Conv2D(128, (3, 3), padding='same', use_bias=use_bias)(down2)
    down2 = BatchNormalization()(down2)
    down2 = LeakyReLU(alpha=0.1)(down2)
    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
    # 32

    up1 = UpSampling2D((2, 2))(down2)
    up1 = concatenate([down1, up1], axis=3)
    up1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up1)
    up1 = BatchNormalization()(up1)
    up1 = LeakyReLU(alpha=0.1)(up1)
    up1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up1)
    up1 = BatchNormalization()(up1)
    up1 = LeakyReLU(alpha=0.1)(up1)
    up1 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up1)
    up1 = BatchNormalization()(up1)
    up1 = LeakyReLU(alpha=0.1)(up1)
    # 128

    up0 = UpSampling2D((2, 2))(up1)
    up0 = concatenate([down0, up0], axis=3)
    up0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up0)
    up0 = BatchNormalization()(up0)
    up0 = LeakyReLU(alpha=0.1)(up0)
    up0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up0)
    up0 = BatchNormalization()(up0)
    up0 = LeakyReLU(alpha=0.1)(up0)
    up0 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up0)
    up0 = BatchNormalization()(up0)
    up0 = LeakyReLU(alpha=0.1)(up0)
    # 256

    up0a = UpSampling2D((2, 2))(up0)
    up0a = concatenate([down0a, up0a], axis=3)
    up0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up0a)
    up0a = BatchNormalization()(up0a)
    up0a = LeakyReLU(alpha=0.1)(up0a)
    up0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up0a)
    up0a = BatchNormalization()(up0a)
    up0a = LeakyReLU(alpha=0.1)(up0a)
    up0a = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up0a)
    up0a = BatchNormalization()(up0a)
    up0a = LeakyReLU(alpha=0.1)(up0a)
    # 512
    classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0a)
    model = Model(inputs=inputs, outputs=classify)
    return model


model = table_net((None, None, 3), 2)


def table_line(img, model, size=(512, 1024), hprob=0.5, vprob=0.5, row=50, col=30, alph=15):
    sizew, sizeh = size
    # [..., ::-1] 最后一维内部反向输出
    # inputBlob, fx, fy = letterbox_image(img[..., ::-1], (sizew, sizeh))
    # pred = model.predict(np.array([np.array(inputBlob)]))
    # pred = model.predict(np.array([np.array(inputBlob)/255.0]))
    img_new = cv2.resize(img, (sizew, sizeh), interpolation=cv2.INTER_AREA)

    pred = model.predict(np.array([img_new]))
    pred = pred[0]

    vpred = pred[..., 1] > vprob   # 横线
    hpred = pred[..., 0] > hprob   # 竖线
    vpred = vpred.astype(int)
    hpred = hpred.astype(int)

    # print("vpred shape", vpred)
    # print("hpred shape", hpred)

    colboxes = get_table_line(vpred, axis=1, lineW=col)
    rowboxes = get_table_line(hpred, axis=0, lineW=row)

    # if len(rowboxes) > 0:
    #     rowboxes = np.array(rowboxes)
    #     rowboxes[:, [0, 2]] = rowboxes[:, [0, 2]]/fx
    #     rowboxes[:, [1, 3]] = rowboxes[:, [1, 3]]/fy
    #     rowboxes = rowboxes.tolist()
    # if len(colboxes) > 0:
    #     colboxes = np.array(colboxes)
    #     colboxes[:, [0, 2]] = colboxes[:, [0, 2]]/fx
    #     colboxes[:, [1, 3]] = colboxes[:, [1, 3]]/fy
    #     colboxes = colboxes.tolist()

    # nrow = len(rowboxes)
    # ncol = len(colboxes)
    # for i in range(nrow):
    #     for j in range(ncol):
    #         rowboxes[i] = line_to_line(rowboxes[i], colboxes[j], 10)
    #         colboxes[j] = line_to_line(colboxes[j], rowboxes[i], 10)

    # 删掉贴着边框的line
    temp_list = []
    threshold = 5
    for line in rowboxes:
        if line[1]-0 <= threshold or size[1]-line[1] <= threshold:
            continue
        temp_list.append(line)
    rowboxes = temp_list

    temp_list = []
    for line in colboxes:
        if line[0]-0 <= threshold or size[0]-line[0] <= threshold:
            continue
        temp_list.append(line)
    colboxes = temp_list

    return rowboxes, colboxes, img_new


def get_outline(points, image_np):
    # 取出x, y的最大值最小值
    x_min = points[0][0]
    x_max = points[-1][0]
    points.sort(key=lambda x: (x[1], x[0]))
    y_min = points[0][1]
    y_max = points[-1][1]

    # 创建空图
    # outline_img = np.zeros(image_size, np.uint8)
    outline_img = np.copy(image_np)
    cv2.rectangle(outline_img, (x_min-5, y_min-5), (x_max+5, y_max+5), (0, 0, 0), 2)
    # cv2.imshow("outline_img", outline_img)
    # cv2.waitKey(0)
    return outline_img


def get_split_line(points, col_lines, image_np):
    # print("get_split_line", image_np.shape)
    points.sort(key=lambda x: (x[1], x[0]))
    # 遍历y坐标，并判断y坐标与上一个y坐标是否存在连接线
    i = 0
    split_line_y = []
    for point in points:
        # 从已分开的线下面开始判断
        if split_line_y:
            if point[1] <= split_line_y[-1] + 5:
                last_y = point[1]
                continue
            if last_y <= split_line_y[-1] + 5:
                last_y = point[1]
                continue

        if i == 0:
            last_y = point[1]
            i += 1
            continue

        current_line = (last_y, point[1])
        split_flag = 1
        for col in col_lines:
            # 只要找到一条col包含就不是分割线
            if current_line[0] >= col[1]-3 and current_line[1] <= col[3]+3:
                split_flag = 0
                # print("img", img.shape)
                # print("col", col)
                # print("current_line", current_line)
                break

        if split_flag:
            split_line_y.append(current_line[0]+5)
            split_line_y.append(current_line[1]-5)

        last_y = point[1]

    # 加上收尾分割线
    points.sort(key=lambda x: (x[1], x[0]))
    y_min = points[0][1]
    y_max = points[-1][1]
    # print("加上收尾分割线", y_min, y_max)
    if y_min-5 < 0:
        split_line_y.append(0)
    else:
        split_line_y.append(y_min-5)
    if y_max+5 > image_np.shape[0]:
        split_line_y.append(image_np.shape[0])
    else:
        split_line_y.append(y_max+5)
    split_line_y = list(set(split_line_y))

    # 剔除两条相隔太近分割线
    temp_split_line_y = []
    split_line_y.sort(key=lambda x: x)
    last_y = -20
    for y in split_line_y:
        # print(y)
        if y - last_y >= 20:
            # print(y, last_y)
            temp_split_line_y.append(y)
            last_y = y
    split_line_y = temp_split_line_y

    # print("split_line_y", split_line_y)

    # 生成分割线
    split_line = []
    last_y = 0
    for y in split_line_y:
        # if y - last_y <= 15:
        #     continue
        split_line.append([(0, y), (image_np.shape[1], y)])
        last_y = y
    split_line.append([(0, 0), (image_np.shape[1], 0)])
    split_line.append([(0, image_np.shape[0]), (image_np.shape[1], image_np.shape[0])])
    split_line.sort(key=lambda x: x[0][1])
    # print("split_line", split_line)

    # 画图画线
    # split_line_img = np.copy(image_np)
    # for y in split_line_y:
    #     cv2.line(split_line_img, (0, y), (image_np.shape[1], y), (0, 0, 0), 1)
    # cv2.imshow("split_line_img", split_line_img)
    # cv2.waitKey(0)

    return split_line, split_line_y


def get_points(row_lines, col_lines, image_size):
    # 创建空图
    row_img = np.zeros(image_size, np.uint8)
    col_img = np.zeros(image_size, np.uint8)

    # 画线
    thresh = 3
    for row in row_lines:
        cv2.line(row_img, (int(row[0]-thresh), int(row[1])), (int(row[2]+thresh), int(row[3])), (255, 255, 255), 1)
    for col in col_lines:
        cv2.line(col_img, (int(col[0]), int(col[1]-thresh)), (int(col[2]), int(col[3]+thresh)), (255, 255, 255), 1)

    # 求出交点
    point_img = np.bitwise_and(row_img, col_img)
    # cv2.imshow("point_img", np.bitwise_not(point_img))
    # cv2.waitKey(0)

    # 识别黑白图中的白色交叉点，将横纵坐标取出
    ys, xs = np.where(point_img > 0)
    points = []
    for i in range(len(xs)):
        points.append((xs[i], ys[i]))
    points.sort(key=lambda x: (x[0], x[1]))
    return points


def get_minAreaRect(image):
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    gray = cv2.bitwise_not(gray)
    thresh = cv2.threshold(gray, 0, 255,
                           cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
    coords = np.column_stack(np.where(thresh > 0))
    return cv2.minAreaRect(coords)


def get_contours(image):
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    ret, binary = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)

    contours, hierarchy = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(image, contours, -1, (0, 0, 255), 3)

    cv2.imshow("get contours", image)
    cv2.waitKey(0)


def fix_outline(image, row_lines, col_lines, points, split_y):
    # 分割线纵坐标
    # print("len(split_y)", split_y)
    if len(split_y) < 2:
        # split_y = [2000., 2000., 2000., 2000.]
        return [], [], []
    elif len(split_y) == 2:
        split_y = [2000., 2000., 2000., 2000.]

    split_y.sort(key=lambda x: x)
    split_y = split_y[1:-1]
    new_split_y = []
    for i in range(1, len(split_y), 2):
        new_split_y.append(int((split_y[i]+split_y[i-1])/2))
    # print("len(new_split_y)", len(new_split_y))

    # 预测线根据分割线纵坐标分为多个分割区域
    row_lines.sort(key=lambda x: (x[3], x[2], x[1], x[0]))
    col_lines.sort(key=lambda x: (x[3], x[2], x[1], x[0]))

    row_count = 0
    col_count = 0
    split_row_list = []
    split_col_list = []
    for y in new_split_y:
        row_lines = row_lines[row_count:]
        col_lines = col_lines[col_count:]
        row_count = 0
        col_count = 0
        for row in row_lines:
            if row[3] <= y:
                row_count += 1
            else:
                split_row_list.append(row_lines[:row_count])
                break
            if row_count == len(row_lines):
                split_row_list.append(row_lines[:row_count])
                break
        for col in col_lines:
            if col[3] <= y:
                col_count += 1
            else:
                split_col_list.append(col_lines[:col_count])
                break
            if col_count == len(col_lines):
                split_col_list.append(col_lines[:col_count])
                break
    if row_count < len(row_lines) - 1:
        row_lines = row_lines[row_count:]
        split_row_list.append(row_lines)
    if col_count < len(col_lines) - 1:
        col_lines = col_lines[col_count:]
        split_col_list.append(col_lines)
    # print("len(split_row_list)", len(split_row_list))
    # print("len(split_col_list)", len(split_col_list))

    # 预测线取上下左右4个边(会有超出表格部分) [(), ()]
    area_row_line = []
    area_col_line = []
    for area in split_row_list:
        if not area:
            continue
        area.sort(key=lambda x: (x[1], x[0]))
        # print(area)
        up_line = area[0]
        bottom_line = area[-1]
        area_row_line.append([up_line, bottom_line])
    for area in split_col_list:
        if not area:
            continue
        area.sort(key=lambda x: x[0])
        left_line = area[0]
        right_line = area[-1]
        area_col_line.append([left_line, right_line])

    # 线交点根据分割线纵坐标分为多个分割区域
    points.sort(key=lambda x: (x[1], x[0]))
    point_count = 0
    split_point_list = []
    for y in new_split_y:
        points = points[point_count:len(points)]
        point_count = 0
        for point in points:
            if point[1] <= y:
                point_count += 1
            else:
                split_point_list.append(points[:point_count])
                break
            if point_count == len(points):
                split_point_list.append(points[:point_count])
                break
    if point_count < len(points) - 1:
        points = points[point_count:len(points)]
        split_point_list.append(points)
    # print("len(split_point_list)", len(split_point_list))

    # 取每个分割区域的4条线(无超出表格部分)
    area_row_line2 = []
    area_col_line2 = []
    for area in split_point_list:
        if not area:
            continue
        area.sort(key=lambda x: (x[0], x[1]))
        left_up = area[0]
        right_bottom = area[-1]

        up_line = [left_up[0], left_up[1], right_bottom[0], left_up[1]]
        bottom_line = [left_up[0], right_bottom[1], right_bottom[0], right_bottom[1]]
        left_line = [left_up[0], left_up[1], left_up[0], right_bottom[1]]
        right_line = [right_bottom[0], left_up[1], right_bottom[0], right_bottom[1]]
        area_row_line2.append([up_line, bottom_line])
        area_col_line2.append([left_line, right_line])

    # 判断超出部分的长度，超出一定长度就补线
    new_row_lines = []
    new_col_lines = []
    longer_row_lines = []
    longer_col_lines = []
    all_longer_row_lines = []
    all_longer_col_lines = []
    # print(len(area_col_line))
    # print(len(area_col_line2))
    for i in range(len(area_row_line)):
        up_line = area_row_line[i][0]
        up_line2 = area_row_line2[i][0]
        bottom_line = area_row_line[i][1]
        bottom_line2 = area_row_line2[i][1]
        left_line = area_col_line[i][0]
        left_line2 = area_col_line2[i][0]
        right_line = area_col_line[i][1]
        right_line2 = area_col_line2[i][1]

        # 补左右两条竖线超出来的线的row
        # print("left_line2[1] - left_line[1]", left_line2[1] - left_line[1])
        if left_line2[1] - left_line[1] >= 10 and right_line2[1] - right_line[1] >= 10:
            if left_line2[1] - left_line[1] >= right_line2[1] - right_line[1]:
                new_row_lines.append([left_line[0], left_line[1], right_line[0], left_line[1]])
                new_col_y = left_line[1]
                # 补了row，要将其他短的col连到row上
                for col in split_col_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= col[1] <= col[3] <= bottom_line2[1]+3:
                        longer_col_lines.append([col[0], min([new_col_y, col[1]]), col[2], col[3]])
            else:
                new_row_lines.append([left_line[0], right_line[1], right_line[0], right_line[1]])
                new_col_y = right_line[1]
                # 补了row，要将其他短的col连到row上
                for col in split_col_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= col[1] <= col[3] <= bottom_line2[1]+3:
                        longer_col_lines.append([col[0], min([new_col_y, col[1]]), col[2], col[3]])
        # print("left_line[3] - left_line2[3]", left_line[3] - left_line2[3])
        if left_line[3] - left_line2[3] >= 10 and right_line[3] - right_line2[3] >= 10:
            if left_line[3] - left_line2[3] >= right_line[3] - right_line2[3]:
                new_row_lines.append([left_line[2], left_line[3], right_line[2], left_line[3]])
                new_col_y = left_line[3]
                # 补了row，要将其他短的col连到row上
                for col in split_col_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= col[1] <= col[3] <= bottom_line2[1]+3:
                        longer_col_lines.append([col[0], col[1], col[2], max([new_col_y, col[3]])])
            else:
                new_row_lines.append([left_line[2], right_line[3], right_line[2], right_line[3]])
                new_col_y = right_line[3]
                # 补了row，要将其他短的col连到row上
                for col in split_col_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= col[1] <= col[3] <= bottom_line2[1]+3:
                        longer_col_lines.append([col[0], col[1], col[2], max([new_col_y, col[3]])])

        # 补上下两条横线超出来的线的col
        if up_line2[0] - up_line[0] >= 10 and bottom_line2[0] - bottom_line[0] >= 10:
            if up_line2[0] - up_line[0] >= bottom_line2[0] - bottom_line[0]:
                new_col_lines.append([up_line[0], up_line[1], up_line[0], bottom_line[1]])
                new_row_x = up_line[0]
                # 补了col，要将其他短的row连到col上
                for row in split_row_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= row[1] <= bottom_line2[1]+3:
                        longer_row_lines.append([min([new_row_x, row[0]]), row[1], row[2], row[3]])
                # new_row_lines.append([bottom_line[0], up_line[1], bottom_line[2], bottom_line[3]])
            else:
                new_col_lines.append([bottom_line[0], up_line[1], bottom_line[0], bottom_line[1]])
                new_row_x = bottom_line[0]
                # 补了col，要将其他短的row连到col上
                for row in split_row_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= row[1] <= bottom_line2[1]+3:
                        longer_row_lines.append([min([new_row_x, row[0]]), row[1], row[2], row[3]])
                # new_row_lines.append([bottom_line[0], up_line[1], up_line[2], up_line[3]])
        if up_line[2] - up_line2[2] >= 10 and bottom_line[2] - bottom_line2[2] >= 10:
            if up_line[2] - up_line2[2] >= bottom_line[2] - bottom_line2[2]:
                new_col_lines.append([up_line[2], up_line[3], up_line[2], bottom_line[3]])
                new_row_x = up_line[2]
                # 补了col，要将其他短的row连到col上
                for row in split_row_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= row[1] <= bottom_line2[1]+3:
                        print([new_row_x, row[2]])
                        longer_row_lines.append([row[0], row[1], max([new_row_x, row[2]]), row[3]])
            else:
                new_col_lines.append([bottom_line[2], up_line[3], bottom_line[2], bottom_line[3]])
                new_row_x = bottom_line[2]
                # 补了col，要将其他短的row连到col上
                for row in split_row_list[i]:
                    # 需判断该线在这个区域中
                    if up_line2[1]-3 <= row[1] <= bottom_line2[1]+3:
                        print([new_row_x, row[2]])
                        longer_row_lines.append([row[0], row[1], max([new_row_x, row[2]]), row[3]])

        if longer_row_lines:
            all_longer_row_lines += longer_row_lines
        else:
            all_longer_row_lines += split_row_list[i]
        if longer_col_lines:
            all_longer_col_lines += longer_col_lines
        else:
            all_longer_col_lines += split_col_list[i]

        # 删除表格内部的补线
        # temp_list = []
        # for row in new_row_lines:
        #     if up_line[1]-5 <= row[1] <= bottom_line[1]+5:
        #         continue
        #     temp_list.append(row)
        # print("fix_outline", new_row_lines)
        # new_row_lines = temp_list
        # print("fix_outline", new_row_lines)
        # temp_list = []
        # for col in new_col_lines:
        #     if left_line[0]-5 <= col[0] <= right_line[0]+5:
        #         continue
        #     temp_list.append(col)
        #
        # new_col_lines = temp_list
        # print("fix_outline", new_col_lines)
        # print("fix_outline", new_row_lines)

        # 删除重复包含的补线
        # temp_list = []
        # for row in new_row_lines:
        #     if up_line[1]-5 <= row[1] <= bottom_line[1]+5:
        #         continue
        #     temp_list.append(row)
        # new_row_lines = temp_list


    # 展示上下左右边框线
    # for i in range(len(area_row_line)):
    #     print("row1", area_row_line[i])
    #     print("row2", area_row_line2[i])
    #     print("col1", area_col_line[i])
    #     print("col2", area_col_line2[i])
    #     cv2.line(image, (int(area_row_line[i][0][0]), int(area_row_line[i][0][1])),
    #              (int(area_row_line[i][0][2]), int(area_row_line[i][0][3])), (0, 255, 0), 2)
    #     cv2.line(image, (int(area_row_line2[i][1][0]), int(area_row_line2[i][1][1])),
    #              (int(area_row_line2[i][1][2]), int(area_row_line2[i][1][3])), (0, 0, 255), 2)
    #     cv2.imshow("fix_outline", image)
    #     cv2.waitKey(0)
    return new_row_lines, new_col_lines, all_longer_row_lines, all_longer_col_lines


def fix_table(row_point_list, col_point_list, split_y, row_lines, col_lines):
    # 分割线纵坐标
    if len(split_y) < 2:
        return []

    # 获取bbox
    bbox = []
    # 每个点获取与其x最相近和y最相近的点
    for i in range(1, len(split_y)):

        # 循环每行
        for row in row_point_list:
            row.sort(key=lambda x: (x[0], x[1]))
            # 行不在该区域跳过
            if row[0][1] <= split_y[i-1] or row[0][1] >= split_y[i]:
                continue

            # print("len(row)", len(row))
            # print("row", row)

            # 循环行中的点
            for j in range(len(row)):
                if j == len(row) - 1:
                    break
                current_point = row[j]
                next_point_in_row_list = row[j+1:]

                # 循环这一行的下一个点
                for next_point_in_row in next_point_in_row_list:
                    # 是否在这一行点找到，找不到就这一行的下个点
                    not_found = 1
                    # 查询下个点所在列
                    next_col = []
                    for col in col_point_list:
                        col.sort(key=lambda x: (x[1], x[0]))
                        # 列不在该区域跳过
                        if col[0][1] <= split_y[i-1] or col[-1][1] >= split_y[i]:
                            continue
                        if col[0][0]-3 <= next_point_in_row[0] <= col[0][0]+3:
                            next_col = col
                            break

                    # 循环匹配当前点和下一列点
                    next_col.sort(key=lambda x: (x[1], x[0]))
                    for point1 in next_col:
                        # 同一行的就跳过
                        if current_point[1]-3 <= point1[1] <= current_point[1]+3:
                            continue
                        if point1[1] <= current_point[1]-3:
                            continue

                        # 候选bbox
                        candidate_bbox = [current_point[0], current_point[1], point1[0], point1[1]]
                        # print("candidate_bbox", candidate_bbox)

                        # 判断该bbox是否存在，判断bbox的上下两条边是否有包含在row中
                        contain_flag1 = 0
                        contain_flag2 = 0
                        for row1 in row_lines:
                            # 行不在该区域跳过
                            if row1[1] <= split_y[i-1] or row1[1] >= split_y[i]:
                                continue
                            # bbox上边框 y一样
                            if not contain_flag1:
                                if row1[1]-3 <= candidate_bbox[1] <= row1[1]+3:
                                    # 格子里的断开线段
                                    row1_break = (max([row1[0], candidate_bbox[0]]),
                                                  row1[1],
                                                  min([row1[2], candidate_bbox[2]]),
                                                  row1[3])

                                    if row1_break[2] - row1_break[0] >= (candidate_bbox[2] - candidate_bbox[0])/3:
                                        contain_flag1 = 1

                            # bbox下边框 y一样
                            if not contain_flag2:
                                if row1[1]-3 <= candidate_bbox[3] <= row1[1]+3:
                                    # 格子里的断开线段
                                    row1_break = (max([row1[0], candidate_bbox[0]]),
                                                  row1[1],
                                                  min([row1[2], candidate_bbox[2]]),
                                                  row1[3])
                                    if row1_break[2] - row1_break[0] >= (candidate_bbox[2] - candidate_bbox[0])/3:
                                        contain_flag2 = 1

                        # 判断该bbox是否存在，判断bbox的左右两条边是否有包含在col中
                        contain_flag3 = 0
                        contain_flag4 = 0
                        for col1 in col_lines:
                            # 列不在该区域跳过
                            if col1[1] <= split_y[i-1] or col1[3] >= split_y[i]:
                                continue
                            # bbox左边线 x一样
                            if not contain_flag3:
                                if col1[0]-3 <= candidate_bbox[0] <= col1[0]+3:
                                    # 格子里的断开线段
                                    col1_break = (col1[0],
                                                  max([col1[1], candidate_bbox[1]]),
                                                  col1[2],
                                                  min([col1[3], candidate_bbox[3]]))
                                    if col1_break[3] - col1_break[1] >= (candidate_bbox[3] - candidate_bbox[1])/3:
                                        contain_flag3 = 1

                            # bbox右边框 x一样
                            if not contain_flag4:
                                if col1[0]-3 <= candidate_bbox[2] <= col1[0]+3:
                                    # 格子里的断开线段
                                    col1_break = (col1[0],
                                                  max([col1[1], candidate_bbox[1]]),
                                                  col1[2],
                                                  min([col1[3], candidate_bbox[3]]))
                                    if col1_break[3] - col1_break[1] >= (candidate_bbox[3] - candidate_bbox[1])/3:
                                        contain_flag4 = 1

                        # 找到了该bbox，并且是存在的
                        if contain_flag1 and contain_flag2 and contain_flag3 and contain_flag4:
                            bbox.append([(candidate_bbox[0], candidate_bbox[1]),
                                         (candidate_bbox[2], candidate_bbox[3])])
                            not_found = 0
                            break

                    if not not_found:
                        break
    return bbox


def delete_close_points(point_list, row_point_list, col_point_list, threshold=5):
    new_point_list = []
    delete_point_list = []
    point_list.sort(key=lambda x: (x[1], x[0]))
    for i in range(len(point_list)):
        point1 = point_list[i]
        if point1 in delete_point_list:
            continue
        if i == len(point_list) - 1:
            new_point_list.append(point1)
            break
        point2 = point_list[i+1]

        # 判断坐标
        if abs(point1[0] - point2[0]) > threshold or abs(point1[1] - point2[1]) > threshold:
            new_point_list.append(point1)
        else:
            # 看两个点上的相同坐标点哪个多，就保留哪个
            count1 = 0
            count2 = 0
            for col in col_point_list:
                if point1[0] == col[0][0]:
                    count1 += len(col)
                elif point2[0] == col[0][0]:
                    count2 += len(col)
            if count1 >= count2:
                new_point_list.append(point1)
                delete_point_list.append(point2)
            else:
                new_point_list.append(point2)
                delete_point_list.append(point1)

    point_list = new_point_list
    new_point_list = []
    delete_point_list = []
    point_list.sort(key=lambda x: (x[0], x[1]))
    for i in range(len(point_list)):
        point1 = point_list[i]
        if point1 in delete_point_list:
            continue
        if i == len(point_list) - 1:
            new_point_list.append(point1)
            break
        point2 = point_list[i+1]

        # 判断坐标
        if abs(point1[0] - point2[0]) > threshold or abs(point1[1] - point2[1]) > threshold:
            new_point_list.append(point1)
        else:
            count1 = 0
            count2 = 0
            for row in row_point_list:
                if point1[0] == row[0][0]:
                    count1 += len(row)
                elif point2[0] == row[0][0]:
                    count2 += len(row)
            if count1 >= count2:
                new_point_list.append(point1)
                delete_point_list.append(point2)
            else:
                new_point_list.append(point2)
                delete_point_list.append(point1)

    return new_point_list


def get_bbox2(image_np, points):
    # # 坐标点按行分
    # row_point_list = []
    # row_point = []
    # points.sort(key=lambda x: (x[0], x[1]))
    # for p in points:
    #     if len(row_point) == 0:
    #         x = p[0]
    #     if x-5 <= p[0] <= x+5:
    #         row_point.append(p)
    #     else:
    #         row_point_list.append(row_point)
    #         row_point = []
    # # 坐标点按列分
    # col_point_list = []
    # col_point = []
    # points.sort(key=lambda x: (x[1], x[0]))
    # for p in points:
    #     if len(col_point) == 0:
    #         y = p[1]
    #     if y-5 <= p[1] <= y+5:
    #         col_point.append(p)
    #     else:
    #         col_point_list.append(col_point)
    #         col_point = []
    row_point_list = get_points_row(points)
    col_point_list = get_points_col(points)
    print("len(points)", len(points))
    for point in points:
        cv2.circle(image_np, point, 1, (0, 255, 0), 1)
    cv2.imshow("points_deleted", image_np)
    points = delete_close_points(points, row_point_list, col_point_list)
    print("len(points)", len(points))
    for point in points:
        cv2.circle(image_np, point, 1, (255, 0, 0), 3)
    cv2.imshow("points_deleted", image_np)
    cv2.waitKey(0)

    row_point_list = get_points_row(points, 5)
    col_point_list = get_points_col(points, 5)

    print("len(row_point_list)", len(row_point_list))
    for row in row_point_list:
        print("row", len(row))

    print("col_point_list", len(col_point_list))
    for col in col_point_list:
        print("col", len(col))

    bbox = []
    for i in range(len(row_point_list)):
        if i == len(row_point_list) - 1:
            break
        # 遍历每个row的point，找到其所在列的下一个点和所在行的下一个点
        current_row = row_point_list[i]
        for j in range(len(current_row)):
            current_point = current_row[j]
            if j == len(current_row) - 1:
                break
            next_row_point = current_row[j+1]
            # 找出当前点所在的col，得到该列下一个point
            current_col = col_point_list[j]
            for k in range(len(current_col)):
                if current_col[k][1] > current_point[1] + 10:
                    next_col_point = current_col[k]
                    break
            next_row = row_point_list[k]
            for k in range(len(next_row)):
                if next_row[k][0] >= next_row_point[0] + 5:
                    next_point = next_row[k]
                    break
            # 得到bbox
            bbox.append([(current_point[0], current_point[1]), (next_point[0], next_point[1])])



    # bbox = []
    # for p in points:
    #     # print("p", p)
    #     p_row = []
    #     p_col = []
    #     for row in row_point_list:
    #         if p[0] == row[0][0]:
    #             for p1 in row:
    #                 if abs(p[1]-p1[1]) <= 5:
    #                     continue
    #                 p_row.append([p1, abs(p[1]-p1[1])])
    #             p_row.sort(key=lambda x: x[1])
    #     for col in col_point_list:
    #         if p[1] == col[0][1]:
    #             for p2 in col:
    #                 if abs(p[0]-p2[0]) <= 5:
    #                     continue
    #                 p_col.append([p2, abs(p[0]-p2[0])])
    #             p_col.sort(key=lambda x: x[1])
    #     if len(p_row) == 0 or len(p_col) == 0:
    #         continue
    #     break_flag = 0
    #     for i in range(len(p_row)):
    #         for j in range(len(p_col)):
    #             # print(p_row[i][0])
    #             # print(p_col[j][0])
    #             another_point = (p_col[j][0][0], p_row[i][0][1])
    #             # print("another_point", another_point)
    #             if abs(p[0]-another_point[0]) <= 5 or abs(p[1]-another_point[1]) <= 5:
    #                 continue
    #             if p[0] >= another_point[0] or p[1] >= another_point[1]:
    #                 continue
    #             if another_point in points:
    #                 box = [p, another_point]
    #                 box.sort(key=lambda x: x[0])
    #                 if box not in bbox:
    #                     bbox.append(box)
    #                     break_flag = 1
    #                     break
    #         if break_flag:
    #             break
    #
    # # delete duplicate
    # delete_bbox = []
    # for i in range(len(bbox)):
    #     for j in range(i+1, len(bbox)):
    #         if bbox[i][0] == bbox[j][0]:
    #             if bbox[i][1][0] - bbox[j][1][0] <= 3 \
    #                     and bbox[i][1][1] - bbox[j][1][1] <= 3:
    #                 delete_bbox.append(bbox[j])
    #         if bbox[i][1] == bbox[j][1]:
    #             if bbox[i][0][0] - bbox[j][0][0] <= 3 \
    #                     and bbox[i][0][1] - bbox[j][0][1] <= 3:
    #                 delete_bbox.append(bbox[j])
    # # delete too small area
    # # for box in bbox:
    # #     if box[1][0] - box[0][0] <=
    # for d_box in delete_bbox:
    #     if d_box in bbox:
    #         bbox.remove(d_box)
    # print bbox
    bbox.sort(key=lambda x: (x[0][0], x[0][1], x[1][0], x[1][1]))
    # origin bbox
    # origin_bbox = []
    # for box in bbox:
    #     origin_bbox.append([(box[0][0], box[0][1] - 40), (box[1][0], box[1][1] - 40)])
    # for box in origin_bbox:
    #     cv2.rectangle(origin_image, box[0], box[1], (0, 0, 255), 2, 8)
    #     cv2.imshow('AlanWang', origin_image)
    #     cv2.waitKey(0)
    for box in bbox:
        cv2.rectangle(image_np, box[0], box[1], (0, 0, 255), 2, 8)
        cv2.imshow('bboxes', image_np)
        cv2.waitKey(0)
    # for point in points:
    #     print(point)
    #     cv2.circle(image_np, point, 1, (0, 0, 255), 3)
    #     cv2.imshow('points', image_np)
    #     cv2.waitKey(0)
    return bbox


def get_bbox1(image_np, points, split_y):
    # 分割线纵坐标
    # print("split_y", split_y)
    if len(split_y) < 2:
        return []

    # 计算行列，剔除相近交点
    row_point_list = get_points_row(points)
    col_point_list = get_points_col(points)
    print("len(row_point_list)", row_point_list)
    print("len(col_point_list)", len(col_point_list))
    # for point in points:
    #     cv2.circle(image_np, point, 1, (0, 255, 0), 1)
    # cv2.imshow("points", image_np)
    points = delete_close_points(points, row_point_list, col_point_list)
    # print("len(points)", len(points))
    # for point in points:
    #     cv2.circle(image_np, point, 1, (255, 0, 0), 3)
    # cv2.imshow("points_deleted", image_np)
    # cv2.waitKey(0)

    # 获取bbox
    bbox = []
    # 每个点获取与其x最相近和y最相近的点
    for i in range(1, len(split_y)):
        for point1 in points:
            if point1[1] <= split_y[i-1] or point1[1] >= split_y[i]:
                continue
            distance_x = 10000
            distance_y = 10000
            x = 0
            y = 0
            threshold = 10
            for point2 in points:
                if point2[1] <= split_y[i-1] or point2[1] >= split_y[i]:
                    continue
                # 最近 x y
                if 2 < point2[0] - point1[0] < distance_x and point2[1] - point1[1] <= threshold:
                    distance_x = point2[0] - point1[0]
                    x = point2[0]

                if 2 < point2[1] - point1[1] < distance_y and point2[0] - point1[0] <= threshold:
                    distance_y = point2[1] - point1[1]
                    y = point2[1]

            if not x or not y:
                continue

            bbox.append([(point1[0], point1[1]), (x, y)])

    # 删除包含关系bbox
    temp_list = []
    for i in range(len(bbox)):
        box1 = bbox[i]
        for j in range(len(bbox)):
            if i == j:
                continue
            box2 = bbox[j]
            contain_flag = 0
            if box2[0][0] <= box1[0][0] <= box1[1][0] <= box2[1][0] and \
                    box2[0][1] <= box1[0][1] <= box1[1][1] <= box2[1][1]:
                contain_flag = 1
                break
        temp_list.append(box1)
    bbox = temp_list

    # 展示
    for box in bbox:
        # print(box[0], box[1])
        # if abs(box[0][1] - box[1][1]) > abs(box[0][0] - box[1][0]):
        #     continue
        cv2.rectangle(image_np, box[0], box[1], (0, 0, 255), 2, 8)
        cv2.imshow('bboxes', image_np)
        cv2.waitKey(0)

    return bbox


def get_bbox0(image_np, row_point_list, col_point_list, split_y, row_lines, col_lines):
    # 分割线纵坐标
    if len(split_y) < 2:
        return []

    # 计算行列，剔除相近交点
    # row_point_list = get_points_row(points)
    # col_point_list = get_points_col(points)
    # points = delete_close_points(points, row_point_list, col_point_list)
    # row_point_list = get_points_row(points)
    # col_point_list = get_points_col(points)

    # 获取bbox
    bbox = []

    # print("get_bbox split_y", split_y)
    # 每个点获取与其x最相近和y最相近的点
    for i in range(1, len(split_y)):

        # 循环每行
        for row in row_point_list:
            row.sort(key=lambda x: (x[0], x[1]))
            # 行不在该区域跳过
            if row[0][1] <= split_y[i-1] or row[0][1] >= split_y[i]:
                continue

            # 循环行中的点
            for j in range(len(row)):
                if j == len(row) - 1:
                    break
                current_point = row[j]
                next_point_in_row = row[j+1]
                # 查询下个点所在列
                next_col = []
                for col in col_point_list:
                    col.sort(key=lambda x: (x[1], x[0]))
                    # 列不在该区域跳过
                    if col[0][1] <= split_y[i-1] or col[-1][1] >= split_y[i]:
                        continue
                    if col[0][0]-3 <= next_point_in_row[0] <= col[0][0]+3:
                        next_col = col
                        break

                # 循环匹配当前点和下一列点
                for point1 in next_col:
                    # 同一行的就跳过
                    if current_point[1]-3 <= point1[1] <= current_point[1]+3:
                        continue
                    if point1[1] <= current_point[1]-3:
                        continue

                    # 候选bbox
                    candidate_bbox = [current_point[0], current_point[1], point1[0], point1[1]]

                    # 判断该bbox是否存在，线条包含关系
                    contain_flag1 = 0
                    contain_flag2 = 0
                    for row1 in row_lines:
                        # 行不在该区域跳过
                        if row1[1] <= split_y[i-1] or row1[1] >= split_y[i]:
                            continue
                        # bbox上边框 y一样
                        if not contain_flag1:
                            if row1[1]-3 <= candidate_bbox[1] <= row1[1]+3:
                                # candidate的x1,x2需被包含在row线中
                                if row1[0]-3 <= candidate_bbox[0] <= candidate_bbox[2] <= row1[2]+3:
                                    contain_flag1 = 1
                        # bbox下边框 y一样
                        if not contain_flag2:
                            if row1[1]-3 <= candidate_bbox[3] <= row1[1]+3:
                                # candidate的x1,x2需被包含在row线中
                                if row1[0]-3 <= candidate_bbox[0] <= candidate_bbox[2] <= row1[2]+3:
                                    contain_flag2 = 1

                    # 找到了该bbox，并且是存在的
                    if contain_flag1 and contain_flag2:
                        bbox.append([(candidate_bbox[0], candidate_bbox[1]),
                                     (candidate_bbox[2], candidate_bbox[3])])
                        break
    return bbox


def get_bbox(image_np, row_point_list, col_point_list, split_y, row_lines, col_lines):
    # 分割线纵坐标
    if len(split_y) < 2:
        return []

    # 获取bbox
    bbox = []
    # 每个点获取与其x最相近和y最相近的点
    for i in range(1, len(split_y)):

        # 循环每行
        for row in row_point_list:
            row.sort(key=lambda x: (x[0], x[1]))
            # 行不在该区域跳过
            if row[0][1] <= split_y[i-1] or row[0][1] >= split_y[i]:
                continue

            # print("len(row)", len(row))
            # print("row", row)

            # 循环行中的点
            for j in range(len(row)):
                if j == len(row) - 1:
                    break
                current_point = row[j]
                next_point_in_row_list = row[j+1:]

                # 循环这一行的下一个点
                for next_point_in_row in next_point_in_row_list:
                    # 是否在这一行点找到，找不到就这一行的下个点
                    not_found = 1
                    # 查询下个点所在列
                    next_col = []
                    for col in col_point_list:
                        col.sort(key=lambda x: (x[1], x[0]))
                        # 列不在该区域跳过
                        if col[0][1] <= split_y[i-1] or col[-1][1] >= split_y[i]:
                            continue
                        if col[0][0]-3 <= next_point_in_row[0] <= col[0][0]+3:
                            next_col = col
                            break

                    # 循环匹配当前点和下一列点
                    next_col.sort(key=lambda x: (x[1], x[0]))
                    for point1 in next_col:
                        # 同一行的就跳过
                        if current_point[1]-3 <= point1[1] <= current_point[1]+3:
                            continue
                        if point1[1] <= current_point[1]-3:
                            continue

                        # 候选bbox
                        candidate_bbox = [current_point[0], current_point[1], point1[0], point1[1]]
                        # print("candidate_bbox", candidate_bbox)

                        # 判断该bbox是否存在，判断bbox的上下两条边是否有包含在row中
                        contain_flag1 = 0
                        contain_flag2 = 0
                        for row1 in row_lines:
                            # 行不在该区域跳过
                            if row1[1] <= split_y[i-1] or row1[1] >= split_y[i]:
                                continue
                            # bbox上边框 y一样
                            if not contain_flag1:
                                if row1[1]-3 <= candidate_bbox[1] <= row1[1]+3:
                                    # 格子里的断开线段
                                    row1_break = (max([row1[0], candidate_bbox[0]]),
                                                  row1[1],
                                                  min([row1[2], candidate_bbox[2]]),
                                                  row1[3])

                                    if row1_break[2] - row1_break[0] >= (candidate_bbox[2] - candidate_bbox[0])/3:
                                        contain_flag1 = 1

                                    # # candidate的x1,x2需被包含在row线中
                                    # if row1[0]-3 <= candidate_bbox[0] <= candidate_bbox[2] <= row1[2]+3:
                                    #     contain_flag1 = 1
                                    #
                                    # # 判断线条有无端点在格子中
                                    # elif candidate_bbox[0] < row1[0] < candidate_bbox[2] \
                                    #         or candidate_bbox[0] < row1[2] < candidate_bbox[2]:
                                    #     # 线条会有缺一点情况，判断长度超过格子一半
                                    #     if row1_break[2] - row1_break[0] >= (candidate_bbox[2] - candidate_bbox[0])/3:
                                    #         contain_flag1 = 1

                            # bbox下边框 y一样
                            if not contain_flag2:
                                if row1[1]-3 <= candidate_bbox[3] <= row1[1]+3:
                                    # 格子里的断开线段
                                    row1_break = (max([row1[0], candidate_bbox[0]]),
                                                  row1[1],
                                                  min([row1[2], candidate_bbox[2]]),
                                                  row1[3])
                                    if row1_break[2] - row1_break[0] >= (candidate_bbox[2] - candidate_bbox[0])/3:
                                        contain_flag2 = 1

                                    # # candidate的x1,x2需被包含在row线中
                                    # if row1[0]-3 <= candidate_bbox[0] <= candidate_bbox[2] <= row1[2]+3:
                                    #     contain_flag2 = 1
                                    #
                                    # # 判断线条有无端点在格子中
                                    # elif candidate_bbox[0] < row1[0] < candidate_bbox[2] \
                                    #         or candidate_bbox[0] < row1[2] < candidate_bbox[2]:
                                    #     # 线条会有缺一点情况，判断长度超过格子一半
                                    #     if row1_break[2] - row1_break[0] >= (candidate_bbox[2] - candidate_bbox[0])/3:
                                    #         contain_flag2 = 1

                        # 判断该bbox是否存在，判断bbox的左右两条边是否有包含在col中
                        contain_flag3 = 0
                        contain_flag4 = 0
                        for col1 in col_lines:
                            # 列不在该区域跳过
                            if col1[1] <= split_y[i-1] or col1[3] >= split_y[i]:
                                continue
                            # bbox左边线 x一样
                            if not contain_flag3:
                                if col1[0]-3 <= candidate_bbox[0] <= col1[0]+3:
                                    # 格子里的断开线段
                                    col1_break = (col1[0],
                                                  max([col1[1], candidate_bbox[1]]),
                                                  col1[2],
                                                  min([col1[3], candidate_bbox[3]]))
                                    if col1_break[3] - col1_break[1] >= (candidate_bbox[3] - candidate_bbox[1])/3:
                                        contain_flag3 = 1
                                    # # candidate的y1,y2需被包含在col线中
                                    # if col1[1]-3 <= candidate_bbox[1] <= candidate_bbox[3] <= col1[3]+3:
                                    #     contain_flag3 = 1
                                    #
                                    # # 判断线条有无端点在格子中
                                    # elif candidate_bbox[1] < col1[1] < candidate_bbox[3] \
                                    #         or candidate_bbox[1] < col1[3] < candidate_bbox[3]:
                                    #     # 线条会有缺一点情况，判断长度超过格子一半
                                    #     if col1_break[3] - col1_break[1] >= (candidate_bbox[3] - candidate_bbox[1])/3:
                                    #         contain_flag3 = 1
                            # bbox右边框 x一样
                            if not contain_flag4:
                                if col1[0]-3 <= candidate_bbox[2] <= col1[0]+3:
                                    # 格子里的断开线段
                                    col1_break = (col1[0],
                                                  max([col1[1], candidate_bbox[1]]),
                                                  col1[2],
                                                  min([col1[3], candidate_bbox[3]]))
                                    if col1_break[3] - col1_break[1] >= (candidate_bbox[3] - candidate_bbox[1])/3:
                                        contain_flag4 = 1

                                    # # candidate的y1,y2需被包含在col线中
                                    # if col1[1]-3 <= candidate_bbox[1] <= candidate_bbox[3] <= col1[3]+3:
                                    #     contain_flag4 = 1
                                    #
                                    # # 判断线条有无端点在格子中
                                    # elif candidate_bbox[1] < col1[1] < candidate_bbox[3] \
                                    #         or candidate_bbox[1] < col1[3] < candidate_bbox[3]:
                                    #     # 线条会有缺一点情况，判断长度超过格子一半
                                    #     if col1_break[3] - col1_break[1] >= (candidate_bbox[3] - candidate_bbox[1])/3:
                                    #         contain_flag4 = 1
                        # 找到了该bbox，并且是存在的
                        if contain_flag1 and contain_flag2 and contain_flag3 and contain_flag4:
                            bbox.append([(candidate_bbox[0], candidate_bbox[1]),
                                         (candidate_bbox[2], candidate_bbox[3])])
                            not_found = 0
                            # print("candidate_bbox", candidate_bbox)
                            # print(contain_flag1, contain_flag2, contain_flag3, contain_flag4)
                            break
                        # else:
                        #     print("candidate_bbox", candidate_bbox)
                        #     print(contain_flag1, contain_flag2, contain_flag3, contain_flag4)

                    if not not_found:
                        break
    return bbox


def get_bbox_by_contours(image_np):
    img_gray = cv2.cvtColor(image_np, cv2.COLOR_BGR2GRAY)
    ret, img_bin = cv2.threshold(img_gray, 127, 255, cv2.THRESH_BINARY)

    # 3.连通域分析
    img_bin, contours, hierarchy = cv2.findContours(img_bin,
                                                    cv2.RETR_LIST,
                                                    cv2.CHAIN_APPROX_SIMPLE)

    # 4.获取最小外接圆 圆心 半径
    center, radius = cv2.minEnclosingTriangle(contours[0])
    center = np.int0(center)

    # 5.绘制最小外接圆
    img_result = image_np.copy()
    cv2.circle(img_result, tuple(center), int(radius), (255, 255, 255), 2)


    # # 读入图片
    # img = image_np
    # cv2.imshow("get_bbox_by_contours ", image_np)
    # # 中值滤波，去噪
    # img = cv2.medianBlur(img, 3)
    # gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # cv2.namedWindow('original', cv2.WINDOW_AUTOSIZE)
    # cv2.imshow('original', gray)
    #
    # # 阈值分割得到二值化图片
    # ret, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
    #
    # # 膨胀操作
    # kernel2 = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
    # bin_clo = cv2.dilate(binary, kernel2, iterations=2)
    #
    # # 连通域分析
    # num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(bin_clo, connectivity=8)
    #
    # # 查看各个返回值
    # # 连通域数量
    # print('num_labels = ',num_labels)
    # # 连通域的信息：对应各个轮廓的x、y、width、height和面积
    # print('stats = ',stats)
    # # 连通域的中心点
    # print('centroids = ',centroids)
    # # 每一个像素的标签1、2、3.。。，同一个连通域的标签是一致的
    # print('labels = ',labels)
    #
    # # 不同的连通域赋予不同的颜色
    # output = np.zeros((img.shape[0], img.shape[1], 3), np.uint8)
    # for i in range(1, num_labels):
    #
    #     mask = labels == i
    #     output[:, :, 0][mask] = np.random.randint(0, 255)
    #     output[:, :, 1][mask] = np.random.randint(0, 255)
    #     output[:, :, 2][mask] = np.random.randint(0, 255)
    # cv2.imshow('oginal', output)
    # cv2.waitKey()
    # cv2.destroyAllWindows()


def get_points_col(points, split_y, threshold=5):
    # 坐标点按行分
    row_point_list = []
    row_point = []
    points.sort(key=lambda x: (x[0], x[1]))
    x = points[0][0]
    for i in range(1, len(split_y)):
        for p in points:
            if p[1] <= split_y[i-1] or p[1] >= split_y[i]:
                continue
            if x-threshold <= p[0] <= x+threshold:
                row_point.append(p)
            else:
                row_point.sort(key=lambda x: (x[1], x[0]))
                if row_point:
                    row_point_list.append(row_point)
                row_point = []
                x = p[0]
                row_point.append(p)
        if row_point:
            row_point_list.append(row_point)

    return row_point_list


def get_points_row(points, split_y, threshold=5):
    # 坐标点按列分
    col_point_list = []
    col_point = []
    points.sort(key=lambda x: (x[1], x[0]))
    y = points[0][1]
    for i in range(len(split_y)):
        for p in points:
            if p[1] <= split_y[i-1] or p[1] >= split_y[i]:
                continue
            if y-threshold <= p[1] <= y+threshold:
                col_point.append(p)
            else:
                col_point.sort(key=lambda x: (x[0], x[1]))
                if col_point:
                    col_point_list.append(col_point)
                col_point = []
                y = p[1]
                col_point.append(p)
        if col_point:
            col_point_list.append(col_point)
    return col_point_list


def get_outline_point(points, split_y):
    # 分割线纵坐标
    # print("get_outline_point split_y", split_y)
    if len(split_y) < 2:
        return []

    outline_2point = []
    points.sort(key=lambda x: (x[1], x[0]))
    for i in range(1, len(split_y)):
        area_points = []
        for point in points:
            if point[1] <= split_y[i-1] or point[1] >= split_y[i]:
                continue
            area_points.append(point)

        if area_points:
            area_points.sort(key=lambda x: (x[1], x[0]))
            outline_2point.append([area_points[0], area_points[-1]])

    return outline_2point


# def merge_row(row_lines):
#     for row in row_lines:
#         for row1 in row_lines:

def get_best_predict_size(image_np):
    sizes = [1280, 1152, 1024, 896, 768, 640, 512, 384, 256, 128]

    min_len = 10000
    best_height = sizes[0]
    for height in sizes:
        if abs(image_np.shape[0] - height) < min_len:
            min_len = abs(image_np.shape[0] - height)
            best_height = height

    min_len = 10000
    best_width = sizes[0]
    for width in sizes:
        if abs(image_np.shape[1] - width) < min_len:
            min_len = abs(image_np.shape[1] - width)
            best_width = width

    return best_height, best_width


def choose_longer_row(lines):
    new_row = []
    jump_row = []
    for i in range(len(lines)):
        row1 = lines[i]
        jump_flag = 0
        if row1 in jump_row:
            continue
        for j in range(i+1, len(lines)):
            row2 = lines[j]
            if row2 in jump_row:
                continue
            if row2[1]-5 <= row1[1] <= row2[1]+5:
                if row1[0] <= row2[0] and row1[2] >= row2[2]:
                    new_row.append(row1)
                    jump_row.append(row1)
                    jump_row.append(row2)
                    jump_flag = 1
                    break
                elif row2[0] <= row1[0] and row2[2] >= row1[2]:
                    new_row.append(row2)
                    jump_row.append(row1)
                    jump_row.append(row2)
                    jump_flag = 1
                    break
        if not jump_flag:
            new_row.append(row1)
            jump_row.append(row1)
    return new_row


def choose_longer_col(lines):
    new_col = []
    jump_col = []
    for i in range(len(lines)):
        col1 = lines[i]
        jump_flag = 0
        if col1 in jump_col:
            continue
        for j in range(i+1, len(lines)):
            col2 = lines[j]
            if col2 in jump_col:
                continue
            if col2[0]-5 <= col1[0] <= col2[0]+5:
                if col1[1] <= col2[1] and col1[3] >= col2[3]:
                    new_col.append(col1)
                    jump_col.append(col1)
                    jump_col.append(col2)
                    jump_flag = 1
                    break
                elif col2[1] <= col1[1] and col2[3] >= col1[3]:
                    new_col.append(col2)
                    jump_col.append(col1)
                    jump_col.append(col2)
                    jump_flag = 1
                    break
        if not jump_flag:
            new_col.append(col1)
            jump_col.append(col1)
    return new_col


def line_fix(image_np):
    image_binary = cv2.cvtColor(image_np, cv2.COLOR_BGR2GRAY)
    rows, cols = image_binary.shape
    scale = 100
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, rows//scale))
    row_erode = cv2.erode(image_binary, kernel, iterations=1)
    cv2.imshow("row_erode", row_erode)

    scale = 100
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (cols//scale, 1))
    col_erode = cv2.erode(image_binary, kernel, iterations=1)
    cv2.imshow("col_erode", col_erode)

    image_erode = row_erode + col_erode
    cv2.imshow("image_erode", image_erode)

    return image_erode


if __name__ == '__main__':
    net = table_net((1024, 768, 3), 2)
    net.summary()