update

.idea/captcha_pro.iml

@@ -8,11 +8,10 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
+  <component name="PackageRequirementsSettings">
+    <option name="requirementsPath" value="" />
+  </component>
   <component name="TemplatesService">
     <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
   </component>
-  <component name="TestRunnerService">
-    <option name="projectConfiguration" value="py.test" />
-    <option name="PROJECT_TEST_RUNNER" value="py.test" />
-  </component>
 </module>

border_recognize/bdr_interface.py

@@ -0,0 +1,102 @@
+import base64
+import json
+import logging
+import os
+import time
+import traceback
+from glob import glob
+import cv2
+import numpy as np
+from PIL import Image
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+import tensorflow as tf
+from flask import Flask, request
+from border_recognize.model import u_net_drag
+from border_recognize.inference_drag import recognize
+from utils import pil_resize, np2bytes, request_post, bytes2np
+
+tf.compat.v1.disable_eager_execution()
+sess = tf.compat.v1.Session(graph=tf.Graph())
+
+
+# 接口配置
+app = Flask(__name__)
+
+
+@app.route('/bdr', methods=['POST'])
+def bdr():
+    start_time = time.time()
+    logging.info("into bdr_interface bdr")
+    try:
+        # 接收网络数据
+        if not request.form:
+            logging.info("bdr no data!")
+            return json.dumps({"data": "", "success": 0})
+        data = request.form.get("data")
+        logging.info("bdr_interface get data time" + str(time.time()-start_time))
+
+        # 加载模型
+        bdr_model = globals().get("global_bdr_model")
+        if bdr_model is None:
+            print("=========== init bdr model ===========")
+            bdr_model = BdrModels().get_model()
+            globals().update({"global_bdr_model": bdr_model})
+
+        # 数据转换
+        data = base64.b64decode(data)
+        image_np = bytes2np(data)
+        # 预测
+        w = recognize(image_np, bdr_model, sess)
+        return json.dumps({"data": w, "success": 1})
+    except:
+        traceback.print_exc()
+        return json.dumps({"data": "", "success": 0})
+    finally:
+        logging.info("bdr interface finish time " + str(time.time()-start_time))
+
+
+class BdrModels:
+    def __init__(self):
+        # python文件所在目录
+        _dir = os.path.abspath(os.path.dirname(__file__))
+
+        # detect
+        model_path = _dir + "/models/drag_f1_0.42.h5"
+        image_shape = (128, 256, 3)
+        with sess.as_default():
+            with sess.graph.as_default():
+                self.model = u_net_drag(input_shape=image_shape)
+                self.model.load_weights(model_path)
+
+    def get_model(self):
+        return self.model
+
+
+def test_bdr_model(from_remote=True):
+    paths = glob("D:/Project/captcha/data/test/yolo_18.jpg")
+    for file_path in paths:
+        img_np = cv2.imread(file_path)
+        h, w = img_np.shape[:2]
+        file_bytes = np2bytes(img_np)
+        file_base64 = base64.b64encode(file_bytes)
+
+        if from_remote:
+            file_json = {"data": file_base64}
+            # _url = "http://192.168.2.102:17000/ocr"
+            _url = "http://127.0.0.1:17000/bdr"
+            result = json.loads(request_post(_url, file_json))
+            if result.get("success"):
+                w = int(result.get("data"))
+                print("w", w)
+                img_new = np.concatenate([img_np[:, w:, :], img_np[:, :w, :]], axis=1)
+                cv2.imshow("img_np", img_np)
+                cv2.imshow("img_new", img_new)
+                cv2.waitKey(0)
+            else:
+                print("failed!")
+
+
+if __name__ == "__main__":
+    # app.run(host='127.0.0.1', port=17000, debug=False)
+    test_bdr_model()

border_recognize/inference_drag.py

@@ -0,0 +1,60 @@
+import os
+import cv2
+import numpy as np
+import tensorflow as tf
+from border_recognize.model import u_net_drag
+from utils import pil_resize
+
+image_shape = (128, 256, 3)
+model_path = "./models/drag_f1_0.42.h5"
+project_dir = os.path.dirname(os.path.abspath(__file__)) + "/../"
+
+
+def recognize(image_np, model=None, sess=None):
+    if sess is None:
+        sess = tf.compat.v1.Session(graph=tf.Graph())
+
+    if model is None:
+        with sess.as_default():
+            with sess.graph.as_default():
+                model = u_net_drag(input_shape=image_shape)
+                model.load_weights(model_path)
+
+    h_origin, w_origin = image_np.shape[:2]
+    img = image_np
+    img = pil_resize(img, image_shape[0], image_shape[1])
+    # cv2.imshow("img", img)
+
+    img = img / 255.
+    X = np.expand_dims(img, 0)
+
+    with sess.as_default():
+        with sess.graph.as_default():
+            pred = model.predict(X)
+
+    pred = pred[0][..., 0]
+    pred = np.expand_dims(pred, axis=-1)
+    pred = np.concatenate([pred]*3, axis=-1)
+    pred = np.uint8(pred*255.)
+    # print(pred.shape)
+    # cv2.imshow("pred", pred)
+
+    w = int(np.argmax(np.mean(np.mean(pred, axis=0), axis=-1)))
+    # print(w)
+    # decode = np.zeros(pred.shape, np.uint8)
+    # decode[:, w, :] = 255
+    # cv2.imshow("decode", decode)
+    #
+    # img_new = np.concatenate([img[:, w:, :], img[:, :w, :]], axis=1)
+    # cv2.imshow("img_new", img_new)
+    # cv2.waitKey(0)
+
+    # 还原
+    w = int(w * w_origin / image_shape[1])
+    return w
+
+
+if __name__ == "__main__":
+    _path = "../data/test/yolo_16.jpg"
+    # _path = "../data/drag/3.jpg"
+    recognize(_path)

border_recognize/model.py

@@ -0,0 +1,129 @@
+
+from keras import Input, Model
+from keras.layers import Dense, Conv2D,  GlobalAveragePooling2D, BatchNormalization,\
+    LeakyReLU, MaxPooling2D, UpSampling2D, Dropout, concatenate
+import keras.backend as K
+
+
+def cnn_net_drag(input_shape, output_shape=260):
+    _input = Input(input_shape)
+
+    use_bias = False
+    down0 = Conv2D(16, (3, 3), use_bias=use_bias)(_input)
+    down0 = BatchNormalization()(down0)
+    down0 = LeakyReLU(alpha=0.1)(down0)
+    down0 = Conv2D(16, (3, 3), use_bias=use_bias)(down0)
+    down0 = BatchNormalization()(down0)
+    down0 = LeakyReLU(alpha=0.1)(down0)
+    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
+
+    down1 = Conv2D(32, (3, 3), use_bias=use_bias)(down0_pool)
+    down1 = BatchNormalization()(down1)
+    down1 = LeakyReLU(alpha=0.1)(down1)
+    down1 = Conv2D(32, (3, 3), use_bias=use_bias)(down1)
+    down1 = BatchNormalization()(down1)
+    down1 = LeakyReLU(alpha=0.1)(down1)
+    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
+
+    down2 = Conv2D(64, (3, 3), use_bias=use_bias)(down1_pool)
+    down2 = BatchNormalization()(down2)
+    down2 = LeakyReLU(alpha=0.1)(down2)
+    down2 = Conv2D(64, (3, 3), use_bias=use_bias)(down2)
+    down2 = BatchNormalization()(down2)
+    down2 = LeakyReLU(alpha=0.1)(down2)
+    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
+
+    down3 = Conv2D(64, (3, 3), use_bias=use_bias)(down2_pool)
+    down3 = BatchNormalization()(down3)
+    down3 = LeakyReLU(alpha=0.1)(down3)
+    down3 = Conv2D(64, (3, 3), use_bias=use_bias)(down3)
+    down3 = BatchNormalization()(down3)
+    down3 = LeakyReLU(alpha=0.1)(down3)
+    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
+
+    gap = GlobalAveragePooling2D()(down3_pool)
+
+    dense = Dense(32, activation="relu")(gap)
+    drop = Dropout(0.2)(dense)
+    dense = Dense(output_shape, activation="softmax")(drop)
+
+    model = Model(_input, dense)
+    model.summary()
+    return model
+
+
+def u_net_drag(input_shape, output_shape=260, cls_num=2):
+    inputs = Input(shape=input_shape)
+    use_bias = False
+
+    # 128
+    down1 = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(inputs)
+    down1 = BatchNormalization()(down1)
+    down1 = LeakyReLU(alpha=0.1)(down1)
+    down1 = Conv2D(16, (1, 1), padding='same', use_bias=use_bias)(down1)
+    down1 = BatchNormalization()(down1)
+    down1 = LeakyReLU(alpha=0.1)(down1)
+    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
+
+    # 64
+    down2 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(down1_pool)
+    down2 = BatchNormalization()(down2)
+    down2 = LeakyReLU(alpha=0.1)(down2)
+    down2 = Conv2D(32, (1, 1), padding='same', use_bias=use_bias)(down2)
+    down2 = BatchNormalization()(down2)
+    down2 = LeakyReLU(alpha=0.1)(down2)
+    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
+
+    # 32
+    down3 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(down2_pool)
+    down3 = BatchNormalization()(down3)
+    down3 = LeakyReLU(alpha=0.1)(down3)
+    down3 = Conv2D(64, (1, 1), padding='same', use_bias=use_bias)(down3)
+    down3 = BatchNormalization()(down3)
+    down3 = LeakyReLU(alpha=0.1)(down3)
+    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
+
+    # 16
+    center = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(down3_pool)
+    center = BatchNormalization()(center)
+    center = LeakyReLU(alpha=0.1)(center)
+    center = Conv2D(64, (1, 1), padding='same', use_bias=use_bias)(center)
+    center = BatchNormalization()(center)
+    center = LeakyReLU(alpha=0.1)(center)
+
+    # 32
+    up3 = UpSampling2D((2, 2))(center)
+    up3 = concatenate([down3, up3], axis=3)
+    up3 = Conv2D(64, (3, 3), padding='same', use_bias=use_bias)(up3)
+    up3 = BatchNormalization()(up3)
+    up3 = LeakyReLU(alpha=0.1)(up3)
+    up3 = Conv2D(64, (1, 1), padding='same', use_bias=use_bias)(up3)
+    up3 = BatchNormalization()(up3)
+    up3 = LeakyReLU(alpha=0.1)(up3)
+
+    # 64
+    up2 = UpSampling2D((2, 2))(up3)
+    up2 = concatenate([down2, up2], axis=3)
+    up2 = Conv2D(32, (3, 3), padding='same', use_bias=use_bias)(up2)
+    up2 = BatchNormalization()(up2)
+    up2 = LeakyReLU(alpha=0.1)(up2)
+    up2 = Conv2D(32, (1, 1), padding='same', use_bias=use_bias)(up2)
+    up2 = BatchNormalization()(up2)
+    up2 = LeakyReLU(alpha=0.1)(up2)
+
+    # 128
+    up1 = UpSampling2D((2, 2))(up2)
+    up1 = K.concatenate([down1, up1], axis=3)
+    up1 = Conv2D(16, (3, 3), padding='same', use_bias=use_bias)(up1)
+    up1 = BatchNormalization()(up1)
+    up1 = LeakyReLU(alpha=0.1)(up1)
+    up1 = Conv2D(16, (1, 1), padding='same', use_bias=use_bias)(up1)
+    up1 = BatchNormalization()(up1)
+    up1 = LeakyReLU(alpha=0.1)(up1)
+
+    classify = Conv2D(1, (1, 1), activation='sigmoid')(up1)
+    # classify = Dense(cls_num, activation="softmax")(up1)
+    model = Model(inputs=inputs, outputs=classify)
+
+    model.summary(line_length=100)
+    return model

chinese_detect/chd_interface.py

@@ -0,0 +1,102 @@
+import base64
+import json
+import logging
+import os
+import time
+import traceback
+from glob import glob
+import cv2
+import numpy as np
+from PIL import Image
+
+from chinese_detect.inference_yolo_char import get_tiny_inference_model, detect
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+import tensorflow as tf
+from flask import Flask, request
+from border_recognize.model import u_net_drag
+from border_recognize.inference_drag import recognize
+from utils import pil_resize, np2bytes, request_post, bytes2np, get_anchors, get_classes, get_colors
+
+tf.compat.v1.disable_eager_execution()
+sess = tf.compat.v1.Session(graph=tf.Graph())
+
+_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
+model_path = _dir + "/models/char_yolo_loss_39.90.h5"
+anchors = get_anchors(_dir + "/yolo_data/my_anchors.txt")
+classes = get_classes(_dir + "/yolo_data/my_classes.txt")
+colors = get_colors(len(classes))
+
+# 接口配置
+app = Flask(__name__)
+
+
+@app.route('/chd', methods=['POST'])
+def chd():
+    start_time = time.time()
+    logging.info("into chd_interface chd")
+    try:
+        # 接收网络数据
+        if not request.form:
+            logging.info("chd no data!")
+            return json.dumps({"data": "", "success": 0})
+        data = request.form.get("data")
+        logging.info("chd_interface get data time" + str(time.time()-start_time))
+
+        # 加载模型
+        chd_model = globals().get("global_chd_model")
+        if chd_model is None:
+            print("=========== init chd model ===========")
+            chd_model = ChdModels().get_model()
+            globals().update({"global_chd_model": chd_model})
+
+        # 数据转换
+        data = base64.b64decode(data)
+        image_np = bytes2np(data)
+        # 预测
+        _, out_boxes, out_classes = detect(image_np, chd_model, sess)
+        return json.dumps({"data": out_boxes, "success": 1})
+    except:
+        traceback.print_exc()
+        return json.dumps({"data": "", "success": 0})
+    finally:
+        logging.info("chd interface finish time " + str(time.time()-start_time))
+
+
+class ChdModels:
+    def __init__(self):
+        # detect
+        with sess.as_default():
+            with sess.graph.as_default():
+                self.model = get_tiny_inference_model(anchors, len(classes), weights_path=model_path)
+
+    def get_model(self):
+        return self.model
+
+
+def test_chd_model(from_remote=True):
+    paths = glob("D:/Project/captcha/data/test/phrase_5.jpg")
+    for file_path in paths:
+        img_np = cv2.imread(file_path)
+        file_bytes = np2bytes(img_np)
+        file_base64 = base64.b64encode(file_bytes)
+
+        if from_remote:
+            file_json = {"data": file_base64}
+            # _url = "http://192.168.2.102:17000/ocr"
+            _url = "http://127.0.0.1:17000/chd"
+            result = json.loads(request_post(_url, file_json))
+            if result.get("success"):
+                out_boxes = result.get("data")
+                print("out_boxes", out_boxes)
+                for box in out_boxes:
+                    cv2.rectangle(img_np, (box[0], box[1]), (box[2], box[3]), (0, 0, 255))
+                cv2.imshow("img_np", img_np)
+                cv2.waitKey(0)
+            else:
+                print("failed!")
+
+
+if __name__ == "__main__":
+    # app.run(host='127.0.0.1', port=17000, debug=False)
+    test_chd_model()

chinese_detect/inference_yolo_char.py

@@ -0,0 +1,106 @@
+import os
+import tensorflow as tf
+import cv2
+from keras.layers import Lambda, Input
+from keras.models import Model
+import numpy as np
+from chinese_detect.model_260 import tiny_yolo_body
+from chinese_detect.post_process import yolo_eval
+from utils import get_classes, get_colors, draw_boxes, get_anchors, pil_resize, np2pil
+
+
+_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
+model_path = _dir + "/models/char_yolo_loss_39.90.h5"
+anchors = get_anchors(_dir + "/yolo_data/my_anchors.txt")
+classes = get_classes(_dir + "/yolo_data/my_classes.txt")
+colors = get_colors(len(classes))
+image_shape = (160, 256, 3)
+
+
+def detect(image_np, model=None, sess=None, draw=True):
+    if sess is None:
+        sess = tf.compat.v1.Session(graph=tf.Graph())
+
+    if model is None:
+        with sess.as_default():
+            with sess.graph.as_default():
+                model = get_tiny_inference_model(anchors, len(classes), weights_path=model_path)
+
+    # create image input
+    h, w = image_np.shape[:2]
+    best_h, best_w = image_shape[:2]
+    image_resize = pil_resize(image_np, best_h, best_w)
+    # image_pil = np2pil(image_resize)
+    image_resize = np.array(image_resize, dtype='float32')
+    image_resize = image_resize.astype('float32') / 255.
+    image_resize = np.expand_dims(image_resize, 0)
+
+    # create image shape input
+    need_shape = np.array([best_h, best_w])
+    need_shape = np.expand_dims(need_shape, 0)
+
+    # inference data
+    with sess.as_default():
+        with sess.graph.as_default():
+            out_boxes, out_scores, out_classes = model.predict([image_resize, need_shape], steps=1)
+    # print("image_size", need_shape)
+    print("out_boxes", out_boxes)
+    print("out_scores", out_scores)
+    # print("out_classes", out_classes)
+    out_boxes = out_boxes.astype(np.int32)
+    out_classes = out_classes.astype(np.int32)
+
+    # 还原
+    out_boxes[:, 0] = h * out_boxes[:, 0] / best_h
+    out_boxes[:, 2] = h * out_boxes[:, 2] / best_h
+    out_boxes[:, 1] = w * out_boxes[:, 1] / best_w
+    out_boxes[:, 3] = w * out_boxes[:, 3] / best_w
+
+    image_pil = np2pil(image_np)
+    if draw:
+        # draw
+        class_names = get_classes("yolo_data/my_classes.txt")
+        colors = get_colors(len(class_names))
+        image_resize, out_boxes = draw_boxes(image_pil, out_boxes, out_classes, out_scores, class_names, colors)
+        image_np_result = cv2.cvtColor(np.array(image_resize), cv2.COLOR_RGB2BGR)
+        cv2.imshow("result", image_np_result)
+        cv2.waitKey(0)
+    else:
+        temp_boxes = []
+        for i, c in reversed(list(enumerate(out_classes))):
+            top, left, bottom, right = out_boxes[i]
+            top = max(0, np.floor(top + 0.5).astype('int32'))
+            left = max(0, np.floor(left + 0.5).astype('int32'))
+            bottom = min(image_pil.size[1], np.floor(bottom + 0.5).astype('int32'))
+            right = min(image_pil.size[0], np.floor(right + 0.5).astype('int32'))
+            temp_boxes.append([(left, top), (right, bottom)])
+        out_boxes = temp_boxes
+    out_boxes = [[int(x[0][0]), int(x[0][1]), int(x[1][0]), int(x[1][1])] for x in out_boxes]
+    return image_np, out_boxes, out_classes
+
+
+def get_tiny_inference_model(anchors, num_classes, weights_path='models/tiny_yolo_weights.h5'):
+    """create the inference model, for Tiny YOLOv3"""
+    image_input = Input(shape=(None, None, 3))
+    image_shape = Input(shape=(2,), dtype='int64', name='image_shape')
+    num_anchors = len(anchors)
+
+    model_body = tiny_yolo_body(image_input, num_anchors//2, num_classes)
+    print('Create Tiny YOLOv3 model with {} anchors and {} classes.'.format(num_anchors, num_classes))
+
+    model_body.load_weights(weights_path)
+    print('Load weights {}.'.format(weights_path))
+
+    boxes, scores, classes = Lambda(yolo_eval,
+                                    name='yolo_eval',
+                                    arguments={'anchors': anchors,
+                                               'num_classes': num_classes}
+                                    )([model_body.output, image_shape])
+    model = Model([model_body.input, image_shape], [boxes, scores, classes])
+    model.summary(120)
+    return model
+
+
+if __name__ == '__main__':
+    image_path = "D:/Project/captcha/data/test/phrase_5.jpg"
+    detect(cv2.imread(image_path))

chinese_detect/loss.py

@@ -0,0 +1,137 @@
+import os
+import sys
+import tensorflow as tf
+import keras.backend as K
+from tensorflow.python.ops.control_flow_ops import while_loop
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+from utils import box_iou
+from chinese_detect.post_process import yolo_head
+
+
+def contrastive_loss(y_true, y_pred):
+    """Contrastive loss from Hadsell-et-al.'06
+    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
+    """
+    margin = 1
+    square_pred = K.square(y_pred)
+    margin_square = K.square(K.maximum(margin - y_pred, 0))
+    return K.mean(y_true * square_pred + (1 - y_true) * margin_square)
+
+
+def focal_loss(gamma=3., alpha=.5, only_tf=True):
+    def focal_loss_fixed(y_true, y_pred):
+        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
+        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
+        if only_tf:
+            return - tf.reduce_sum(alpha * tf.pow(1. - pt_1, gamma) * tf.math.log(1e-07 + pt_1)) \
+                   - tf.reduce_sum((1 - alpha) * tf.pow(pt_0, gamma) * tf.math.log(1. - pt_0 + 1e-07))
+        else:
+            return - K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(K.epsilon()+pt_1)) \
+                   - K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0 + K.epsilon()))
+
+    return focal_loss_fixed
+
+
+def l1_loss():
+    def mae(y_true, y_pred):
+        return tf.reduce_mean(tf.abs(y_pred-y_true)) * 100
+    return mae
+
+
+def l2_loss():
+    def mse(y_true, y_pred):
+        return tf.reduce_mean(tf.square(y_true - y_pred))
+    return mse
+
+
+def l2_focal_loss(threshold=0.2):
+    def mse(y_true, y_pred):
+        y_minus = tf.where(tf.abs(y_pred-y_true) <= threshold, tf.abs(y_pred-y_true), 1000*tf.abs(y_pred-y_true))
+        return tf.reduce_mean(tf.square(y_minus))
+    return mse
+
+
+def l1_focal_loss(threshold=0.2):
+    def mae(y_true, y_pred):
+        y_minus = tf.where(tf.abs(y_pred-y_true) <= threshold, 0., tf.abs(y_pred-y_true))
+        return tf.reduce_sum(tf.abs(y_minus))
+    return mae
+
+
+def l3_loss():
+    def l3_loss_fixed(y_true, y_pred):
+        return tf.reduce_mean(tf.abs(tf.pow(y_pred-y_true, 3)))
+    return l3_loss_fixed
+
+
+def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
+    """Return yolo_loss tensor
+    Parameters
+    ----------
+    yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
+    y_true: list of array, the output of preprocess_true_boxes
+    anchors: array, shape=(N, 2), wh
+    num_classes: integer
+    ignore_thresh: float, the iou threshold whether to ignore object confidence loss
+    Returns
+    -------
+    loss: tensor, shape=(1,)
+    """
+    from keras import backend as K
+    # default setting
+    num_layers = len(anchors)//3
+    yolo_outputs = args[:num_layers]
+    y_true = args[num_layers:]
+    anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
+    input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
+    grid_shapes = [K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers)]
+    loss = 0
+    # batch size, tensor
+    m = K.shape(yolo_outputs[0])[0]
+    mf = K.cast(m, K.dtype(yolo_outputs[0]))
+
+    for l in range(num_layers):
+        object_mask = y_true[l][..., 4:5]
+        true_class_probs = y_true[l][..., 5:]
+
+        grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
+                                                     anchors[anchor_mask[l]], num_classes, input_shape, calc_loss=True)
+        pred_box = K.concatenate([pred_xy, pred_wh])
+
+        # Darknet raw box to calculate loss.
+        raw_true_xy = y_true[l][..., :2]*grid_shapes[l][::-1] - grid
+        raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] * input_shape[::-1])
+        # avoid log(0)=-inf
+        raw_true_wh = K.switch(object_mask, raw_true_wh, K.zeros_like(raw_true_wh))
+        box_loss_scale = 2 - y_true[l][..., 2:3]*y_true[l][..., 3:4]
+
+        # Find ignore mask, iterate over each of batch.
+        ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True)
+        object_mask_bool = K.cast(object_mask, 'bool')
+
+        def loop_body(b, ignore_mask):
+            true_box = tf.boolean_mask(y_true[l][b, ..., 0:4], object_mask_bool[b,...,0])
+            iou = box_iou(pred_box[b], true_box)
+            best_iou = K.max(iou, axis=-1)
+            ignore_mask = ignore_mask.write(b, K.cast(best_iou<ignore_thresh, K.dtype(true_box)))
+            return b+1, ignore_mask
+        _, ignore_mask = while_loop(lambda b, *args: b < m, loop_body, [0, ignore_mask])
+        ignore_mask = ignore_mask.stack()
+        ignore_mask = K.expand_dims(ignore_mask, -1)
+
+        # K.binary_crossentropy is helpful to avoid exp overflow.
+        xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(raw_true_xy, raw_pred[..., 0:2], from_logits=True)
+        wh_loss = object_mask * box_loss_scale * 0.5 * K.square(raw_true_wh-raw_pred[..., 2:4])
+        confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[..., 4:5], from_logits=True) + \
+                          (1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[..., 4:5], from_logits=True) * ignore_mask
+        class_loss = object_mask * K.binary_crossentropy(true_class_probs, raw_pred[..., 5:], from_logits=True)
+
+        xy_loss = K.sum(xy_loss) / mf
+        wh_loss = K.sum(wh_loss) / mf
+        confidence_loss = K.sum(confidence_loss) / mf
+        class_loss = K.sum(class_loss) / mf
+        loss += xy_loss * 10 + wh_loss * 10 + confidence_loss
+        # if print_loss:
+        #     loss = tf.Print(loss, [loss, xy_loss, wh_loss, confidence_loss, class_loss, K.sum(ignore_mask)], message='loss: ')
+    return loss

chinese_detect/model_260.py

@@ -0,0 +1,656 @@
+"""YOLO_v3 Model Defined in Keras."""
+
+from functools import wraps
+
+import numpy as np
+import tensorflow as tf
+from keras import backend as K, Input
+# keras2.6.0 and keras2.1.5
+# from keras.engine import Layer
+from keras.layers import Conv2D, Add, ZeroPadding2D, UpSampling2D, Concatenate, MaxPooling2D, Dense, \
+    GlobalAveragePooling2D, Multiply, Lambda, Layer
+from keras.layers.advanced_activations import LeakyReLU
+# keras2.6.0 and keras2.1.5
+# from keras.layers.normalization import BatchNormalization
+from tensorflow.keras.layers import BatchNormalization
+from keras.models import Model
+from keras.regularizers import l2
+
+from utils import compose
+
+
+def yolo_net(input_shape, anchors, num_classes, load_pretrained=True,
+             weights_path='models/tiny_yolo_weights.h5'):
+    """create the training model, for Tiny YOLOv3"""
+    # get a new session
+    # ops.reset_default_graph()
+    K.clear_session()
+    image_input = Input(shape=(None, None, 1))
+    h, w = input_shape
+    num_anchors = len(anchors)
+
+    y_true = [Input(shape=(h//{0: 32, 1: 16}[l], w//{0: 32, 1: 16}[l],
+                           num_anchors//2, num_classes+5)) for l in range(2)]
+
+    model_body = tiny_yolo_body(image_input, num_anchors//2, num_classes)
+    print('Create Tiny YOLOv3 model with {} anchors and {} classes.'.format(num_anchors, num_classes))
+
+    if load_pretrained:
+        model_body.load_weights(weights_path)
+        print('Load weights {}.'.format(weights_path))
+
+    model_loss = Lambda(yolo_loss, output_shape=(1,), name='yolo_loss',
+                        arguments={'anchors': anchors, 'num_classes': num_classes, 'ignore_thresh': .5})(
+        [*model_body.output, *y_true])
+    model = Model([model_body.input, *y_true], model_loss)
+    model.summary(120)
+    return model
+
+
+@wraps(Conv2D)
+def DarknetConv2D(*args, **kwargs):
+    """Wrapper to set Darknet parameters for Convolution2D."""
+    darknet_conv_kwargs = {'kernel_regularizer': l2(5e-4)}
+    darknet_conv_kwargs['padding'] = 'valid' if kwargs.get('strides')==(2,2) else 'same'
+    darknet_conv_kwargs.update(kwargs)
+    return Conv2D(*args, **darknet_conv_kwargs)
+
+
+def DarknetConv2D_BN_Leaky(*args, **kwargs):
+    """Darknet Convolution2D followed by BatchNormalization and LeakyReLU."""
+    no_bias_kwargs = {'use_bias': False}
+    no_bias_kwargs.update(kwargs)
+    return compose(
+        DarknetConv2D(*args, **no_bias_kwargs),
+        BatchNormalization(),
+        LeakyReLU(alpha=0.1))
+
+
+def resblock_body(x, num_filters, num_blocks):
+    '''A series of resblocks starting with a downsampling Convolution2D'''
+    # Darknet uses left and top padding instead of 'same' mode
+    x = ZeroPadding2D(((1,0),(1,0)))(x)
+    x = DarknetConv2D_BN_Leaky(num_filters, (3,3), strides=(2,2))(x)
+    for i in range(num_blocks):
+        y = compose(
+                DarknetConv2D_BN_Leaky(num_filters//2, (1,1)),
+                DarknetConv2D_BN_Leaky(num_filters, (3,3)))(x)
+        x = Add()([x,y])
+    return x
+
+
+class SeBlock(Layer):
+    def __init__(self, reduction=4, **kwargs):
+        super(SeBlock, self).__init__(**kwargs)
+        self.reduction = reduction
+
+    def build(self, input_shape):
+        # 构建layer时需要实现
+        # 手动将该自定义层参数加入，否则参数为0
+        self.pool = GlobalAveragePooling2D(name="my_pool")
+        self.expand_1 = Lambda(lambda x: K.expand_dims(x, axis=1))
+        self.expand_2 = Lambda(lambda x: K.expand_dims(x, axis=1))
+        self.dense_1 = Dense(int(input_shape[-1]) // self.reduction, use_bias=False, activation="relu", name='my_dense_1')
+        self.dense_2 = Dense(int(input_shape[-1]), use_bias=False, activation="hard_sigmoid", name='my_dense_2')
+
+        # keras2.2.0以下需要单独加，keras2.6.0不用加
+        # self.dense_1.build((input_shape[0], 1, 1, input_shape[-1]))
+        # self.dense_2.build((input_shape[0], 1, 1, input_shape[-1] // self.reduction))
+
+        self._trainable_weights += self.dense_1._trainable_weights
+        self._trainable_weights += self.dense_2._trainable_weights
+        super(SeBlock, self).build(input_shape)
+
+    def call(self, inputs):
+        x = self.pool(inputs)
+        x = self.expand_1(x)
+        x = self.expand_2(x)
+        x = self.dense_1(x)
+        x = self.dense_2(x)
+        # 给通道加权重
+        return Multiply()([inputs, x])
+
+
+def darknet_body(x):
+    '''Darknent body having 52 Convolution2D layers'''
+    x = DarknetConv2D_BN_Leaky(32, (3,3))(x)
+    x = resblock_body(x, 64, 1)
+    x = resblock_body(x, 128, 2)
+    x = resblock_body(x, 256, 8)
+    x = resblock_body(x, 512, 8)
+    x = resblock_body(x, 1024, 4)
+    return x
+
+
+def make_last_layers(x, num_filters, out_filters):
+    '''6 Conv2D_BN_Leaky layers followed by a Conv2D_linear layer'''
+    x = compose(
+            DarknetConv2D_BN_Leaky(num_filters, (1,1)),
+            DarknetConv2D_BN_Leaky(num_filters*2, (3,3)),
+            DarknetConv2D_BN_Leaky(num_filters, (1,1)),
+            DarknetConv2D_BN_Leaky(num_filters*2, (3,3)),
+            DarknetConv2D_BN_Leaky(num_filters, (1,1)))(x)
+    y = compose(
+            DarknetConv2D_BN_Leaky(num_filters*2, (3,3)),
+            DarknetConv2D(out_filters, (1,1)))(x)
+    return x, y
+
+
+def yolo_body(inputs, num_anchors, num_classes):
+    """Create YOLO_V3 model CNN body in Keras."""
+    darknet = Model(inputs, darknet_body(inputs))
+    x, y1 = make_last_layers(darknet.output, 512, num_anchors*(num_classes+5))
+
+    x = compose(
+            DarknetConv2D_BN_Leaky(256, (1,1)),
+            UpSampling2D(2))(x)
+    x = Concatenate()([x,darknet.layers[152].output])
+    x, y2 = make_last_layers(x, 256, num_anchors*(num_classes+5))
+
+    x = compose(
+            DarknetConv2D_BN_Leaky(128, (1,1)),
+            UpSampling2D(2))(x)
+    x = Concatenate()([x,darknet.layers[92].output])
+    x, y3 = make_last_layers(x, 128, num_anchors*(num_classes+5))
+
+    return Model(inputs, [y1,y2,y3])
+
+
+def tiny_yolo_body(inputs, num_anchors, num_classes):
+    """Create Tiny YOLO_v3 model CNN body in keras."""
+    x1 = compose(
+            DarknetConv2D_BN_Leaky(16, (3,3)),
+            MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'),
+            DarknetConv2D_BN_Leaky(32, (3,3)),
+            MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'),
+            DarknetConv2D_BN_Leaky(64, (3,3)),
+            MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'),
+            DarknetConv2D_BN_Leaky(128, (3,3)),
+            MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'),
+            DarknetConv2D_BN_Leaky(256, (3,3)))(inputs)
+    x2 = compose(
+            MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'),
+            DarknetConv2D_BN_Leaky(512, (3,3)),
+            MaxPooling2D(pool_size=(2,2), strides=(1,1), padding='same'),
+            DarknetConv2D_BN_Leaky(1024, (3,3)),
+            DarknetConv2D_BN_Leaky(256, (1,1)))(x1)
+    y1 = compose(
+            DarknetConv2D_BN_Leaky(512, (3,3)),
+            DarknetConv2D(num_anchors*(num_classes+5), (1,1)))(x2)
+
+    x2 = compose(
+            DarknetConv2D_BN_Leaky(128, (1,1)),
+            UpSampling2D(2))(x2)
+    y2 = compose(
+            Concatenate(),
+            DarknetConv2D_BN_Leaky(256, (3,3)),
+            DarknetConv2D(num_anchors*(num_classes+5), (1,1)))([x2,x1])
+    model = Model(inputs, [y1,y2])
+    model.summary(120)
+    return model
+
+
+def tiny_yolo_se_body(inputs, num_anchors, num_classes):
+    """Create Tiny YOLO_v3 model CNN body in keras."""
+    x1 = compose(
+        DarknetConv2D_BN_Leaky(16, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(32, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(64, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(128, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(256, (3, 3)),
+    )(inputs)
+    x1 = SeBlock()(x1)
+
+    x2 = compose(
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(512, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(1, 1), padding='same'),
+        DarknetConv2D_BN_Leaky(1024, (3, 3)),
+        DarknetConv2D_BN_Leaky(256, (1, 1)),
+    )(x1)
+    x2 = SeBlock()(x2)
+
+    y1 = compose(
+        DarknetConv2D_BN_Leaky(512, (3, 3)),
+        DarknetConv2D(num_anchors*(num_classes+5), (1, 1))
+    )(x2)
+    y1 = SeBlock()(y1)
+
+    x2 = compose(
+        DarknetConv2D_BN_Leaky(128, (1, 1)),
+        UpSampling2D(2)
+    )(x2)
+    x2 = SeBlock()(x2)
+
+    y2 = compose(
+        Concatenate(),
+        DarknetConv2D_BN_Leaky(256, (3, 3)),
+        DarknetConv2D(num_anchors*(num_classes+5), (1, 1))
+    )([x2, x1])
+    y2 = SeBlock()(y2)
+
+    model = Model(inputs, [y1, y2])
+    model.summary(120)
+    return model
+
+
+def tinier_yolo_se_body(inputs, num_anchors, num_classes):
+    """Create Tiny YOLO_v3 model CNN body in keras."""
+    x1 = compose(
+        DarknetConv2D_BN_Leaky(8, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(16, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(32, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(64, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(128, (3, 3)),
+    )(inputs)
+    x1 = SeBlock()(x1)
+
+    x2 = compose(
+        MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
+        DarknetConv2D_BN_Leaky(256, (3, 3)),
+        MaxPooling2D(pool_size=(2, 2), strides=(1, 1), padding='same'),
+        DarknetConv2D_BN_Leaky(512, (3, 3)),
+        DarknetConv2D_BN_Leaky(128, (1, 1)),
+    )(x1)
+    x2 = SeBlock()(x2)
+
+    y1 = compose(
+        DarknetConv2D_BN_Leaky(256, (3, 3)),
+        DarknetConv2D(num_anchors*(num_classes+5), (1, 1))
+    )(x2)
+    y1 = SeBlock()(y1)
+
+    x2 = compose(
+        DarknetConv2D_BN_Leaky(64, (1, 1)),
+        UpSampling2D(2)
+    )(x2)
+    x2 = SeBlock()(x2)
+
+    y2 = compose(
+        Concatenate(),
+        DarknetConv2D_BN_Leaky(128, (3, 3)),
+        DarknetConv2D(num_anchors*(num_classes+5), (1, 1))
+    )([x2, x1])
+    y2 = SeBlock()(y2)
+
+    model = Model(inputs, [y1, y2])
+    model.summary(120)
+    return model
+
+
+def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
+    """Convert final layer features to bounding box parameters."""
+    num_anchors = len(anchors)
+    # Reshape to batch, height, width, num_anchors, box_params.
+    anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
+
+    grid_shape = K.shape(feats)[1:3] # height, width
+    grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
+        [1, grid_shape[1], 1, 1])
+    grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
+        [grid_shape[0], 1, 1, 1])
+    grid = K.concatenate([grid_x, grid_y])
+    grid = K.cast(grid, K.dtype(feats))
+
+    feats = K.reshape(
+        feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
+
+    # Adjust preditions to each spatial grid point and anchor size.
+    box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
+    box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
+    box_confidence = K.sigmoid(feats[..., 4:5])
+    box_class_probs = K.sigmoid(feats[..., 5:])
+
+    if calc_loss == True:
+        return grid, feats, box_xy, box_wh
+    return box_xy, box_wh, box_confidence, box_class_probs
+
+
+def yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape):
+    '''Get corrected boxes'''
+    box_yx = box_xy[..., ::-1]
+    box_hw = box_wh[..., ::-1]
+    input_shape = K.cast(input_shape, K.dtype(box_yx))
+    image_shape = K.cast(image_shape, K.dtype(box_yx))
+    new_shape = K.round(image_shape * K.min(input_shape/image_shape))
+    offset = (input_shape-new_shape)/2./input_shape
+    scale = input_shape/new_shape
+    box_yx = (box_yx - offset) * scale
+    box_hw *= scale
+
+    box_mins = box_yx - (box_hw / 2.)
+    box_maxes = box_yx + (box_hw / 2.)
+    boxes =  K.concatenate([
+        box_mins[..., 0:1],  # y_min
+        box_mins[..., 1:2],  # x_min
+        box_maxes[..., 0:1],  # y_max
+        box_maxes[..., 1:2]  # x_max
+    ])
+
+    # Scale boxes back to original image shape.
+    boxes *= K.concatenate([image_shape, image_shape])
+    return boxes
+
+
+def yolo_boxes_and_scores(feats, anchors, num_classes, input_shape, image_shape):
+    '''Process Conv layer output'''
+    box_xy, box_wh, box_confidence, box_class_probs = yolo_head(feats,
+        anchors, num_classes, input_shape)
+    boxes = yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape)
+    boxes = K.reshape(boxes, [-1, 4])
+    box_scores = box_confidence * box_class_probs
+    box_scores = K.reshape(box_scores, [-1, num_classes])
+    return boxes, box_scores
+
+
+def yolo_eval(yolo_outputs,
+              anchors,
+              num_classes,
+              image_shape,
+              max_boxes=20,
+              score_threshold=.6,
+              iou_threshold=.5):
+    """Evaluate YOLO model on given input and return filtered boxes."""
+    num_layers = len(yolo_outputs)
+    print("yolo_outputs", yolo_outputs[0])
+    print("num_layers", num_layers)
+    print("image_shape", image_shape)
+
+    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]] # default setting
+    input_shape = K.shape(yolo_outputs[0])[1:3] * 32
+    boxes = []
+    box_scores = []
+    for l in range(num_layers):
+        _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
+            anchors[anchor_mask[l]], num_classes, input_shape, image_shape)
+        boxes.append(_boxes)
+        box_scores.append(_box_scores)
+    boxes = K.concatenate(boxes, axis=0)
+    box_scores = K.concatenate(box_scores, axis=0)
+
+    mask = box_scores >= score_threshold
+    max_boxes_tensor = K.constant(max_boxes, dtype='int32')
+    boxes_ = []
+    scores_ = []
+    classes_ = []
+    for c in range(num_classes):
+        # TODO: use keras backend instead of tf.
+        class_boxes = tf.boolean_mask(boxes, mask[:, c])
+        class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
+        nms_index = tf.image.non_max_suppression(
+            class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)
+        class_boxes = K.gather(class_boxes, nms_index)
+        class_box_scores = K.gather(class_box_scores, nms_index)
+        classes = K.ones_like(class_box_scores, 'int32') * c
+        boxes_.append(class_boxes)
+        scores_.append(class_box_scores)
+        classes_.append(classes)
+    boxes_ = K.concatenate(boxes_, axis=0)
+    scores_ = K.concatenate(scores_, axis=0)
+    classes_ = K.concatenate(classes_, axis=0)
+
+    return boxes_, scores_, classes_
+
+
+def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes):
+    '''Preprocess true boxes to training input format
+
+    Parameters
+    ----------
+    true_boxes: array, shape=(m, T, 5)
+        Absolute x_min, y_min, x_max, y_max, class_id relative to input_shape.
+    input_shape: array-like, hw, multiples of 32
+    anchors: array, shape=(N, 2), wh
+    num_classes: integer
+
+    Returns
+    -------
+    y_true: list of array, shape like yolo_outputs, xywh are reletive value
+
+    '''
+    assert (true_boxes[..., 4]<num_classes).all(), 'class id must be less than num_classes'
+    num_layers = len(anchors)//3 # default setting
+    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]]
+
+    true_boxes = np.array(true_boxes, dtype='float32')
+    input_shape = np.array(input_shape, dtype='int32')
+    boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2
+    boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2]
+    true_boxes[..., 0:2] = boxes_xy/input_shape[::-1]
+    true_boxes[..., 2:4] = boxes_wh/input_shape[::-1]
+
+    m = true_boxes.shape[0]
+    grid_shapes = [input_shape//{0:32, 1:16, 2:8}[l] for l in range(num_layers)]
+    y_true = [np.zeros((m,grid_shapes[l][0],grid_shapes[l][1],len(anchor_mask[l]),5+num_classes),
+        dtype='float32') for l in range(num_layers)]
+
+    # Expand dim to apply broadcasting.
+    anchors = np.expand_dims(anchors, 0)
+    anchor_maxes = anchors / 2.
+    anchor_mins = -anchor_maxes
+    valid_mask = boxes_wh[..., 0]>0
+
+    for b in range(m):
+        # Discard zero rows.
+        wh = boxes_wh[b, valid_mask[b]]
+        if len(wh)==0: continue
+        # Expand dim to apply broadcasting.
+        wh = np.expand_dims(wh, -2)
+        box_maxes = wh / 2.
+        box_mins = -box_maxes
+
+        intersect_mins = np.maximum(box_mins, anchor_mins)
+        intersect_maxes = np.minimum(box_maxes, anchor_maxes)
+        intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.)
+        intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
+        box_area = wh[..., 0] * wh[..., 1]
+        anchor_area = anchors[..., 0] * anchors[..., 1]
+        iou = intersect_area / (box_area + anchor_area - intersect_area)
+
+        # Find best anchor for each true box
+        best_anchor = np.argmax(iou, axis=-1)
+
+        for t, n in enumerate(best_anchor):
+            for l in range(num_layers):
+                if n in anchor_mask[l]:
+                    i = np.floor(true_boxes[b,t,0]*grid_shapes[l][1]).astype('int32')
+                    j = np.floor(true_boxes[b,t,1]*grid_shapes[l][0]).astype('int32')
+                    k = anchor_mask[l].index(n)
+                    c = true_boxes[b,t, 4].astype('int32')
+                    y_true[l][b, j, i, k, 0:4] = true_boxes[b,t, 0:4]
+                    y_true[l][b, j, i, k, 4] = 1
+                    y_true[l][b, j, i, k, 5+c] = 1
+
+    return y_true
+
+
+def box_iou(b1, b2):
+    '''Return iou tensor
+
+    Parameters
+    ----------
+    b1: tensor, shape=(i1,...,iN, 4), xywh
+    b2: tensor, shape=(j, 4), xywh
+
+    Returns
+    -------
+    iou: tensor, shape=(i1,...,iN, j)
+
+    '''
+
+    # Expand dim to apply broadcasting.
+    b1 = K.expand_dims(b1, -2)
+    b1_xy = b1[..., :2]
+    b1_wh = b1[..., 2:4]
+    b1_wh_half = b1_wh/2.
+    b1_mins = b1_xy - b1_wh_half
+    b1_maxes = b1_xy + b1_wh_half
+
+    # Expand dim to apply broadcasting.
+    b2 = K.expand_dims(b2, 0)
+    b2_xy = b2[..., :2]
+    b2_wh = b2[..., 2:4]
+    b2_wh_half = b2_wh/2.
+    b2_mins = b2_xy - b2_wh_half
+    b2_maxes = b2_xy + b2_wh_half
+
+    intersect_mins = K.maximum(b1_mins, b2_mins)
+    intersect_maxes = K.minimum(b1_maxes, b2_maxes)
+    intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.)
+    intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
+    b1_area = b1_wh[..., 0] * b1_wh[..., 1]
+    b2_area = b2_wh[..., 0] * b2_wh[..., 1]
+    iou = intersect_area / (b1_area + b2_area - intersect_area)
+
+    return iou
+
+
+def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
+    """Return yolo_loss tensor
+
+    Parameters
+    ----------
+    yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
+    y_true: list of array, the output of preprocess_true_boxes
+    anchors: array, shape=(N, 2), wh
+    num_classes: integer
+    ignore_thresh: float, the iou threshold whether to ignore object confidence loss
+
+    Returns
+    -------
+    loss: tensor, shape=(1,)
+
+    """
+    num_layers = len(anchors)//3 # default setting
+    yolo_outputs = args[:num_layers]
+    y_true = args[num_layers:]
+    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]]
+    input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
+    grid_shapes = [K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers)]
+    loss = 0
+    m = K.shape(yolo_outputs[0])[0] # batch size, tensor
+    mf = K.cast(m, K.dtype(yolo_outputs[0]))
+
+    for l in range(num_layers):
+        object_mask = y_true[l][..., 4:5]
+        true_class_probs = y_true[l][..., 5:]
+
+        grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
+             anchors[anchor_mask[l]], num_classes, input_shape, calc_loss=True)
+        pred_box = K.concatenate([pred_xy, pred_wh])
+
+        # Darknet raw box to calculate loss.
+        raw_true_xy = y_true[l][..., :2]*grid_shapes[l][::-1] - grid
+        raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] * input_shape[::-1])
+        raw_true_wh = K.switch(object_mask, raw_true_wh, K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
+        box_loss_scale = 2 - y_true[l][...,2:3]*y_true[l][...,3:4]
+
+        # Find ignore mask, iterate over each of batch.
+        ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True)
+        object_mask_bool = K.cast(object_mask, 'bool')
+        def loop_body(b, ignore_mask):
+            true_box = tf.boolean_mask(y_true[l][b,...,0:4], object_mask_bool[b,...,0])
+            iou = box_iou(pred_box[b], true_box)
+            best_iou = K.max(iou, axis=-1)
+            ignore_mask = ignore_mask.write(b, K.cast(best_iou<ignore_thresh, K.dtype(true_box)))
+            return b+1, ignore_mask
+
+        # keras2.6.0 and keras2.1.5
+        # _, ignore_mask = K.control_flow_ops.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask])
+        _, ignore_mask = tf.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask])
+        ignore_mask = ignore_mask.stack()
+        ignore_mask = K.expand_dims(ignore_mask, -1)
+
+        # K.binary_crossentropy is helpful to avoid exp overflow.
+        xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(raw_true_xy, raw_pred[...,0:2], from_logits=True)
+        wh_loss = object_mask * box_loss_scale * 0.5 * K.square(raw_true_wh-raw_pred[...,2:4])
+        confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
+            (1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
+        class_loss = object_mask * K.binary_crossentropy(true_class_probs, raw_pred[...,5:], from_logits=True)
+
+        xy_loss = K.sum(xy_loss) / mf
+        wh_loss = K.sum(wh_loss) / mf
+        confidence_loss = K.sum(confidence_loss) / mf
+        class_loss = K.sum(class_loss) / mf
+        loss += xy_loss + wh_loss + confidence_loss + class_loss
+        # loss += (xy_loss + wh_loss + confidence_loss) * 2
+        # loss += xy_loss + confidence_loss + 2*wh_loss
+        # loss += xy_loss * 10 + wh_loss * 10 + confidence_loss
+        if print_loss:
+            loss = tf.Print(loss, [loss, xy_loss, wh_loss, confidence_loss, class_loss, K.sum(ignore_mask)], message='loss: ')
+    return loss
+
+
+def new_yolo_loss(input_shape, anchors, num_classes, ignore_thresh=.8, print_loss=False):
+    """Return yolo_loss tensor
+    """
+    def yolo_loss_fixed(y_true, y_pred):
+        num_layers = len(anchors)//3 # default setting
+        yolo_outputs = y_pred
+        print("y_true.shape", y_true.shape)
+        print("y_pred.shape", y_pred.shape)
+        anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
+        input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
+        grid_shapes = [K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers)]
+        loss = 0
+        m = K.shape(yolo_outputs[0])[0] # batch size, tensor
+        mf = K.cast(m, K.dtype(yolo_outputs[0]))
+
+        for l in range(num_layers):
+            object_mask = y_true[l][..., 4:5]
+            true_class_probs = y_true[l][..., 5:]
+
+            grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
+                                                         anchors[anchor_mask[l]], num_classes, input_shape, calc_loss=True)
+            pred_box = K.concatenate([pred_xy, pred_wh])
+
+            # Darknet raw box to calculate loss.
+            raw_true_xy = y_true[l][..., :2]*grid_shapes[l][::-1] - grid
+            raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] * input_shape[::-1])
+            raw_true_wh = K.switch(object_mask, raw_true_wh, K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
+            box_loss_scale = 2 - y_true[l][...,2:3]*y_true[l][...,3:4]
+
+            # Find ignore mask, iterate over each of batch.
+            ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True)
+            object_mask_bool = K.cast(object_mask, 'bool')
+            def loop_body(b, ignore_mask):
+                true_box = tf.boolean_mask(y_true[l][b,...,0:4], object_mask_bool[b,...,0])
+                iou = box_iou(pred_box[b], true_box)
+                best_iou = K.max(iou, axis=-1)
+                ignore_mask = ignore_mask.write(b, K.cast(best_iou<ignore_thresh, K.dtype(true_box)))
+                return b+1, ignore_mask
+            _, ignore_mask = K.control_flow_ops.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask])
+            ignore_mask = ignore_mask.stack()
+            ignore_mask = K.expand_dims(ignore_mask, -1)
+
+            # K.binary_crossentropy is helpful to avoid exp overflow.
+            xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(raw_true_xy, raw_pred[...,0:2], from_logits=True)
+            wh_loss = object_mask * box_loss_scale * 0.5 * K.square(raw_true_wh-raw_pred[...,2:4])
+            confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
+                              (1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
+            class_loss = object_mask * K.binary_crossentropy(true_class_probs, raw_pred[...,5:], from_logits=True)
+
+            xy_loss = K.sum(xy_loss) / mf
+            wh_loss = K.sum(wh_loss) / mf
+            confidence_loss = K.sum(confidence_loss) / mf
+            class_loss = K.sum(class_loss) / mf
+            # loss += xy_loss + wh_loss + confidence_loss + class_loss
+            loss += (xy_loss + wh_loss + confidence_loss) * 2
+            if print_loss:
+                loss = tf.Print(loss, [loss, xy_loss, wh_loss, confidence_loss, class_loss, K.sum(ignore_mask)], message='loss: ')
+        return loss
+
+    # h, w = input_shape
+    # y_true = [Input(shape=(h//{0: 32, 1: 16}[l], w//{0: 32, 1: 16}[l],
+    #           len(anchors)//2, num_classes+5)) for l in range(2)]
+
+    return yolo_loss_fixed
+

chinese_detect/post_process.py

@@ -0,0 +1,211 @@
+import numpy as np
+import cv2
+from PIL import Image
+from keras import backend as K
+import tensorflow as tf
+
+
+def return_seal_part(image_seal_list, num=3):
+    image_part_list = []
+    for image_part, image_seal, position in image_seal_list:
+        if image_seal is None:
+            image_part_list.append(image_part)
+            continue
+        y_min, y_max, x_min, x_max = position
+        image_part[y_min:y_max, x_min:x_max, :] = image_seal
+        image_part_list.append(image_part)
+
+    # 先将宽的部分拼接
+    image_width_list = []
+    for i in range(0, len(image_part_list), num):
+        image_width = np.concatenate([image_part_list[i], image_part_list[i + 1], image_part_list[i + 2]], axis=1)
+        image_width_list.append(image_width)
+    # 高的部分拼接
+    image_return = np.concatenate([x for x in image_width_list], axis=0)
+    print("image_return.shape", image_return.shape)
+    cv2.namedWindow("image_return", 0)
+    cv2.resizeWindow("image_return", 1000, 800)
+    cv2.imshow("image_return", image_return)
+    cv2.waitKey(0)
+    return image_return
+
+
+def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
+    """Convert final layer features to bounding box parameters."""
+    num_anchors = len(anchors)
+    # feats = K.constant(feats)
+    # Reshape to batch, height, width, num_anchors, box_params.
+    anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
+
+    grid_shape = K.shape(feats)[1:3]  # height, width
+    grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
+                    [1, grid_shape[1], 1, 1])
+    grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
+                    [grid_shape[0], 1, 1, 1])
+    grid = K.concatenate([grid_x, grid_y])
+    grid = K.cast(grid, K.dtype(feats))
+
+    feats = K.reshape(
+        feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
+
+    # Adjust predictions to each spatial grid point and anchor size.
+    box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
+    box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
+    box_confidence = K.sigmoid(feats[..., 4:5])
+    box_class_probs = K.sigmoid(feats[..., 5:])
+
+    if calc_loss:
+        return grid, feats, box_xy, box_wh
+    return box_xy, box_wh, box_confidence, box_class_probs
+
+
+def yolo_eval(outputs,
+              anchors,
+              num_classes,
+              max_boxes=10,
+              score_threshold=.1,
+              iou_threshold=.1):
+    """Evaluate YOLO model on given input and return filtered boxes."""
+    # num_layers = len(anchors) // 3
+    # yolo_outputs = outputs[:num_layers]
+    # image_shape = outputs[num_layers]
+
+    yolo_outputs = outputs[0]
+    print("yolo_outputs", yolo_outputs[0])
+    num_layers = len(yolo_outputs)
+    image_shape = outputs[1]
+
+    print("num_layers", num_layers)
+    print("image_shape", image_shape.shape)
+
+    anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]] # default setting
+    input_shape = K.shape(yolo_outputs[0])[1:3] * 32
+    boxes = []
+    box_scores = []
+    for l in range(num_layers):
+        _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
+                                                    anchors[anchor_mask[l]],
+                                                    num_classes, input_shape, image_shape)
+        boxes.append(_boxes)
+        box_scores.append(_box_scores)
+    boxes = K.concatenate(boxes, axis=0)
+    box_scores = K.concatenate(box_scores, axis=0)
+
+    mask = box_scores >= score_threshold
+    max_boxes_tensor = K.constant(max_boxes, dtype='int32')
+    boxes_ = []
+    scores_ = []
+    classes_ = []
+    for c in range(num_classes):
+        # TODO: use keras backend instead of tf.
+        class_boxes = tf.boolean_mask(boxes, mask[:, c])
+        class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
+        nms_index = tf.image.non_max_suppression(
+            class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)
+        class_boxes = K.gather(class_boxes, nms_index)
+        class_box_scores = K.gather(class_box_scores, nms_index)
+        classes = K.ones_like(class_box_scores, 'int32') * c
+        boxes_.append(class_boxes)
+        scores_.append(class_box_scores)
+        classes_.append(classes)
+    boxes_ = K.concatenate(boxes_, axis=0)
+    scores_ = K.concatenate(scores_, axis=0)
+    classes_ = K.concatenate(classes_, axis=0)
+
+    return boxes_, scores_, classes_
+
+
+def letterbox_image(image, size):
+    '''resize image with unchanged aspect ratio using padding'''
+    iw, ih = image.size
+    w, h = size
+    scale = min(w/iw, h/ih)
+    nw = int(iw*scale)
+    nh = int(ih*scale)
+
+    image = image.resize((nw,nh), Image.BICUBIC)
+    new_image = Image.new('RGB', size, (128,128,128))
+    new_image.paste(image, ((w-nw)//2, (h-nh)//2))
+    return new_image
+
+
+def yolo_eval_new(yolo_outputs,
+              anchors,
+              num_classes,
+              image_shape,
+              max_boxes=20,
+              score_threshold=.6,
+              iou_threshold=.5):
+    """Evaluate YOLO model on given input and return filtered boxes."""
+    num_layers = len(yolo_outputs)
+    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]] # default setting
+    input_shape = K.shape(yolo_outputs[0])[1:3] * 32
+    boxes = []
+    box_scores = []
+    for l in range(num_layers):
+        _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
+                                                    anchors[anchor_mask[l]], num_classes, input_shape, image_shape)
+        boxes.append(_boxes)
+        box_scores.append(_box_scores)
+    boxes = K.concatenate(boxes, axis=0)
+    box_scores = K.concatenate(box_scores, axis=0)
+
+    mask = box_scores >= score_threshold
+    max_boxes_tensor = K.constant(max_boxes, dtype='int32')
+    boxes_ = []
+    scores_ = []
+    classes_ = []
+    for c in range(num_classes):
+        # TODO: use keras backend instead of tf.
+        class_boxes = tf.boolean_mask(boxes, mask[:, c])
+        class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
+        nms_index = tf.image.non_max_suppression(
+            class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)
+        class_boxes = K.gather(class_boxes, nms_index)
+        class_box_scores = K.gather(class_box_scores, nms_index)
+        classes = K.ones_like(class_box_scores, 'int32') * c
+        boxes_.append(class_boxes)
+        scores_.append(class_box_scores)
+        classes_.append(classes)
+    boxes_ = K.concatenate(boxes_, axis=0)
+    scores_ = K.concatenate(scores_, axis=0)
+    classes_ = K.concatenate(classes_, axis=0)
+
+    return boxes_, scores_, classes_
+
+
+def yolo_boxes_and_scores(feats, anchors, num_classes, input_shape, image_shape):
+    """Process Conv layer output"""
+    box_xy, box_wh, box_confidence, box_class_probs = yolo_head(feats,
+                                                                anchors, num_classes, input_shape)
+    boxes = yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape)
+    boxes = K.reshape(boxes, [-1, 4])
+    box_scores = box_confidence * box_class_probs
+    box_scores = K.reshape(box_scores, [-1, num_classes])
+    return boxes, box_scores
+
+
+def yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape):
+    """Get corrected boxes"""
+    box_yx = box_xy[..., ::-1]
+    box_hw = box_wh[..., ::-1]
+    input_shape = K.cast(input_shape, K.dtype(box_yx))
+    image_shape = K.cast(image_shape, K.dtype(box_yx))
+    new_shape = K.round(image_shape * K.min(input_shape/image_shape))
+    offset = (input_shape-new_shape)/2./input_shape
+    scale = input_shape/new_shape
+    box_yx = (box_yx - offset) * scale
+    box_hw *= scale
+
+    box_mins = box_yx - (box_hw / 2.)
+    box_maxes = box_yx + (box_hw / 2.)
+    boxes =  K.concatenate([
+        box_mins[..., 0:1],  # y_min
+        box_mins[..., 1:2],  # x_min
+        box_maxes[..., 0:1],  # y_max
+        box_maxes[..., 1:2]  # x_max
+    ])
+
+    # Scale boxes back to original image shape.
+    boxes *= K.concatenate([image_shape, image_shape])
+    return boxes

chinese_detect/yolo_data/convert.py

@@ -0,0 +1,262 @@
+#! /usr/bin/env python
+"""
+Reads Darknet config and weights and creates Keras model with TF backend.
+
+"""
+
+import argparse
+import configparser
+import io
+import os
+from collections import defaultdict
+
+import numpy as np
+from keras import backend as K
+from keras.layers import (Conv2D, Input, ZeroPadding2D, Add,
+                          UpSampling2D, MaxPooling2D, Concatenate)
+from keras.layers.advanced_activations import LeakyReLU
+from keras.layers import BatchNormalization
+from keras.models import Model
+from keras.regularizers import l2
+from keras.utils.vis_utils import plot_model as plot
+
+
+parser = argparse.ArgumentParser(description='Darknet To Keras Converter.')
+parser.add_argument('config_path', help='Path to Darknet cfg file.')
+parser.add_argument('weights_path', help='Path to Darknet weights file.')
+parser.add_argument('output_path', help='Path to output Keras model file.')
+parser.add_argument(
+    '-p',
+    '--plot_model',
+    help='Plot generated Keras model and save as image.',
+    action='store_true')
+parser.add_argument(
+    '-w',
+    '--weights_only',
+    help='Save as Keras weights file instead of model file.',
+    action='store_true')
+
+def unique_config_sections(config_file):
+    """Convert all config sections to have unique names.
+
+    Adds unique suffixes to config sections for compability with configparser.
+    """
+    section_counters = defaultdict(int)
+    output_stream = io.StringIO()
+    with open(config_file) as fin:
+        for line in fin:
+            if line.startswith('['):
+                section = line.strip().strip('[]')
+                _section = section + '_' + str(section_counters[section])
+                section_counters[section] += 1
+                line = line.replace(section, _section)
+            output_stream.write(line)
+    output_stream.seek(0)
+    return output_stream
+
+# %%
+def _main(args):
+    config_path = os.path.expanduser(args.config_path)
+    weights_path = os.path.expanduser(args.weights_path)
+    assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
+        config_path)
+    assert weights_path.endswith(
+        '.weights'), '{} is not a .weights file'.format(weights_path)
+
+    output_path = os.path.expanduser(args.output_path)
+    assert output_path.endswith(
+        '.h5'), 'output path {} is not a .h5 file'.format(output_path)
+    output_root = os.path.splitext(output_path)[0]
+
+    # Load weights and config.
+    print('Loading weights.')
+    weights_file = open(weights_path, 'rb')
+    major, minor, revision = np.ndarray(
+        shape=(3, ), dtype='int32', buffer=weights_file.read(12))
+    if (major*10+minor)>=2 and major<1000 and minor<1000:
+        seen = np.ndarray(shape=(1,), dtype='int64', buffer=weights_file.read(8))
+    else:
+        seen = np.ndarray(shape=(1,), dtype='int32', buffer=weights_file.read(4))
+    print('Weights Header: ', major, minor, revision, seen)
+
+    print('Parsing Darknet config.')
+    unique_config_file = unique_config_sections(config_path)
+    cfg_parser = configparser.ConfigParser()
+    cfg_parser.read_file(unique_config_file)
+
+    print('Creating Keras model.')
+    input_layer = Input(shape=(None, None, 3))
+    prev_layer = input_layer
+    all_layers = []
+
+    weight_decay = float(cfg_parser['net_0']['decay']
+                         ) if 'net_0' in cfg_parser.sections() else 5e-4
+    count = 0
+    out_index = []
+    for section in cfg_parser.sections():
+        print('Parsing section {}'.format(section))
+        if section.startswith('convolutional'):
+            filters = int(cfg_parser[section]['filters'])
+            size = int(cfg_parser[section]['size'])
+            stride = int(cfg_parser[section]['stride'])
+            pad = int(cfg_parser[section]['pad'])
+            activation = cfg_parser[section]['activation']
+            batch_normalize = 'batch_normalize' in cfg_parser[section]
+
+            padding = 'same' if pad == 1 and stride == 1 else 'valid'
+
+            # Setting weights.
+            # Darknet serializes convolutional weights as:
+            # [bias/beta, [gamma, mean, variance], conv_weights]
+            prev_layer_shape = K.int_shape(prev_layer)
+
+            weights_shape = (size, size, prev_layer_shape[-1], filters)
+            darknet_w_shape = (filters, weights_shape[2], size, size)
+            weights_size = np.product(weights_shape)
+
+            print('conv2d', 'bn'
+            if batch_normalize else '  ', activation, weights_shape)
+
+            conv_bias = np.ndarray(
+                shape=(filters, ),
+                dtype='float32',
+                buffer=weights_file.read(filters * 4))
+            count += filters
+
+            if batch_normalize:
+                bn_weights = np.ndarray(
+                    shape=(3, filters),
+                    dtype='float32',
+                    buffer=weights_file.read(filters * 12))
+                count += 3 * filters
+
+                bn_weight_list = [
+                    bn_weights[0],  # scale gamma
+                    conv_bias,  # shift beta
+                    bn_weights[1],  # running mean
+                    bn_weights[2]  # running var
+                ]
+
+            conv_weights = np.ndarray(
+                shape=darknet_w_shape,
+                dtype='float32',
+                buffer=weights_file.read(weights_size * 4))
+            count += weights_size
+
+            # DarkNet conv_weights are serialized Caffe-style:
+            # (out_dim, in_dim, height, width)
+            # We would like to set these to Tensorflow order:
+            # (height, width, in_dim, out_dim)
+            conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
+            conv_weights = [conv_weights] if batch_normalize else [
+                conv_weights, conv_bias
+            ]
+
+            # Handle activation.
+            act_fn = None
+            if activation == 'leaky':
+                pass  # Add advanced activation later.
+            elif activation != 'linear':
+                raise ValueError(
+                    'Unknown activation function `{}` in section {}'.format(
+                        activation, section))
+
+            # Create Conv2D layer
+            if stride>1:
+                # Darknet uses left and top padding instead of 'same' mode
+                prev_layer = ZeroPadding2D(((1,0),(1,0)))(prev_layer)
+            conv_layer = (Conv2D(
+                filters, (size, size),
+                strides=(stride, stride),
+                kernel_regularizer=l2(weight_decay),
+                use_bias=not batch_normalize,
+                weights=conv_weights,
+                activation=act_fn,
+                padding=padding))(prev_layer)
+
+            if batch_normalize:
+                conv_layer = (BatchNormalization(
+                    weights=bn_weight_list))(conv_layer)
+            prev_layer = conv_layer
+
+            if activation == 'linear':
+                all_layers.append(prev_layer)
+            elif activation == 'leaky':
+                act_layer = LeakyReLU(alpha=0.1)(prev_layer)
+                prev_layer = act_layer
+                all_layers.append(act_layer)
+
+        elif section.startswith('route'):
+            ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
+            layers = [all_layers[i] for i in ids]
+            if len(layers) > 1:
+                print('Concatenating route layers:', layers)
+                concatenate_layer = Concatenate()(layers)
+                all_layers.append(concatenate_layer)
+                prev_layer = concatenate_layer
+            else:
+                skip_layer = layers[0]  # only one layer to route
+                all_layers.append(skip_layer)
+                prev_layer = skip_layer
+
+        elif section.startswith('maxpool'):
+            size = int(cfg_parser[section]['size'])
+            stride = int(cfg_parser[section]['stride'])
+            all_layers.append(
+                MaxPooling2D(
+                    pool_size=(size, size),
+                    strides=(stride, stride),
+                    padding='same')(prev_layer))
+            prev_layer = all_layers[-1]
+
+        elif section.startswith('shortcut'):
+            index = int(cfg_parser[section]['from'])
+            activation = cfg_parser[section]['activation']
+            assert activation == 'linear', 'Only linear activation supported.'
+            all_layers.append(Add()([all_layers[index], prev_layer]))
+            prev_layer = all_layers[-1]
+
+        elif section.startswith('upsample'):
+            stride = int(cfg_parser[section]['stride'])
+            assert stride == 2, 'Only stride=2 supported.'
+            all_layers.append(UpSampling2D(stride)(prev_layer))
+            prev_layer = all_layers[-1]
+
+        elif section.startswith('yolo'):
+            out_index.append(len(all_layers)-1)
+            all_layers.append(None)
+            prev_layer = all_layers[-1]
+
+        elif section.startswith('net'):
+            pass
+
+        else:
+            raise ValueError(
+                'Unsupported section header type: {}'.format(section))
+
+    # Create and save model.
+    if len(out_index)==0: out_index.append(len(all_layers)-1)
+    model = Model(inputs=input_layer, outputs=[all_layers[i] for i in out_index])
+    print(model.summary())
+    if args.weights_only:
+        model.save_weights('{}'.format(output_path))
+        print('Saved Keras weights to {}'.format(output_path))
+    else:
+        model.save('{}'.format(output_path))
+        print('Saved Keras model to {}'.format(output_path))
+
+    # Check to see if all weights have been read.
+    remaining_weights = len(weights_file.read()) / 4
+    weights_file.close()
+    print('Read {} of {} from Darknet weights.'.format(count, count +
+                                                       remaining_weights))
+    if remaining_weights > 0:
+        print('Warning: {} unused weights'.format(remaining_weights))
+
+    if args.plot_model:
+        plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
+        print('Saved model plot to {}.png'.format(output_root))
+
+
+if __name__ == '__main__':
+    _main(parser.parse_args())

chinese_detect/yolo_data/kmeans.py

@@ -0,0 +1,101 @@
+import numpy as np
+
+
+class YOLO_Kmeans:
+
+    def __init__(self, cluster_number, filename):
+        self.cluster_number = cluster_number
+        self.filename = filename
+
+    def iou(self, boxes, clusters):  # 1 box -> k clusters
+        n = boxes.shape[0]
+        k = self.cluster_number
+
+        box_area = boxes[:, 0] * boxes[:, 1]
+        box_area = box_area.repeat(k)
+        box_area = np.reshape(box_area, (n, k))
+
+        cluster_area = clusters[:, 0] * clusters[:, 1]
+        cluster_area = np.tile(cluster_area, [1, n])
+        cluster_area = np.reshape(cluster_area, (n, k))
+
+        box_w_matrix = np.reshape(boxes[:, 0].repeat(k), (n, k))
+        cluster_w_matrix = np.reshape(np.tile(clusters[:, 0], (1, n)), (n, k))
+        min_w_matrix = np.minimum(cluster_w_matrix, box_w_matrix)
+
+        box_h_matrix = np.reshape(boxes[:, 1].repeat(k), (n, k))
+        cluster_h_matrix = np.reshape(np.tile(clusters[:, 1], (1, n)), (n, k))
+        min_h_matrix = np.minimum(cluster_h_matrix, box_h_matrix)
+        inter_area = np.multiply(min_w_matrix, min_h_matrix)
+
+        result = inter_area / (box_area + cluster_area - inter_area)
+        return result
+
+    def avg_iou(self, boxes, clusters):
+        accuracy = np.mean([np.max(self.iou(boxes, clusters), axis=1)])
+        return accuracy
+
+    def kmeans(self, boxes, k, dist=np.median):
+        box_number = boxes.shape[0]
+        distances = np.empty((box_number, k))
+        last_nearest = np.zeros((box_number,))
+        np.random.seed()
+        clusters = boxes[np.random.choice(
+            box_number, k, replace=False)]  # init k clusters
+        while True:
+
+            distances = 1 - self.iou(boxes, clusters)
+
+            current_nearest = np.argmin(distances, axis=1)
+            if (last_nearest == current_nearest).all():
+                break  # clusters won't change
+            for cluster in range(k):
+                clusters[cluster] = dist(  # update clusters
+                    boxes[current_nearest == cluster], axis=0)
+
+            last_nearest = current_nearest
+
+        return clusters
+
+    def result2txt(self, data):
+        f = open("yolo_anchors.txt", 'w')
+        row = np.shape(data)[0]
+        for i in range(row):
+            if i == 0:
+                x_y = "%d,%d" % (data[i][0], data[i][1])
+            else:
+                x_y = ", %d,%d" % (data[i][0], data[i][1])
+            f.write(x_y)
+        f.close()
+
+    def txt2boxes(self):
+        f = open(self.filename, 'r')
+        dataSet = []
+        for line in f:
+            infos = line.split(" ")
+            length = len(infos)
+            for i in range(1, length):
+                width = int(infos[i].split(",")[2]) - \
+                    int(infos[i].split(",")[0])
+                height = int(infos[i].split(",")[3]) - \
+                    int(infos[i].split(",")[1])
+                dataSet.append([width, height])
+        result = np.array(dataSet)
+        f.close()
+        return result
+
+    def txt2clusters(self):
+        all_boxes = self.txt2boxes()
+        result = self.kmeans(all_boxes, k=self.cluster_number)
+        result = result[np.lexsort(result.T[0, None])]
+        self.result2txt(result)
+        print("K anchors:\n {}".format(result))
+        print("Accuracy: {:.2f}%".format(
+            self.avg_iou(all_boxes, result) * 100))
+
+
+if __name__ == "__main__":
+    cluster_number = 6
+    filename = "../../data/detect2/map.txt"
+    kmeans = YOLO_Kmeans(cluster_number, filename)
+    kmeans.txt2clusters()

chinese_detect/yolo_data/my_anchors.txt

@@ -0,0 +1 @@
+36,36,  38,38,  40,40,  42,42,  44,44,  46,46

chinese_detect/yolo_data/my_classes.txt

@@ -0,0 +1 @@
+char

chinese_detect/yolo_data/yolov3-tiny.cfg

@@ -0,0 +1,182 @@
+[net]
+# Testing
+batch=1
+subdivisions=1
+# Training
+# batch=64
+# subdivisions=2
+width=416
+height=416
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=16
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=1
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+###########
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+
+[yolo]
+mask = 3,4,5
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 8
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+[yolo]
+mask = 1,2,3
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1

chinese_order/inference_phrase.py

@@ -0,0 +1,60 @@
+import os
+import random
+from itertools import product
+
+import cv2
+import numpy as np
+from click_captcha.model import mobile_net, cnn_net, u_net_drag, lstm_phrase, text_cnn_phrase
+from click_captcha.utils import pil_resize
+
+vocabulary_len = 5792
+sequence_len = 6
+weights_path = "./models/e08-f10.85-phrase.h5"
+project_dir = os.path.dirname(os.path.abspath(__file__)) + "/../"
+
+
+def recognize(char_list):
+    model = lstm_phrase((sequence_len, vocabulary_len))
+    model.load_weights(weights_path)
+
+    if len(char_list) > sequence_len:
+        return None
+
+    char_path = "../data/phrase/char.txt"
+    with open(char_path, "r") as f:
+        char_map_list = f.readlines()
+    char_dict = {}
+    char_dict_reverse = {}
+    for i in range(len(char_map_list)):
+        char_dict[char_map_list[i][:-1]] = i
+        char_dict_reverse[i] = char_map_list[i][:-1]
+    char_dict_reverse[vocabulary_len] = ""
+
+    index_list = [char_dict[x] for x in char_list]
+
+    products = list(product(index_list, repeat=len(index_list)))
+    all_index_list = []
+    for index_list in products:
+        index_list = list(index_list)
+        if len(set(index_list)) != len(index_list):
+            continue
+        index_list = index_list + [vocabulary_len]*(sequence_len-len(index_list))
+        all_index_list.append(index_list)
+
+    X = np.array(all_index_list)
+    pred = model.predict(X)
+    for i in range(len(pred)):
+        p = all_index_list[i]
+        print(pred[i], "".join([char_dict_reverse[x] for x in p]))
+    decode = all_index_list[int(np.argmax(pred))]
+    decode = "".join([char_dict_reverse[x] for x in decode])
+    print("".join(char_list))
+    print(decode)
+
+
+if __name__ == "__main__":
+    text = "吃饭了"
+    text = [x for x in text]
+    random.shuffle(text)
+    print("text", text)
+    recognize(text)

chinese_order/pre_process.py

@@ -0,0 +1,1322 @@
+import copy
+import json
+import os
+import random
+import re
+import sys
+import time
+import traceback
+from glob import glob
+from itertools import combinations, product
+
+import chardet
+import cv2
+import jieba
+import numpy as np
+from PIL import ImageFont, ImageDraw, Image
+from captcha.image import ImageCaptcha
+from keras_preprocessing.sequence import pad_sequences
+from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+from click_captcha.utils import np2pil, pil2np, pil_resize, pil_rotate, pil2np_a, np2pil_a, pil_resize_a, pil_rotate_a
+
+
+def gen_siamese(paths, batch_size=32, shape=(40, 40), cls_num=1):
+    num = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/click/"
+
+    i = 0
+    while True:
+        if i >= num:
+            i = 0
+            random.shuffle(paths)
+
+        height, width = shape[:2]
+        X1 = np.zeros((batch_size, height, width, 1))
+        X2 = np.zeros((batch_size, height, width, 1))
+        Y = np.zeros((batch_size, 2))
+
+        for j in range(batch_size):
+            # 生成标注数据
+            img1, img2, label = paths[i][:-1].split("\t")
+            # print(img1, img2, label)
+            img1 = cv2.imread(data_path + img1)
+            img1 = pil_resize(img1, shape[0], shape[1])
+            img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+            img1 = np.expand_dims(img1, axis=-1)
+            img2 = cv2.imread(data_path + img2)
+            img2 = pil_resize(img2, shape[0], shape[1])
+            img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
+            img2 = np.expand_dims(img2, axis=-1)
+            if label == "1":
+                label = np.array([0, 1])
+            else:
+                label = np.array([1, 0])
+            X1[j] = img1
+            X2[j] = img2
+            Y[j] = label
+
+        yield {"input_1": X1, "input_2": X2}, {"output": Y}
+
+
+def gen_mobile(paths, batch_size=32, shape=(40, 40), cls_num=5710, data_path="click"):
+    num = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/" + data_path + "/"
+
+    i = 0
+    random.shuffle(paths)
+    while True:
+        if i >= num:
+            i = 0
+            random.shuffle(paths)
+
+        height, width = shape[:2]
+        if len(shape) > 2:
+            channel = 3
+        else:
+            channel = 1
+        X = np.zeros((batch_size, height, width, channel))
+        Y = np.zeros((batch_size, cls_num))
+
+        j = 0
+        error_num = 0
+        while j < batch_size:
+        # for j in range(batch_size):
+            if i >= num:
+                random.shuffle(paths)
+                i = 0
+            path = paths[i].split(os.sep)[-1]
+            char_index = int(path.split("_")[0])
+            label = np.zeros(cls_num)
+            # print("char_index", char_index)
+            label[char_index] = 1
+            # print("label", np.argmax(label), char_index)
+
+
+            img1 = cv2.imread(data_path + path)
+            img1 = pil_resize(img1, shape[0], shape[1])
+            img1 = img1 / 255.
+            # img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+            # img1 = np.expand_dims(img1, axis=-1)
+
+            X[j] = img1
+            Y[j] = label
+            i += 1
+            j += 1
+        # print("error_num", error_num)
+        yield X, Y
+
+
+def gen_yolo_char(paths, batch_size, input_shape, anchors, num_classes, box_num=6):
+    """data generator for fit_generator"""
+    n = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/detect/"
+    i = 0
+    while True:
+        image_data = []
+        box_data = []
+
+        batch_cnt = 0
+        while batch_cnt < batch_size:
+            try:
+                if i == 0:
+                    np.random.shuffle(paths)
+
+                ss = paths[i][:-1].split(" ")
+                image_path = ss[0]
+                image = cv2.imread(data_path+image_path)
+                image = pil_resize(image, input_shape[0], input_shape[1])
+                image_show = copy.deepcopy(image)
+                image = image / 255.
+                box = np.array([np.array(list(map(int, box.split(',')))) for box in ss[1:]])
+
+                # box数不同，复制
+                if box.shape[0] < box_num:
+                    box = np.concatenate([box, box[:2, :]], axis=0)
+
+                # show
+                # box_show = box.tolist()
+                # for b in box_show:
+                #     print("box", b)
+                #     cv2.rectangle(image_show, (b[0], b[1]), (b[2], b[3]), (255, 0, 0), 2)
+                # cv2.imshow("image_show", image_show)
+                # cv2.waitKey(0)
+
+                image_data.append(image)
+                box_data.append(box)
+                i = (i+1) % n
+                batch_cnt += 1
+            except:
+                i = (i+1) % n
+                continue
+
+            # print
+            # print(image.shape)
+            # image_show = (image*255).astype(np.uint8)
+            # print("annotation_lines[i]", annotation_lines[i])
+            # for _b in box:
+            #     print(_b)
+            #     cv2.rectangle(image_show, (int(_b[0]), int(_b[1])), (int(_b[2]), int(_b[3])), (0, 255, 0), 1)
+            # cv2.imshow("image", image_show)
+            # cv2.waitKey(0)
+
+        image_data = np.array(image_data)
+        box_data = np.array(box_data)
+        # print(image_data.shape, box_data.shape)
+        y_true = preprocess_true_boxes(box_data, input_shape, anchors, num_classes)
+        yield [image_data, *y_true], np.zeros(batch_size)
+
+
+def gen_yolo_puzzle(paths, batch_size, input_shape, anchors, num_classes, box_num=1):
+    """data generator for fit_generator"""
+    n = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/detect2/"
+    i = 0
+    while True:
+        image_data = []
+        box_data = []
+
+        batch_cnt = 0
+        while batch_cnt < batch_size:
+            try:
+                if i == 0:
+                    np.random.shuffle(paths)
+
+                ss = paths[i][:-1].split(" ")
+                image_path = ss[0]
+                image = cv2.imread(data_path+image_path)
+                image = pil_resize(image, input_shape[0], input_shape[1])
+                image_show = copy.deepcopy(image)
+                image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+                image = 255. - image
+                image = np.uint8(image)
+                # cv2.imshow("image", image)
+                # cv2.waitKey(0)
+                image = np.expand_dims(image, -1)
+                image = image / 255.
+                box = np.array([np.array(list(map(int, box.split(',')))) for box in ss[1:]])
+
+                # box数不同，复制
+                if box.shape[0] < box_num:
+                    box = np.concatenate([box, box[:2, :]], axis=0)
+
+                # show
+                # box_show = box.tolist()
+                # for b in box_show:
+                #     print("box", b)
+                #     cv2.rectangle(image_show, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2)
+                # cv2.imshow("image_show", image_show)
+                # cv2.waitKey(0)
+
+                image_data.append(image)
+                box_data.append(box)
+                i = (i+1) % n
+                batch_cnt += 1
+            except:
+                i = (i+1) % n
+                continue
+
+            # print
+            # print(image.shape)
+            # image_show = (image*255).astype(np.uint8)
+            # for _b in box:
+            #     print(_b)
+            #     cv2.rectangle(image_show, (int(_b[0]), int(_b[1])), (int(_b[2]), int(_b[3])), (0, 255, 0), 1)
+            # cv2.imshow("image", image_show)
+            # cv2.waitKey(0)
+
+        image_data = np.array(image_data)
+        box_data = np.array(box_data)
+        # print(image_data.shape, box_data.shape)
+        y_true = preprocess_true_boxes(box_data, input_shape, anchors, num_classes)
+        yield [image_data, *y_true], np.zeros(batch_size)
+
+
+def gen_drag(paths, batch_size=32, shape=(128, 256), cls_num=2):
+    num = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/drag/"
+
+    map_path = data_path+"map.txt"
+    with open(map_path, "r") as f:
+        _list = f.readlines()
+    map_dict = {}
+    for s in _list:
+        ss = s[:-1].split(" ")
+        map_dict[ss[0]] = ss[1]
+
+    i = 0
+    random.shuffle(paths)
+    while True:
+        if i >= num:
+            i = 0
+            random.shuffle(paths)
+
+        height, width = shape[:2]
+        if len(shape) > 2:
+            channel = 3
+        else:
+            channel = 1
+        X = np.zeros((batch_size, height, width, channel))
+        Y = np.zeros((batch_size, height, width, 1))
+
+        for j in range(batch_size):
+            if i >= num:
+                random.shuffle(paths)
+                i = 0
+            path = paths[i].split(os.sep)[-1]
+            w_index = int(map_dict.get(path))
+            # label = np.zeros(cls_num)
+            # print("char_index", char_index)
+            # label[w_index] = 1
+            # print("label", np.argmax(label), char_index)
+
+            img1 = cv2.imread(data_path + path)
+            img1 = pil_resize(img1, shape[0], shape[1])
+            # cv2.imshow("image", img1)
+            label = np.full((shape[0], shape[1], 1), 0, dtype='uint8')
+            label[:, w_index, 0] = 1
+            # label[:, w_index, 1] = 1
+            # cv2.imshow("label", np.expand_dims(label[..., 0], -1))
+            # cv2.waitKey(0)
+            img1 = img1 / 255.
+            # img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+            # img1 = np.expand_dims(img1, axis=-1)
+            i += 1
+
+            X[j] = img1
+            Y[j] = label
+
+        yield X, Y
+
+
+def gen_phrase(map_list, batch_size=32, shape=(5707, 3)):
+    voc_dim, timesteps = shape[:2]
+
+    data_list = []
+    for line in map_list:
+        data_list.append([eval(line[:-3]), int(line[-2:-1])])
+    num = len(data_list)
+
+    i = 0
+    random.shuffle(data_list)
+    while True:
+        X = np.zeros((batch_size, timesteps))
+        Y = np.zeros((batch_size, 1))
+
+        for j in range(batch_size):
+            if i >= num:
+                random.shuffle(data_list)
+                i = 0
+            data = data_list[i]
+
+            d_list = [x for x in data[0]]
+            d_list = d_list + [voc_dim]*(timesteps-len(d_list))
+            X[j] = np.array(d_list)
+            Y[j] = data[1]
+            i += 1
+        yield X, Y
+
+
+def generate_data_siamese(char_num=6, char_shape=(40, 40)):
+    bg_paths = glob("../data/base/*")
+    char_path = "../data/chinese_2500.txt"
+    with open(char_path, "r") as f:
+        char_str = f.read()
+
+    data_dir = "../data/click/"
+    for i in range(1000):
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        char_list = [char_str[x] for x in random.sample(range(len(char_str)), char_num)]
+        image_np, position_list, tips_image_list = char_on_image(bg, char_list, char_shape)
+
+        # for j in range(len(char_list)):
+        #     char = char_list[j]
+        #     p = position_list[j]
+        #     print(char)
+        #     cv2.rectangle(image_np, [p[1], p[0]], [p[1]+char_shape[1], p[0]+char_shape[0]], (255, 0, 0), 2)
+        #     cv2.imshow("generate_data", image_np)
+        #     cv2.waitKey(0)
+
+        # 保存tips图片
+        tips_path_list = []
+        for k in range(len(tips_image_list)):
+            tips_image = tips_image_list[k]
+            tips_path = str(i) + "_" + str(k) + ".jpg"
+            tips_path_list.append(tips_path)
+            cv2.imwrite(data_dir+tips_path, tips_image)
+        # 保存文字区域图片
+        char_path_list = []
+        for j in range(len(char_list)):
+            p = position_list[j]
+            char_path = str(i) + "_" + str(len(tips_image_list)+j) + ".jpg"
+            char_path_list.append(char_path)
+            cv2.imwrite(data_dir+char_path, image_np[p[0]:p[0]+char_shape[0], p[1]:p[1]+char_shape[1], :])
+
+        # 生成映射数据
+        with open("../data/click/map.txt", "a") as f:
+            for j in range(len(tips_path_list)):
+                tips_path = tips_path_list[j]
+                for k in range(len(char_path_list)):
+                    char_path = char_path_list[k]
+                    if j == k:
+                        f.write(tips_path + "\t" + char_path + "\t" + str(1) + "\n")
+                    else:
+                        f.write(tips_path + "\t" + char_path + "\t" + str(0) + "\n")
+
+
+def generate_data_mobile(char_num=6, char_shape=(40, 40), image_shape=(160, 260)):
+    # (40,40) (160, 260)
+    # (80,80) (360, 590)
+
+    bg_paths = glob("../data/base/*")
+    char_path = "../data/chinese_5710.txt"
+    with open(char_path, "r") as f:
+        char_str = f.read()
+
+    data_dir = "../data/click/"
+    # 每个字生成多张图片
+    for i in range(0, len(char_str)):
+        if i % 100 == 0:
+            print("Loop", i)
+
+        char = char_str[i]
+        # 生成带背景图数
+        image_cnt = 1
+        char_list = [char] * image_cnt
+
+        tips_cnt = 0
+        image_cnt = 0
+        tips_list = []
+        for l in range(50):
+            # 背景图
+            bg = cv2.imread(random.sample(bg_paths, 1)[0])
+            if random.choice([0, 0, 1]):
+                bg = distort_image(bg)
+            if random.choice([0, 0, 1]):
+                bg = flip_image(bg)
+
+            # 生成4张tips图，6张带背景的旋转图
+            image_np, p_list, t_list = char_on_image(bg, char_list, char_shape, image_shape)
+            tips_list += t_list
+            for p in p_list:
+                char_path = str(i) + "_" + str(image_cnt) + "_2" + ".jpg"
+                cv2.imwrite(data_dir+char_path, image_np[p[0]:p[0]+char_shape[0], p[1]:p[1]+char_shape[1], :])
+                image_cnt += 1
+        for tips_image in tips_list[:4]:
+            tips_path = str(i) + "_" + str(tips_cnt) + "_1" + ".jpg"
+            cv2.imwrite(data_dir+tips_path, tips_image)
+            tips_cnt += 1
+
+
+def generate_data_yolo_char(char_num=6, char_shape=(40, 40), image_shape=(160, 256)):
+    bg_paths = glob("../data/base/*")
+    char_path = "../data/chinese_5710.txt"
+    with open(char_path, "r") as f:
+        char_str = f.read()
+
+    data_dir = "../data/detect/"
+    # with open(data_dir+"map.txt", "w") as f:
+    #     f.write("")
+
+    for i in range(40000, 60000):
+        if i % 1000 == 0:
+            print("Loop", i)
+
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        if random.choice([0, 0, 1]):
+            bg = distort_image(bg)
+        if random.choice([0, 0, 1]):
+            bg = flip_image(bg)
+
+        char_list = [char_str[x] for x in random.sample(range(len(char_str)), char_num)]
+        image_np, position_list, tips_image_list = char_on_image(bg, char_list, char_shape, image_shape, 4)
+
+        if i < 5000:
+            tips_image_np, tips_position_list = get_tips_image(tips_image_list, char_shape, image_shape)
+
+        image_np_path = str(i) + ".jpg"
+        cv2.imwrite(data_dir+image_np_path, image_np)
+
+        if i < 5000:
+            tips_image_np_path = str(i) + "_0.jpg"
+            cv2.imwrite(data_dir+tips_image_np_path, tips_image_np)
+
+        # 生成映射数据
+        with open(data_dir+"map.txt", "a") as f:
+            box_str = ""
+            for p in position_list:
+                box_str += str(p[1]) + "," + str(p[0]) + "," + \
+                           str(p[1]+char_shape[1]) + "," + str(p[0]+char_shape[0]) +\
+                           "," + str(0) + " "
+            #     cv2.rectangle(image_np, (p[1], p[0]), (p[1]+char_shape[1], p[0]+char_shape[0]), (255, 0, 0), 2)
+            # cv2.imshow("image_np", image_np)
+            box_str = box_str[:-1]
+            f.write(image_np_path + " " + box_str + "\n")
+
+            if i < 5000:
+                box_str = ""
+                for p in tips_position_list:
+                    box_str += str(p[1]) + "," + str(p[0]) + "," + \
+                               str(p[1]+char_shape[1]) + "," + str(p[0]+char_shape[0]) + \
+                               "," + str(0) + " "
+                #     cv2.rectangle(tips_image_np, (p[1], p[0]), (p[1]+char_shape[1], p[0]+char_shape[0]), (255, 0, 0), 2)
+                # cv2.imshow("tips_image_np", tips_image_np)
+                # cv2.waitKey(0)
+                box_str = box_str[:-1]
+                f.write(tips_image_np_path + " " + box_str + "\n")
+
+
+def generate_data_yolo_puzzle(image_shape=(160, 256)):
+    bg_paths = glob("../data/base/*.jpeg")
+
+    data_dir = "../data/detect2/"
+    with open(data_dir+"map.txt", "w") as f:
+        f.write("")
+
+    for i in range(0, 10000):
+        if i % 1000 == 0:
+            print("Loop", i)
+
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        if random.choice([0, 0, 1]):
+            bg = distort_image(bg)
+        if random.choice([0, 0, 1]):
+            bg = flip_image(bg)
+
+        r = random.randint(35, 60)
+        puzzle_shape = (r, r)
+        image_np, position_list = puzzle_on_image(bg, puzzle_shape, image_shape)
+
+        image_np_path = str(i) + ".jpg"
+        cv2.imwrite(data_dir+image_np_path, image_np)
+
+        # 生成映射数据
+        with open(data_dir+"map.txt", "a") as f:
+            box_str = ""
+            for p in position_list:
+                box_str += str(p[1]) + "," + str(p[0]) + "," + \
+                           str(p[1]+puzzle_shape[1]) + "," + str(p[0]+puzzle_shape[0]) + \
+                           "," + str(0) + " "
+            #     cv2.rectangle(image_np, (p[1], p[0]), (p[1]+puzzle_shape[1], p[0]+puzzle_shape[0]), (255, 0, 0), 2)
+            # cv2.imshow("image_np", image_np)
+            # cv2.waitKey(0)
+            box_str = box_str[:-1]
+            f.write(image_np_path + " " + box_str + "\n")
+
+
+def generate_data_drag_image(image_shape=(160, 260)):
+    bg_paths = glob("../data/base/*")
+
+    data_dir = "../data/drag/"
+    with open(data_dir+"map.txt", "w") as f:
+        f.write("")
+
+    for i in range(10000):
+        if i % 1000 == 0:
+            print("Loop", i)
+
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        bg = pil_resize(bg, image_shape[0], image_shape[1])
+        if random.choice([0, 0, 1]):
+            bg = distort_image(bg)
+        if random.choice([0, 0, 1]):
+            bg = flip_image(bg)
+
+        image_np, clip_line = get_drag_image(bg)
+
+        image_np_path = str(i) + ".jpg"
+        cv2.imwrite(data_dir+image_np_path, image_np)
+
+        # 生成映射数据
+        with open(data_dir+"map.txt", "a") as f:
+            f.write(image_np_path + " " + str(clip_line[0][0]) + "\n")
+
+
+def generate_data_phrase():
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/phrase/"
+
+    char_path = data_path+"char.txt"
+    with open(char_path, "r") as f:
+        char_list = f.readlines()
+    char_dict = {}
+    for i in range(len(char_list)):
+        char_dict[char_list[i]] = i
+
+    phrase_list = []
+    phrase_path = data_path+"phrase3.txt"
+    with open(phrase_path, "r") as f:
+        phrase_list += f.readlines()
+    phrase_path = data_path+"phrase4.txt"
+    with open(phrase_path, "r") as f:
+        phrase_list += f.readlines()
+    phrase_path = data_path+"phrase5.txt"
+    with open(phrase_path, "r") as f:
+        phrase_list += f.readlines()
+    phrase_set = set(phrase_list)
+
+    map_path = data_path+"map3.txt"
+    with open(map_path, "w") as f:
+        f.write("")
+    data_list = []
+    start_time = time.time()
+    i = 0
+    negative_way_flag = False
+    for phrase in phrase_list:
+        if i % 500000 == 0:
+            with open(map_path, "a") as f:
+                f.writelines(data_list)
+            data_list = []
+            print("Loop", i, len(phrase_list), time.time()-start_time)
+            start_time = time.time()
+        i += 1
+
+        # 正样本
+        index_list = []
+        for char in phrase[:-1]:
+            index_list.append(char_dict.get(char+"\n"))
+        data_list.append(str(index_list) + " 1\n")
+
+        # 负样本
+        if negative_way_flag:
+            index1 = random.randint(0, len(index_list)-1)
+            find_flag = False
+            while not find_flag:
+                index2 = random.randint(0, len(index_list)-1)
+                if index1 != index2:
+                    find_flag = True
+            temp = index_list[index1]
+            index_list[index1] = index_list[index2]
+            index_list[index2] = temp
+            if "".join([char_list[x][:-1] for x in index_list]) + "\n" not in phrase_set:
+                data_list.append(str(index_list) + " 0\n")
+        else:
+            products = list(product(index_list, repeat=len(index_list)))
+            random.shuffle(products)
+            negative_cnt = 0
+            for p in products:
+                if negative_cnt >= 2:
+                    break
+                p = list(p)
+                if len(set(p)) != len(p):
+                    continue
+                if p != index_list and "".join([char_list[x][:-1] for x in p]) + "\n" not in phrase_set:
+                    data_list.append(str(p) + " 0\n")
+                    negative_cnt += 1
+
+    with open(map_path, "a") as f:
+        f.writelines(data_list)
+
+
+def generate_data_phrase_raw(word_len=5):
+    paths = glob("D:/Chinese_corpus/answer/*/*.txt")
+
+    phrase_path = "../data/phrase/phrase" + str(word_len) + "_new.txt"
+    triple_list = []
+    reg = "[^\u4e00-\u9fa5]"
+    start_time = time.time()
+    for i in range(len(paths)):
+        if i % 1000 == 0:
+            with open(phrase_path, "w") as f:
+                f.writelines(triple_list)
+            print("Loop", i, len(paths), time.time()-start_time)
+            start_time = time.time()
+            triple_list = []
+
+        with open(paths[i], "rb") as f:
+            _b = f.read()
+        try:
+            text = _b.decode("gbk")
+        except:
+            try:
+                text = _b.decode("gb2312")
+            except:
+                try:
+                    text = _b.decode("gb18030")
+                except:
+                    print(chardet.detect(_b), "is None")
+
+        filter_word = ["的"]
+        for word in filter_word:
+            text = re.sub(word, "#"*len(word), text)
+
+        word_list = jieba.lcut(text, cut_all=False, HMM=True)
+
+        for j in range(1, len(word_list)):
+            current = word_list[j]
+            current_re = re.search(reg, current)
+            last = word_list[j-1]
+            last_re = re.search(reg, last)
+
+            if current_re:
+                continue
+            if len(current) == word_len:
+                triple_list.append(current + "\n")
+            elif len(current) + len(last) == word_len and not last_re:
+                triple_list.append(last+current + "\n")
+
+        triple_list = list(set(triple_list))
+
+    print("len(triple_list)", len(triple_list))
+    with open(phrase_path, "w") as f:
+        f.writelines(triple_list)
+
+
+def char_on_image(image_np, char_list, char_shape, image_shape, tip_char_num=1):
+    position_list = []
+    for char in char_list:
+        # 获取单字图片
+        char_image_pil = get_char_image(char, char_shape)
+
+        image_np = pil_resize(image_np, image_shape[0], image_shape[1])
+
+        # h, w
+        fg_w, fg_h = char_image_pil.size[:2]
+        bg_h, bg_w = image_np.shape[:2]
+
+        # 字体放置的位置，且位置不重叠
+        find_flag = 0
+        while not find_flag:
+            position_h = random.randint(0, bg_h-fg_h)
+            position_w = random.randint(0, bg_w-fg_w)
+            if len(position_list) < 1:
+                find_flag = 1
+                break
+            for p in position_list:
+                if get_iou(position_w, position_h, position_w+fg_w, position_h+fg_h,
+                           p[1], p[0], p[1]+fg_w, p[0]+fg_h) > 0:
+                    find_flag = 0
+                    break
+                else:
+                    find_flag = 1
+        position_list.append([position_h, position_w])
+        # 字体添加到背景图上
+        # image_np = get_image_roi(image_np, char_image_np, position_h, position_w)
+        image_np = get_image_paste(image_np, char_image_pil, position_h, position_w)
+
+    # 生成提示图片
+    image_list = []
+    for char in char_list[:tip_char_num]:
+        char_image_pil = get_char_image(char, char_shape, rotate=False, bg_color=(255, 255, 255, 255))
+        char_image_np = pil2np_a(char_image_pil)
+        # char_image_np = pil_resize(char_image_np, char_shape[0], char_shape[1])
+        image_list.append(char_image_np)
+    tips_image_np = np.concatenate(image_list, axis=1)
+    # 加干扰
+    tips_image_np = create_noise(tips_image_np)
+
+    # 切割
+    image_list = []
+    for i in range(tip_char_num):
+        image_list.append(tips_image_np[:, i*char_shape[1]:(i+1)*char_shape[1], :])
+    return image_np, position_list, image_list
+
+
+def get_char_image(char, char_shape, rotate=True, bg_color=(0, 0, 0, 0)):
+    # 创建空图
+    image_pil = Image.new('RGBA', (80, 80), bg_color)
+
+    # 空图上写字
+    # font_size = 35 # (40, 40)
+    font_size = 75 # (80, 80)
+    font_type_list = glob("../font/*")
+    font_type = random.sample(font_type_list, 1)[0]
+    font_config = ImageFont.truetype(font_type, int(font_size))
+    dr = ImageDraw.Draw(image_pil)
+    fill_color = random_color()
+    fill_color = (fill_color[0], fill_color[1], fill_color[2])
+    dr.text((3, -6), char, font=font_config, fill=fill_color)
+
+    if rotate:
+        if random.choice([0, 1]):
+            angle = random.randint(0, 80)
+        else:
+            angle = random.randint(280, 360)
+        image_pil = image_pil.rotate(angle, expand=False, fillcolor=bg_color)
+
+    # image_pil.show("1")
+    image_pil = image_pil.resize(char_shape)
+    # cv2.imshow("get_char_image", pil2np(image_pil))
+    # cv2.waitKey(0)
+    return image_pil
+
+
+def get_tips_image(tips_image_list, char_shape, image_shape, roatate=True):
+    new_list = []
+    for img in tips_image_list:
+        if random.choice([0, 0, 1]):
+            angle = random.randint(0, 360)
+            img = pil_rotate(img, angle, (255, 255, 255))
+        new_list.append(img)
+
+    tips_image_np = np.concatenate(new_list, axis=1)
+
+    new_image = np.full((image_shape[0], image_shape[1], 3), 0, np.uint8)
+    new_image[:tips_image_np.shape[0], :tips_image_np.shape[1], :] = tips_image_np
+
+    position_list = []
+    for i in range(len(new_list)):
+        h = 0
+        w = i*char_shape[1]
+        position_list.append([h, w])
+    return new_image, position_list
+
+
+def get_image_roi(image_bg, image_fg, roi_h, roi_w):
+    # h, w
+    fg_h, fg_w = image_fg.shape[:2]
+    bg_h, bg_w = image_bg.shape[:2]
+
+    # roi取值范围
+    roi = image_bg[roi_h:roi_h+fg_h, roi_w:roi_w+fg_w]
+
+    # 获取bg中非fg字体部分的掩码，相当于排除fg的字体部分，只保留bg的除fg字体外的部分
+    img_fg_gray = cv2.cvtColor(image_fg, cv2.COLOR_BGR2GRAY)
+    ret, mask = cv2.threshold(img_fg_gray, 0, 255, cv2.THRESH_OTSU)
+    bg_roi = cv2.bitwise_and(roi, roi, mask=mask)
+
+    # 获取fg中字体部分的掩码，相当于排除fg中的白色背景，只保留fg的字体部分
+    mask_inv = cv2.bitwise_not(mask)
+    fg_roi = cv2.bitwise_and(image_fg, image_fg, mask=mask_inv)
+    # 膨胀腐蚀去掉白色颗粒
+    # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
+    # fg_roi = cv2.erode(fg_roi, kernel)
+    # fg_roi = cv2.dilate(fg_roi, kernel)
+
+    # bg的除字体外背景部分 + fg的字体部分
+    image_roi = cv2.add(bg_roi, fg_roi)
+
+    # 将roi部分放回原bg
+    image_bg[roi_h:roi_h+fg_h, roi_w:roi_w+fg_w, :] = image_roi
+
+    # cv2.imshow("image_fg", image_fg)
+    # cv2.imshow("get_image_roi", image_bg)
+    # cv2.waitKey(0)
+    return image_bg
+
+
+def get_image_paste(image_bg, image_fg, roi_h, roi_w):
+    fg_h, fg_w = image_fg.size[:2]
+    image_bg = cv2.cvtColor(image_bg, cv2.COLOR_BGR2BGRA)
+    image_bg = np2pil_a(image_bg)
+    # image_fg = np2pil_a(image_fg)
+    image_bg.paste(image_fg, (roi_w, roi_h), image_fg)
+    image_bg = pil2np(image_bg)
+
+    # cv2.imshow("get_image_paste", image_bg)
+    # cv2.waitKey(0)
+    return image_bg
+
+
+def random_color(dims=3):
+    color = [0]*dims
+
+    find_flag = 0
+    while not find_flag:
+        for dim in range(dims):
+            color[dim] = random.randint(0, 255)
+            if color[dim] <= 125:
+                find_flag = 1
+
+    # RGB
+    # color_list = [
+    #     [207, 91, 85],
+    #     [0, 201, 88],
+    #     [117, 74, 57],
+    #     [210, 210, 27],
+    #     [160, 157, 152],
+    #     [181, 210, 210],
+    #     [27, 112, 107],
+    #     [87, 26, 44],
+    #     [115, 19, 20],
+    #     [161, 210, 68],
+    #     [210, 108, 12],
+    #     [112, 9, 142],
+    #     [50, 41, 84],
+    #     [72, 52, 210],
+    #     [210, 177, 89],
+    #     [148, 200, 89],
+    #     [173, 116, 109],
+    #     [185, 185, 210],
+    #     [181, 7, 210],
+    #     [80, 210, 30],
+    #     [65, 72, 98],
+    #     [210, 123, 109],
+    #     [19, 64, 95],
+    #     [128, 21, 210],
+    #     [129, 137, 60]
+    # ]
+    # color = random.sample(color_list, 1)[0]
+    return tuple(color)
+
+
+def create_noise(image_np):
+    ic = ImageCaptcha()
+    image_pil = np2pil(image_np)
+    image_pil = ic.create_noise_curve(image_pil, random_color())
+    image_pil = ic.create_noise_curve(image_pil, random_color())
+    image_pil = ic.create_noise_dots(image_pil, random_color())
+    image_np = pil2np(image_pil)
+    return image_np
+
+
+def get_iou(x1, y1, x2, y2, a1, b1, a2, b2):
+    # 相交区域左上角横坐标
+    ax = max(x1, a1)
+    # 相交区域左上角纵坐标
+    ay = max(y1, b1)
+    # 相交区域右下角横坐标
+    bx = min(x2, a2)
+    # 相交区域右下角纵坐标
+    by = min(y2, b2)
+
+    area_n = (x2 - x1) * (y2 - y1)
+    area_m = (a2 - a1) * (b2 - b1)
+
+    w = max(0, bx - ax)
+    h = max(0, by - ay)
+    area_x = w * h
+
+    return area_x / (area_n + area_m - area_x)
+
+
+def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes):
+    """Preprocess true boxes to training input format
+    Parameters
+    ----------
+    true_boxes: array, shape=(m, T, 5)
+        Absolute x_min, y_min, x_max, y_max, class_id relative to input_shape.
+    input_shape: array-like, hw, multiples of 32
+    anchors: array, shape=(N, 2), wh
+    num_classes: integer
+    Returns
+    -------
+    y_true: list of array, shape like yolo_outputs, xywh are reletive value
+    """
+    # print(true_boxes[..., 4])
+    # print(num_classes)
+    assert (true_boxes[..., 4] < num_classes).all(), 'class id must be less than num_classes'
+    # default setting
+    num_layers = len(anchors)//3
+    anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
+
+    true_boxes = np.array(true_boxes, dtype='float32')
+    input_shape = np.array(input_shape, dtype='int32')
+    boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2
+    boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2]
+    true_boxes[..., 0:2] = boxes_xy/input_shape[::-1]
+    true_boxes[..., 2:4] = boxes_wh/input_shape[::-1]
+
+    m = true_boxes.shape[0]
+    grid_shapes = [input_shape//{0: 32, 1: 16, 2: 8}[l] for l in range(num_layers)]
+    y_true = [np.zeros((m, grid_shapes[l][0], grid_shapes[l][1],len(anchor_mask[l]), 5+num_classes),
+                       dtype='float32') for l in range(num_layers)]
+
+    # Expand dim to apply broadcasting.
+    anchors = np.expand_dims(anchors, 0)
+    anchor_maxes = anchors / 2.
+    anchor_mins = -anchor_maxes
+    valid_mask = boxes_wh[..., 0] > 0
+
+    for b in range(m):
+        # Discard zero rows.
+        wh = boxes_wh[b, valid_mask[b]]
+        if len(wh) == 0:
+            continue
+        # Expand dim to apply broadcasting.
+        wh = np.expand_dims(wh, -2)
+        box_maxes = wh / 2.
+        box_mins = -box_maxes
+
+        intersect_mins = np.maximum(box_mins, anchor_mins)
+        intersect_maxes = np.minimum(box_maxes, anchor_maxes)
+        intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.)
+        intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
+        box_area = wh[..., 0] * wh[..., 1]
+        anchor_area = anchors[..., 0] * anchors[..., 1]
+        iou = intersect_area / (box_area + anchor_area - intersect_area)
+
+        # Find best anchor for each true box
+        best_anchor = np.argmax(iou, axis=-1)
+
+        for t, n in enumerate(best_anchor):
+            for l in range(num_layers):
+                if n in anchor_mask[l]:
+                    i = np.floor(true_boxes[b,t,0]*grid_shapes[l][1]).astype('int32')
+                    j = np.floor(true_boxes[b,t,1]*grid_shapes[l][0]).astype('int32')
+                    k = anchor_mask[l].index(n)
+                    c = true_boxes[b, t, 4].astype('int32')
+                    y_true[l][b, j, i, k, 0:4] = true_boxes[b, t, 0:4]
+                    y_true[l][b, j, i, k, 4] = 1
+                    y_true[l][b, j, i, k, 5+c] = 1
+
+    return y_true
+
+
+def get_puzzle(shape=(80, 80)):
+    # 创建空图
+    image_pil = Image.new('RGBA', (shape[1], shape[0]), (255, 255, 255, 0))
+    draw = ImageDraw.Draw(image_pil)
+
+    # 居中创建矩形
+    rec_shape = (40, 40)
+    left_up_point = [int((shape[0]-rec_shape[0])/2), int((shape[1]-rec_shape[1])/2)]
+    right_down_point = [left_up_point[0]+rec_shape[0], left_up_point[1]+rec_shape[1]]
+
+    # 透明度
+    flag = random.choice([0, 1])
+    if flag:
+        alpha = random.randint(100, 150)
+    else:
+        alpha = random.randint(160, 255)
+
+    # 背景色
+    if flag:
+        r = random.randint(0, 30)
+    else:
+        r = random.randint(100, 180)
+    # r = random.randint(0, 255)
+    fill_color = (r, r, r, alpha)
+
+    # 边缘色
+    if random.choice([0, 1, 1]):
+        if flag:
+            r = random.randint(140, 170)
+        else:
+            r = random.randint(70, 100)
+        outline_color = (r, r, r)
+    else:
+        outline_color = (fill_color[0], fill_color[1], fill_color[2])
+
+    draw.rectangle((left_up_point[1],
+                    left_up_point[0],
+                    left_up_point[1]+rec_shape[1],
+                    left_up_point[0]+rec_shape[0]),
+                   fill=fill_color,
+                   outline=outline_color)
+
+    # 拼图的圆或半圆
+    radius = random.randint(int(rec_shape[0] / 3 / 2), int(rec_shape[0] / 3 / 1.2))
+    center_list = [[left_up_point[1], int((right_down_point[0]+left_up_point[0])/2), 1],
+                   [right_down_point[1], int((right_down_point[0]+left_up_point[0])/2), 1],
+                   [int((right_down_point[1]+left_up_point[1])/2), left_up_point[0], 0],
+                   [int((right_down_point[1]+left_up_point[1])/2), right_down_point[0], 0]
+                   ]
+    circle_num = random.randint(1, 4)
+    # print("circle_num", circle_num)
+    center_list = random.sample(center_list, circle_num)
+
+    min_w, min_h = left_up_point[1], left_up_point[0]
+    max_w, max_h = right_down_point[1], right_down_point[0]
+    for center in center_list:
+        w, h = center[:2]
+        is_width = center[2]
+
+        # 判断长宽
+        into_ratio = random.randint(int(1/2*radius), int(3/4*radius))
+        if is_width:
+            # 挑选圆是凸还是凹进去
+            if random.choice([0, 1]):
+                center = (center[0]+into_ratio, center[1])
+            else:
+                center = (center[0]-into_ratio, center[1])
+        else:
+            if random.choice([0, 1]):
+                center = (center[0], center[1]+into_ratio)
+            else:
+                center = (center[0], center[1]-into_ratio)
+
+        # 判断透明度
+        color = fill_color
+        if is_width:
+            if left_up_point[1] <= center[0] <= right_down_point[1]:
+                color = (0, 0, 0, 0)
+        else:
+            if left_up_point[0] <= center[1] <= right_down_point[0]:
+                color = (0, 0, 0, 0)
+
+        # print("center, color, alpha", center, color, alpha)
+        draw.ellipse([(center[0]-radius, center[1]-radius),
+                      (center[0]+radius, center[1]+radius)],
+                     fill=color,
+                     outline=outline_color)
+
+        # 修补内部圆的边缘颜色
+        if color[3] == alpha:
+            if is_width:
+                if center[0] < w:
+                    draw.rectangle((w,
+                                    h-radius,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=fill_color)
+                else:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w,
+                                    h+radius),
+                                   fill=fill_color)
+            else:
+                if center[1] < h:
+                    draw.rectangle((w-radius,
+                                    h,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=fill_color)
+                else:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w+radius,
+                                    h),
+                                   fill=fill_color)
+        # 修补外部圆的边缘颜色
+        else:
+            if is_width:
+                if center[0] > w:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w,
+                                    h+radius),
+                                   fill=(0, 0, 0, 0))
+                else:
+                    draw.rectangle((w,
+                                    h-radius,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=(0, 0, 0, 0))
+            else:
+                if center[1] > h:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w+radius,
+                                    h),
+                                   fill=(0, 0, 0, 0))
+                else:
+                    draw.rectangle((w-radius,
+                                    h,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=(0, 0, 0, 0))
+
+        # 新增面积
+        if color[3] == alpha:
+            if center[0]-radius <= min_w:
+                min_w = center[0]-radius
+            if center[0]+radius >= max_w:
+                max_w = center[0]+radius
+            if center[1]-radius <= min_h:
+                min_h = center[1]-radius
+            if center[1]+radius >= max_h:
+                max_h = center[1]+radius
+
+    image_pil = image_pil.crop([min_w, min_h, max_w+1, max_h+1])
+    # image_pil.show("2")
+    return image_pil
+
+
+def puzzle_on_image(image_np, puzzle_shape, image_shape):
+    position_list = []
+    # 获取拼图图片
+    puzzle_image_pil = get_puzzle()
+    puzzle_image_pil = puzzle_image_pil.resize(puzzle_shape)
+    image_np = pil_resize(image_np, image_shape[0], image_shape[1])
+
+    # h, w
+    fg_w, fg_h = puzzle_image_pil.size[:2]
+    bg_h, bg_w = image_np.shape[:2]
+    # 拼图放置的位置
+    position_h = random.randint(0, bg_h-fg_h)
+    position_w = random.randint(0, bg_w-fg_w)
+    position_list.append([position_h, position_w])
+    # for p in position_list:
+    #     cv2.rectangle(image_np, (p[1], p[0]),
+    #                   (p[1]+puzzle_shape[1], p[0]+puzzle_shape[0]),
+    #                   (0, 0, 255), 1)
+    # 拼图添加到背景图上
+    image_np = get_image_paste(image_np, puzzle_image_pil, position_h, position_w)
+    # cv2.imshow("puzzle_on_image", image_np)
+    # cv2.waitKey(0)
+    return image_np, position_list
+
+
+def distort_image(image_np, hue=.1, sat=1.5, val=1.5):
+    """
+    图像失真
+    :return:
+    """
+    def rand(a=0, b=1):
+        return np.random.rand()*(b-a) + a
+
+    # cv2.imshow("distort_image1", image_np)
+    hue = rand(-hue, hue)
+    sat = rand(1, sat) if rand() < .5 else 1/rand(1, sat)
+    val = rand(1, val) if rand() < .5 else 1/rand(1, val)
+
+    image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
+    x = rgb_to_hsv(image_np/255.)
+    x[..., 0] += hue
+    x[..., 0][x[..., 0] > 1] -= 1
+    x[..., 0][x[..., 0] < 0] += 1
+    x[..., 1] *= sat
+    x[..., 2] *= val
+    x[x > 1] = 1
+    x[x < 0] = 0
+    image_np = hsv_to_rgb(x)
+    image_np = cv2.cvtColor(np.uint8(image_np*255), cv2.COLOR_RGB2BGR)
+
+    # cv2.imshow("distort_image2", image_np)
+    # cv2.waitKey(0)
+    return image_np
+
+
+def flip_image(image_np):
+    # cv2.imshow("flip_image1", image_np)
+    if random.choice([0, 1]):
+        # 水平翻转
+        image_np = cv2.flip(image_np, 1)
+    else:
+        # 垂直翻转
+        image_np = cv2.flip(image_np, 0)
+    # cv2.imshow("flip_image2", image_np)
+    # cv2.waitKey(0)
+    return image_np
+
+
+def get_drag_image(image_np):
+    h, w = image_np.shape[:2]
+
+    # 取一定高度图片
+    clip_h = random.randint(int(1/4*h), int(3/4*h))
+    image_clip = image_np[:clip_h, ...]
+
+    # 将图片在一定宽度截断，重新拼接
+    clip_w = random.randint(int(1/6*w), int(5/6*w))
+    image_w1 = image_clip[:, :clip_w, ...]
+    image_w2 = image_clip[:, clip_w:, ...]
+    image_new = np.concatenate([image_w2, image_w1], axis=1)
+
+    # 分割线
+    clip_line = [(image_w2.shape[1], 0), (image_w2.shape[1], clip_h)]
+
+    # show
+    # print(clip_line)
+    # cv2.line(image_new, clip_line[0], clip_line[1], (0, 0, 255), 2)
+    # cv2.imshow("get_drag_image", image_new)
+    # cv2.waitKey(0)
+    return image_new, clip_line
+
+
+def get_real_data_puzzle(shape=(160, 256)):
+    paths = glob("../data/detect2_real/*")
+    i = 10000
+    for p in paths:
+        image = cv2.imread(p)
+        image = pil_resize(image, shape[0], shape[1])
+        cv2.imwrite("../data/detect2_real/"+str(i)+".jpg", image)
+        i += 1
+
+        image = distort_image(image)
+        cv2.imwrite("../data/detect2_real/"+str(i)+".jpg", image)
+        i += 1
+
+        image = flip_image(image)
+        cv2.imwrite("../data/detect2_real/"+str(i)+".jpg", image)
+        i += 1
+
+
+def read_label_puzzle():
+    paths = glob("../data/detect2_real/*.json")
+    map_path = "../data/detect2_real/map.txt"
+    with open(map_path, "a") as f:
+        for p in paths:
+            with open(p, "r") as fp:
+                _dict = json.loads(fp.read())
+            points = _dict.get("shapes")[0].get("points")
+            image_path = _dict.get("imagePath")
+            ps = [str(int(points[0][0])), str(int(points[0][1])),
+                  str(int(points[1][0])), str(int(points[1][1]))]
+            p_str = ",".join(ps)
+            f.write(image_path + " " + p_str + ",0" + "\n")
+
+
+def fix_map_txt():
+    path = "../data/map.txt"
+    with open(path, "r") as f:
+        _list = f.readlines()
+
+    with open("../data/map_new.txt", "w") as f:
+        new_list = []
+        for line in _list:
+            ss = line.split(" ")
+            ps = ss[-1][:-1].split(",")[:-1]
+
+            if random.choice([0, 1, 1, 1]):
+                pix = random.choice([1, 2, 2, 3, 3, 4, 4])
+                for i in range(len(ps)):
+                    if i < 2:
+                        ps[i] = str(int(ps[i]) - pix)
+                    else:
+                        ps[i] = str(int(ps[i]) + pix)
+            new_line = ss[0] + " " + ",".join(ps) + ",0\n"
+            new_list.append(new_line)
+            print("line", line)
+            print("new_line", new_line)
+        f.writelines(new_list)
+
+
+def get_char_map():
+    path = "../data/phrase/phrase3.txt"
+    with open(path, "r") as f:
+        _list = f.readlines()
+    path = "../data/phrase/phrase4.txt"
+    with open(path, "r") as f:
+        _list += f.readlines()
+    path = "../data/phrase/phrase5.txt"
+    with open(path, "r") as f:
+        _list += f.readlines()
+
+    _str = "".join(_list)
+    _str = re.sub("\n", "", _str)
+    _list = list(set([x+"\n" for x in _str]))
+    _list.sort(key=lambda x: x)
+    with open("../data/phrase/char.txt", "w") as f:
+        f.writelines(_list)
+
+
+if __name__ == "__main__":
+    # from click_captcha.utils import get_classes, get_anchors
+    # annotation_path = '../data/detect/map.txt'
+    # log_dir = 'yolo_data/logs/000/'
+    # classes_path = 'yolo_data/my_classes.txt'
+    # anchors_path = 'yolo_data/tiny_yolo_anchors.txt'
+    # class_names = get_classes(classes_path)
+    # num_classes = len(class_names)
+    # anchors = get_anchors(anchors_path)
+    #
+    # with open(annotation_path) as f:
+    #     lines = f.readlines()
+    # random.shuffle(lines)
+    # input_shape = (160, 256)
+    # g = gen_yolo(lines, 10, input_shape, anchors, num_classes)
+    # list(g)
+
+    generate_data_phrase()
+
+    # generate_data_yolo_puzzle()
+    # gen_yolo_puzzle()
+    # _path = "../data/base/0b16d1f1a4e017d4a7ab5779263887f1.jpeg"
+    # get_drag_image(cv2.imread(_path))
+
+    # for ii in range(10):
+    #     im = get_puzzle()
+
+    # with open("../data/chinese.txt", "r") as f:
+    #     _str = f.read()
+    #
+    # _list = [c for c in _str]
+    # _list = list(set(_list))
+    # _str = "".join(_list)
+
+    # with open("../data/chinese.txt", "w") as f:
+    #     f.write(_str)

chinese_recognize/inference_char.py

@@ -0,0 +1,36 @@
+import os
+
+import cv2
+import numpy as np
+from click_captcha.model import mobile_net, cnn_net
+from click_captcha.utils import pil_resize
+
+image_shape = (40, 40, 3)
+weights_path = "./models/char_f1_0.93.h5"
+project_dir = os.path.dirname(os.path.abspath(__file__)) + "/../"
+
+
+def recognize(image_path):
+    model = cnn_net(input_shape=image_shape)
+    model.load_weights(weights_path)
+
+    img = cv2.imread(image_path)
+    img = pil_resize(img, image_shape[0], image_shape[1])
+    cv2.imshow("img", img)
+    cv2.waitKey(0)
+    img = img / 255.
+    X = np.expand_dims(img, 0)
+
+    pred = model.predict(X)
+    index = int(np.argmax(pred))
+    with open(project_dir + "data/chinese_5710.txt") as f:
+        char_str = f.read()
+    char = char_str[index]
+    print("recognize chinese", char)
+    return char
+
+
+if __name__ == "__main__":
+    _path = "../data/test/char_6.jpg"
+    # _path = "../data/click/2019_73_1.jpg"
+    recognize(_path)

chinese_recognize/pre_process.py

@@ -0,0 +1,1322 @@
+import copy
+import json
+import os
+import random
+import re
+import sys
+import time
+import traceback
+from glob import glob
+from itertools import combinations, product
+
+import chardet
+import cv2
+import jieba
+import numpy as np
+from PIL import ImageFont, ImageDraw, Image
+from captcha.image import ImageCaptcha
+from keras_preprocessing.sequence import pad_sequences
+from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+from click_captcha.utils import np2pil, pil2np, pil_resize, pil_rotate, pil2np_a, np2pil_a, pil_resize_a, pil_rotate_a
+
+
+def gen_siamese(paths, batch_size=32, shape=(40, 40), cls_num=1):
+    num = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/click/"
+
+    i = 0
+    while True:
+        if i >= num:
+            i = 0
+            random.shuffle(paths)
+
+        height, width = shape[:2]
+        X1 = np.zeros((batch_size, height, width, 1))
+        X2 = np.zeros((batch_size, height, width, 1))
+        Y = np.zeros((batch_size, 2))
+
+        for j in range(batch_size):
+            # 生成标注数据
+            img1, img2, label = paths[i][:-1].split("\t")
+            # print(img1, img2, label)
+            img1 = cv2.imread(data_path + img1)
+            img1 = pil_resize(img1, shape[0], shape[1])
+            img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+            img1 = np.expand_dims(img1, axis=-1)
+            img2 = cv2.imread(data_path + img2)
+            img2 = pil_resize(img2, shape[0], shape[1])
+            img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
+            img2 = np.expand_dims(img2, axis=-1)
+            if label == "1":
+                label = np.array([0, 1])
+            else:
+                label = np.array([1, 0])
+            X1[j] = img1
+            X2[j] = img2
+            Y[j] = label
+
+        yield {"input_1": X1, "input_2": X2}, {"output": Y}
+
+
+def gen_mobile(paths, batch_size=32, shape=(40, 40), cls_num=5710, data_path="click"):
+    num = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/" + data_path + "/"
+
+    i = 0
+    random.shuffle(paths)
+    while True:
+        if i >= num:
+            i = 0
+            random.shuffle(paths)
+
+        height, width = shape[:2]
+        if len(shape) > 2:
+            channel = 3
+        else:
+            channel = 1
+        X = np.zeros((batch_size, height, width, channel))
+        Y = np.zeros((batch_size, cls_num))
+
+        j = 0
+        error_num = 0
+        while j < batch_size:
+        # for j in range(batch_size):
+            if i >= num:
+                random.shuffle(paths)
+                i = 0
+            path = paths[i].split(os.sep)[-1]
+            char_index = int(path.split("_")[0])
+            label = np.zeros(cls_num)
+            # print("char_index", char_index)
+            label[char_index] = 1
+            # print("label", np.argmax(label), char_index)
+
+
+            img1 = cv2.imread(data_path + path)
+            img1 = pil_resize(img1, shape[0], shape[1])
+            img1 = img1 / 255.
+            # img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+            # img1 = np.expand_dims(img1, axis=-1)
+
+            X[j] = img1
+            Y[j] = label
+            i += 1
+            j += 1
+        # print("error_num", error_num)
+        yield X, Y
+
+
+def gen_yolo_char(paths, batch_size, input_shape, anchors, num_classes, box_num=6):
+    """data generator for fit_generator"""
+    n = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/detect/"
+    i = 0
+    while True:
+        image_data = []
+        box_data = []
+
+        batch_cnt = 0
+        while batch_cnt < batch_size:
+            try:
+                if i == 0:
+                    np.random.shuffle(paths)
+
+                ss = paths[i][:-1].split(" ")
+                image_path = ss[0]
+                image = cv2.imread(data_path+image_path)
+                image = pil_resize(image, input_shape[0], input_shape[1])
+                image_show = copy.deepcopy(image)
+                image = image / 255.
+                box = np.array([np.array(list(map(int, box.split(',')))) for box in ss[1:]])
+
+                # box数不同，复制
+                if box.shape[0] < box_num:
+                    box = np.concatenate([box, box[:2, :]], axis=0)
+
+                # show
+                # box_show = box.tolist()
+                # for b in box_show:
+                #     print("box", b)
+                #     cv2.rectangle(image_show, (b[0], b[1]), (b[2], b[3]), (255, 0, 0), 2)
+                # cv2.imshow("image_show", image_show)
+                # cv2.waitKey(0)
+
+                image_data.append(image)
+                box_data.append(box)
+                i = (i+1) % n
+                batch_cnt += 1
+            except:
+                i = (i+1) % n
+                continue
+
+            # print
+            # print(image.shape)
+            # image_show = (image*255).astype(np.uint8)
+            # print("annotation_lines[i]", annotation_lines[i])
+            # for _b in box:
+            #     print(_b)
+            #     cv2.rectangle(image_show, (int(_b[0]), int(_b[1])), (int(_b[2]), int(_b[3])), (0, 255, 0), 1)
+            # cv2.imshow("image", image_show)
+            # cv2.waitKey(0)
+
+        image_data = np.array(image_data)
+        box_data = np.array(box_data)
+        # print(image_data.shape, box_data.shape)
+        y_true = preprocess_true_boxes(box_data, input_shape, anchors, num_classes)
+        yield [image_data, *y_true], np.zeros(batch_size)
+
+
+def gen_yolo_puzzle(paths, batch_size, input_shape, anchors, num_classes, box_num=1):
+    """data generator for fit_generator"""
+    n = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/detect2/"
+    i = 0
+    while True:
+        image_data = []
+        box_data = []
+
+        batch_cnt = 0
+        while batch_cnt < batch_size:
+            try:
+                if i == 0:
+                    np.random.shuffle(paths)
+
+                ss = paths[i][:-1].split(" ")
+                image_path = ss[0]
+                image = cv2.imread(data_path+image_path)
+                image = pil_resize(image, input_shape[0], input_shape[1])
+                image_show = copy.deepcopy(image)
+                image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+                image = 255. - image
+                image = np.uint8(image)
+                # cv2.imshow("image", image)
+                # cv2.waitKey(0)
+                image = np.expand_dims(image, -1)
+                image = image / 255.
+                box = np.array([np.array(list(map(int, box.split(',')))) for box in ss[1:]])
+
+                # box数不同，复制
+                if box.shape[0] < box_num:
+                    box = np.concatenate([box, box[:2, :]], axis=0)
+
+                # show
+                # box_show = box.tolist()
+                # for b in box_show:
+                #     print("box", b)
+                #     cv2.rectangle(image_show, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2)
+                # cv2.imshow("image_show", image_show)
+                # cv2.waitKey(0)
+
+                image_data.append(image)
+                box_data.append(box)
+                i = (i+1) % n
+                batch_cnt += 1
+            except:
+                i = (i+1) % n
+                continue
+
+            # print
+            # print(image.shape)
+            # image_show = (image*255).astype(np.uint8)
+            # for _b in box:
+            #     print(_b)
+            #     cv2.rectangle(image_show, (int(_b[0]), int(_b[1])), (int(_b[2]), int(_b[3])), (0, 255, 0), 1)
+            # cv2.imshow("image", image_show)
+            # cv2.waitKey(0)
+
+        image_data = np.array(image_data)
+        box_data = np.array(box_data)
+        # print(image_data.shape, box_data.shape)
+        y_true = preprocess_true_boxes(box_data, input_shape, anchors, num_classes)
+        yield [image_data, *y_true], np.zeros(batch_size)
+
+
+def gen_drag(paths, batch_size=32, shape=(128, 256), cls_num=2):
+    num = len(paths)
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/drag/"
+
+    map_path = data_path+"map.txt"
+    with open(map_path, "r") as f:
+        _list = f.readlines()
+    map_dict = {}
+    for s in _list:
+        ss = s[:-1].split(" ")
+        map_dict[ss[0]] = ss[1]
+
+    i = 0
+    random.shuffle(paths)
+    while True:
+        if i >= num:
+            i = 0
+            random.shuffle(paths)
+
+        height, width = shape[:2]
+        if len(shape) > 2:
+            channel = 3
+        else:
+            channel = 1
+        X = np.zeros((batch_size, height, width, channel))
+        Y = np.zeros((batch_size, height, width, 1))
+
+        for j in range(batch_size):
+            if i >= num:
+                random.shuffle(paths)
+                i = 0
+            path = paths[i].split(os.sep)[-1]
+            w_index = int(map_dict.get(path))
+            # label = np.zeros(cls_num)
+            # print("char_index", char_index)
+            # label[w_index] = 1
+            # print("label", np.argmax(label), char_index)
+
+            img1 = cv2.imread(data_path + path)
+            img1 = pil_resize(img1, shape[0], shape[1])
+            # cv2.imshow("image", img1)
+            label = np.full((shape[0], shape[1], 1), 0, dtype='uint8')
+            label[:, w_index, 0] = 1
+            # label[:, w_index, 1] = 1
+            # cv2.imshow("label", np.expand_dims(label[..., 0], -1))
+            # cv2.waitKey(0)
+            img1 = img1 / 255.
+            # img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+            # img1 = np.expand_dims(img1, axis=-1)
+            i += 1
+
+            X[j] = img1
+            Y[j] = label
+
+        yield X, Y
+
+
+def gen_phrase(map_list, batch_size=32, shape=(5707, 3)):
+    voc_dim, timesteps = shape[:2]
+
+    data_list = []
+    for line in map_list:
+        data_list.append([eval(line[:-3]), int(line[-2:-1])])
+    num = len(data_list)
+
+    i = 0
+    random.shuffle(data_list)
+    while True:
+        X = np.zeros((batch_size, timesteps))
+        Y = np.zeros((batch_size, 1))
+
+        for j in range(batch_size):
+            if i >= num:
+                random.shuffle(data_list)
+                i = 0
+            data = data_list[i]
+
+            d_list = [x for x in data[0]]
+            d_list = d_list + [voc_dim]*(timesteps-len(d_list))
+            X[j] = np.array(d_list)
+            Y[j] = data[1]
+            i += 1
+        yield X, Y
+
+
+def generate_data_siamese(char_num=6, char_shape=(40, 40)):
+    bg_paths = glob("../data/base/*")
+    char_path = "../data/chinese_2500.txt"
+    with open(char_path, "r") as f:
+        char_str = f.read()
+
+    data_dir = "../data/click/"
+    for i in range(1000):
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        char_list = [char_str[x] for x in random.sample(range(len(char_str)), char_num)]
+        image_np, position_list, tips_image_list = char_on_image(bg, char_list, char_shape)
+
+        # for j in range(len(char_list)):
+        #     char = char_list[j]
+        #     p = position_list[j]
+        #     print(char)
+        #     cv2.rectangle(image_np, [p[1], p[0]], [p[1]+char_shape[1], p[0]+char_shape[0]], (255, 0, 0), 2)
+        #     cv2.imshow("generate_data", image_np)
+        #     cv2.waitKey(0)
+
+        # 保存tips图片
+        tips_path_list = []
+        for k in range(len(tips_image_list)):
+            tips_image = tips_image_list[k]
+            tips_path = str(i) + "_" + str(k) + ".jpg"
+            tips_path_list.append(tips_path)
+            cv2.imwrite(data_dir+tips_path, tips_image)
+        # 保存文字区域图片
+        char_path_list = []
+        for j in range(len(char_list)):
+            p = position_list[j]
+            char_path = str(i) + "_" + str(len(tips_image_list)+j) + ".jpg"
+            char_path_list.append(char_path)
+            cv2.imwrite(data_dir+char_path, image_np[p[0]:p[0]+char_shape[0], p[1]:p[1]+char_shape[1], :])
+
+        # 生成映射数据
+        with open("../data/click/map.txt", "a") as f:
+            for j in range(len(tips_path_list)):
+                tips_path = tips_path_list[j]
+                for k in range(len(char_path_list)):
+                    char_path = char_path_list[k]
+                    if j == k:
+                        f.write(tips_path + "\t" + char_path + "\t" + str(1) + "\n")
+                    else:
+                        f.write(tips_path + "\t" + char_path + "\t" + str(0) + "\n")
+
+
+def generate_data_mobile(char_num=6, char_shape=(40, 40), image_shape=(160, 260)):
+    # (40,40) (160, 260)
+    # (80,80) (360, 590)
+
+    bg_paths = glob("../data/base/*")
+    char_path = "../data/chinese_5710.txt"
+    with open(char_path, "r") as f:
+        char_str = f.read()
+
+    data_dir = "../data/click/"
+    # 每个字生成多张图片
+    for i in range(0, len(char_str)):
+        if i % 100 == 0:
+            print("Loop", i)
+
+        char = char_str[i]
+        # 生成带背景图数
+        image_cnt = 1
+        char_list = [char] * image_cnt
+
+        tips_cnt = 0
+        image_cnt = 0
+        tips_list = []
+        for l in range(50):
+            # 背景图
+            bg = cv2.imread(random.sample(bg_paths, 1)[0])
+            if random.choice([0, 0, 1]):
+                bg = distort_image(bg)
+            if random.choice([0, 0, 1]):
+                bg = flip_image(bg)
+
+            # 生成4张tips图，6张带背景的旋转图
+            image_np, p_list, t_list = char_on_image(bg, char_list, char_shape, image_shape)
+            tips_list += t_list
+            for p in p_list:
+                char_path = str(i) + "_" + str(image_cnt) + "_2" + ".jpg"
+                cv2.imwrite(data_dir+char_path, image_np[p[0]:p[0]+char_shape[0], p[1]:p[1]+char_shape[1], :])
+                image_cnt += 1
+        for tips_image in tips_list[:4]:
+            tips_path = str(i) + "_" + str(tips_cnt) + "_1" + ".jpg"
+            cv2.imwrite(data_dir+tips_path, tips_image)
+            tips_cnt += 1
+
+
+def generate_data_yolo_char(char_num=6, char_shape=(40, 40), image_shape=(160, 256)):
+    bg_paths = glob("../data/base/*")
+    char_path = "../data/chinese_5710.txt"
+    with open(char_path, "r") as f:
+        char_str = f.read()
+
+    data_dir = "../data/detect/"
+    # with open(data_dir+"map.txt", "w") as f:
+    #     f.write("")
+
+    for i in range(40000, 60000):
+        if i % 1000 == 0:
+            print("Loop", i)
+
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        if random.choice([0, 0, 1]):
+            bg = distort_image(bg)
+        if random.choice([0, 0, 1]):
+            bg = flip_image(bg)
+
+        char_list = [char_str[x] for x in random.sample(range(len(char_str)), char_num)]
+        image_np, position_list, tips_image_list = char_on_image(bg, char_list, char_shape, image_shape, 4)
+
+        if i < 5000:
+            tips_image_np, tips_position_list = get_tips_image(tips_image_list, char_shape, image_shape)
+
+        image_np_path = str(i) + ".jpg"
+        cv2.imwrite(data_dir+image_np_path, image_np)
+
+        if i < 5000:
+            tips_image_np_path = str(i) + "_0.jpg"
+            cv2.imwrite(data_dir+tips_image_np_path, tips_image_np)
+
+        # 生成映射数据
+        with open(data_dir+"map.txt", "a") as f:
+            box_str = ""
+            for p in position_list:
+                box_str += str(p[1]) + "," + str(p[0]) + "," + \
+                           str(p[1]+char_shape[1]) + "," + str(p[0]+char_shape[0]) +\
+                           "," + str(0) + " "
+            #     cv2.rectangle(image_np, (p[1], p[0]), (p[1]+char_shape[1], p[0]+char_shape[0]), (255, 0, 0), 2)
+            # cv2.imshow("image_np", image_np)
+            box_str = box_str[:-1]
+            f.write(image_np_path + " " + box_str + "\n")
+
+            if i < 5000:
+                box_str = ""
+                for p in tips_position_list:
+                    box_str += str(p[1]) + "," + str(p[0]) + "," + \
+                               str(p[1]+char_shape[1]) + "," + str(p[0]+char_shape[0]) + \
+                               "," + str(0) + " "
+                #     cv2.rectangle(tips_image_np, (p[1], p[0]), (p[1]+char_shape[1], p[0]+char_shape[0]), (255, 0, 0), 2)
+                # cv2.imshow("tips_image_np", tips_image_np)
+                # cv2.waitKey(0)
+                box_str = box_str[:-1]
+                f.write(tips_image_np_path + " " + box_str + "\n")
+
+
+def generate_data_yolo_puzzle(image_shape=(160, 256)):
+    bg_paths = glob("../data/base/*.jpeg")
+
+    data_dir = "../data/detect2/"
+    with open(data_dir+"map.txt", "w") as f:
+        f.write("")
+
+    for i in range(0, 10000):
+        if i % 1000 == 0:
+            print("Loop", i)
+
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        if random.choice([0, 0, 1]):
+            bg = distort_image(bg)
+        if random.choice([0, 0, 1]):
+            bg = flip_image(bg)
+
+        r = random.randint(35, 60)
+        puzzle_shape = (r, r)
+        image_np, position_list = puzzle_on_image(bg, puzzle_shape, image_shape)
+
+        image_np_path = str(i) + ".jpg"
+        cv2.imwrite(data_dir+image_np_path, image_np)
+
+        # 生成映射数据
+        with open(data_dir+"map.txt", "a") as f:
+            box_str = ""
+            for p in position_list:
+                box_str += str(p[1]) + "," + str(p[0]) + "," + \
+                           str(p[1]+puzzle_shape[1]) + "," + str(p[0]+puzzle_shape[0]) + \
+                           "," + str(0) + " "
+            #     cv2.rectangle(image_np, (p[1], p[0]), (p[1]+puzzle_shape[1], p[0]+puzzle_shape[0]), (255, 0, 0), 2)
+            # cv2.imshow("image_np", image_np)
+            # cv2.waitKey(0)
+            box_str = box_str[:-1]
+            f.write(image_np_path + " " + box_str + "\n")
+
+
+def generate_data_drag_image(image_shape=(160, 260)):
+    bg_paths = glob("../data/base/*")
+
+    data_dir = "../data/drag/"
+    with open(data_dir+"map.txt", "w") as f:
+        f.write("")
+
+    for i in range(10000):
+        if i % 1000 == 0:
+            print("Loop", i)
+
+        bg = cv2.imread(random.sample(bg_paths, 1)[0])
+        bg = pil_resize(bg, image_shape[0], image_shape[1])
+        if random.choice([0, 0, 1]):
+            bg = distort_image(bg)
+        if random.choice([0, 0, 1]):
+            bg = flip_image(bg)
+
+        image_np, clip_line = get_drag_image(bg)
+
+        image_np_path = str(i) + ".jpg"
+        cv2.imwrite(data_dir+image_np_path, image_np)
+
+        # 生成映射数据
+        with open(data_dir+"map.txt", "a") as f:
+            f.write(image_np_path + " " + str(clip_line[0][0]) + "\n")
+
+
+def generate_data_phrase():
+    data_path = os.path.dirname(os.path.abspath(__file__)) + "/../data/phrase/"
+
+    char_path = data_path+"char.txt"
+    with open(char_path, "r") as f:
+        char_list = f.readlines()
+    char_dict = {}
+    for i in range(len(char_list)):
+        char_dict[char_list[i]] = i
+
+    phrase_list = []
+    phrase_path = data_path+"phrase3.txt"
+    with open(phrase_path, "r") as f:
+        phrase_list += f.readlines()
+    phrase_path = data_path+"phrase4.txt"
+    with open(phrase_path, "r") as f:
+        phrase_list += f.readlines()
+    phrase_path = data_path+"phrase5.txt"
+    with open(phrase_path, "r") as f:
+        phrase_list += f.readlines()
+    phrase_set = set(phrase_list)
+
+    map_path = data_path+"map3.txt"
+    with open(map_path, "w") as f:
+        f.write("")
+    data_list = []
+    start_time = time.time()
+    i = 0
+    negative_way_flag = False
+    for phrase in phrase_list:
+        if i % 500000 == 0:
+            with open(map_path, "a") as f:
+                f.writelines(data_list)
+            data_list = []
+            print("Loop", i, len(phrase_list), time.time()-start_time)
+            start_time = time.time()
+        i += 1
+
+        # 正样本
+        index_list = []
+        for char in phrase[:-1]:
+            index_list.append(char_dict.get(char+"\n"))
+        data_list.append(str(index_list) + " 1\n")
+
+        # 负样本
+        if negative_way_flag:
+            index1 = random.randint(0, len(index_list)-1)
+            find_flag = False
+            while not find_flag:
+                index2 = random.randint(0, len(index_list)-1)
+                if index1 != index2:
+                    find_flag = True
+            temp = index_list[index1]
+            index_list[index1] = index_list[index2]
+            index_list[index2] = temp
+            if "".join([char_list[x][:-1] for x in index_list]) + "\n" not in phrase_set:
+                data_list.append(str(index_list) + " 0\n")
+        else:
+            products = list(product(index_list, repeat=len(index_list)))
+            random.shuffle(products)
+            negative_cnt = 0
+            for p in products:
+                if negative_cnt >= 2:
+                    break
+                p = list(p)
+                if len(set(p)) != len(p):
+                    continue
+                if p != index_list and "".join([char_list[x][:-1] for x in p]) + "\n" not in phrase_set:
+                    data_list.append(str(p) + " 0\n")
+                    negative_cnt += 1
+
+    with open(map_path, "a") as f:
+        f.writelines(data_list)
+
+
+def generate_data_phrase_raw(word_len=5):
+    paths = glob("D:/Chinese_corpus/answer/*/*.txt")
+
+    phrase_path = "../data/phrase/phrase" + str(word_len) + "_new.txt"
+    triple_list = []
+    reg = "[^\u4e00-\u9fa5]"
+    start_time = time.time()
+    for i in range(len(paths)):
+        if i % 1000 == 0:
+            with open(phrase_path, "w") as f:
+                f.writelines(triple_list)
+            print("Loop", i, len(paths), time.time()-start_time)
+            start_time = time.time()
+            triple_list = []
+
+        with open(paths[i], "rb") as f:
+            _b = f.read()
+        try:
+            text = _b.decode("gbk")
+        except:
+            try:
+                text = _b.decode("gb2312")
+            except:
+                try:
+                    text = _b.decode("gb18030")
+                except:
+                    print(chardet.detect(_b), "is None")
+
+        filter_word = ["的"]
+        for word in filter_word:
+            text = re.sub(word, "#"*len(word), text)
+
+        word_list = jieba.lcut(text, cut_all=False, HMM=True)
+
+        for j in range(1, len(word_list)):
+            current = word_list[j]
+            current_re = re.search(reg, current)
+            last = word_list[j-1]
+            last_re = re.search(reg, last)
+
+            if current_re:
+                continue
+            if len(current) == word_len:
+                triple_list.append(current + "\n")
+            elif len(current) + len(last) == word_len and not last_re:
+                triple_list.append(last+current + "\n")
+
+        triple_list = list(set(triple_list))
+
+    print("len(triple_list)", len(triple_list))
+    with open(phrase_path, "w") as f:
+        f.writelines(triple_list)
+
+
+def char_on_image(image_np, char_list, char_shape, image_shape, tip_char_num=1):
+    position_list = []
+    for char in char_list:
+        # 获取单字图片
+        char_image_pil = get_char_image(char, char_shape)
+
+        image_np = pil_resize(image_np, image_shape[0], image_shape[1])
+
+        # h, w
+        fg_w, fg_h = char_image_pil.size[:2]
+        bg_h, bg_w = image_np.shape[:2]
+
+        # 字体放置的位置，且位置不重叠
+        find_flag = 0
+        while not find_flag:
+            position_h = random.randint(0, bg_h-fg_h)
+            position_w = random.randint(0, bg_w-fg_w)
+            if len(position_list) < 1:
+                find_flag = 1
+                break
+            for p in position_list:
+                if get_iou(position_w, position_h, position_w+fg_w, position_h+fg_h,
+                           p[1], p[0], p[1]+fg_w, p[0]+fg_h) > 0:
+                    find_flag = 0
+                    break
+                else:
+                    find_flag = 1
+        position_list.append([position_h, position_w])
+        # 字体添加到背景图上
+        # image_np = get_image_roi(image_np, char_image_np, position_h, position_w)
+        image_np = get_image_paste(image_np, char_image_pil, position_h, position_w)
+
+    # 生成提示图片
+    image_list = []
+    for char in char_list[:tip_char_num]:
+        char_image_pil = get_char_image(char, char_shape, rotate=False, bg_color=(255, 255, 255, 255))
+        char_image_np = pil2np_a(char_image_pil)
+        # char_image_np = pil_resize(char_image_np, char_shape[0], char_shape[1])
+        image_list.append(char_image_np)
+    tips_image_np = np.concatenate(image_list, axis=1)
+    # 加干扰
+    tips_image_np = create_noise(tips_image_np)
+
+    # 切割
+    image_list = []
+    for i in range(tip_char_num):
+        image_list.append(tips_image_np[:, i*char_shape[1]:(i+1)*char_shape[1], :])
+    return image_np, position_list, image_list
+
+
+def get_char_image(char, char_shape, rotate=True, bg_color=(0, 0, 0, 0)):
+    # 创建空图
+    image_pil = Image.new('RGBA', (80, 80), bg_color)
+
+    # 空图上写字
+    # font_size = 35 # (40, 40)
+    font_size = 75 # (80, 80)
+    font_type_list = glob("../font/*")
+    font_type = random.sample(font_type_list, 1)[0]
+    font_config = ImageFont.truetype(font_type, int(font_size))
+    dr = ImageDraw.Draw(image_pil)
+    fill_color = random_color()
+    fill_color = (fill_color[0], fill_color[1], fill_color[2])
+    dr.text((3, -6), char, font=font_config, fill=fill_color)
+
+    if rotate:
+        if random.choice([0, 1]):
+            angle = random.randint(0, 80)
+        else:
+            angle = random.randint(280, 360)
+        image_pil = image_pil.rotate(angle, expand=False, fillcolor=bg_color)
+
+    # image_pil.show("1")
+    image_pil = image_pil.resize(char_shape)
+    # cv2.imshow("get_char_image", pil2np(image_pil))
+    # cv2.waitKey(0)
+    return image_pil
+
+
+def get_tips_image(tips_image_list, char_shape, image_shape, roatate=True):
+    new_list = []
+    for img in tips_image_list:
+        if random.choice([0, 0, 1]):
+            angle = random.randint(0, 360)
+            img = pil_rotate(img, angle, (255, 255, 255))
+        new_list.append(img)
+
+    tips_image_np = np.concatenate(new_list, axis=1)
+
+    new_image = np.full((image_shape[0], image_shape[1], 3), 0, np.uint8)
+    new_image[:tips_image_np.shape[0], :tips_image_np.shape[1], :] = tips_image_np
+
+    position_list = []
+    for i in range(len(new_list)):
+        h = 0
+        w = i*char_shape[1]
+        position_list.append([h, w])
+    return new_image, position_list
+
+
+def get_image_roi(image_bg, image_fg, roi_h, roi_w):
+    # h, w
+    fg_h, fg_w = image_fg.shape[:2]
+    bg_h, bg_w = image_bg.shape[:2]
+
+    # roi取值范围
+    roi = image_bg[roi_h:roi_h+fg_h, roi_w:roi_w+fg_w]
+
+    # 获取bg中非fg字体部分的掩码，相当于排除fg的字体部分，只保留bg的除fg字体外的部分
+    img_fg_gray = cv2.cvtColor(image_fg, cv2.COLOR_BGR2GRAY)
+    ret, mask = cv2.threshold(img_fg_gray, 0, 255, cv2.THRESH_OTSU)
+    bg_roi = cv2.bitwise_and(roi, roi, mask=mask)
+
+    # 获取fg中字体部分的掩码，相当于排除fg中的白色背景，只保留fg的字体部分
+    mask_inv = cv2.bitwise_not(mask)
+    fg_roi = cv2.bitwise_and(image_fg, image_fg, mask=mask_inv)
+    # 膨胀腐蚀去掉白色颗粒
+    # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
+    # fg_roi = cv2.erode(fg_roi, kernel)
+    # fg_roi = cv2.dilate(fg_roi, kernel)
+
+    # bg的除字体外背景部分 + fg的字体部分
+    image_roi = cv2.add(bg_roi, fg_roi)
+
+    # 将roi部分放回原bg
+    image_bg[roi_h:roi_h+fg_h, roi_w:roi_w+fg_w, :] = image_roi
+
+    # cv2.imshow("image_fg", image_fg)
+    # cv2.imshow("get_image_roi", image_bg)
+    # cv2.waitKey(0)
+    return image_bg
+
+
+def get_image_paste(image_bg, image_fg, roi_h, roi_w):
+    fg_h, fg_w = image_fg.size[:2]
+    image_bg = cv2.cvtColor(image_bg, cv2.COLOR_BGR2BGRA)
+    image_bg = np2pil_a(image_bg)
+    # image_fg = np2pil_a(image_fg)
+    image_bg.paste(image_fg, (roi_w, roi_h), image_fg)
+    image_bg = pil2np(image_bg)
+
+    # cv2.imshow("get_image_paste", image_bg)
+    # cv2.waitKey(0)
+    return image_bg
+
+
+def random_color(dims=3):
+    color = [0]*dims
+
+    find_flag = 0
+    while not find_flag:
+        for dim in range(dims):
+            color[dim] = random.randint(0, 255)
+            if color[dim] <= 125:
+                find_flag = 1
+
+    # RGB
+    # color_list = [
+    #     [207, 91, 85],
+    #     [0, 201, 88],
+    #     [117, 74, 57],
+    #     [210, 210, 27],
+    #     [160, 157, 152],
+    #     [181, 210, 210],
+    #     [27, 112, 107],
+    #     [87, 26, 44],
+    #     [115, 19, 20],
+    #     [161, 210, 68],
+    #     [210, 108, 12],
+    #     [112, 9, 142],
+    #     [50, 41, 84],
+    #     [72, 52, 210],
+    #     [210, 177, 89],
+    #     [148, 200, 89],
+    #     [173, 116, 109],
+    #     [185, 185, 210],
+    #     [181, 7, 210],
+    #     [80, 210, 30],
+    #     [65, 72, 98],
+    #     [210, 123, 109],
+    #     [19, 64, 95],
+    #     [128, 21, 210],
+    #     [129, 137, 60]
+    # ]
+    # color = random.sample(color_list, 1)[0]
+    return tuple(color)
+
+
+def create_noise(image_np):
+    ic = ImageCaptcha()
+    image_pil = np2pil(image_np)
+    image_pil = ic.create_noise_curve(image_pil, random_color())
+    image_pil = ic.create_noise_curve(image_pil, random_color())
+    image_pil = ic.create_noise_dots(image_pil, random_color())
+    image_np = pil2np(image_pil)
+    return image_np
+
+
+def get_iou(x1, y1, x2, y2, a1, b1, a2, b2):
+    # 相交区域左上角横坐标
+    ax = max(x1, a1)
+    # 相交区域左上角纵坐标
+    ay = max(y1, b1)
+    # 相交区域右下角横坐标
+    bx = min(x2, a2)
+    # 相交区域右下角纵坐标
+    by = min(y2, b2)
+
+    area_n = (x2 - x1) * (y2 - y1)
+    area_m = (a2 - a1) * (b2 - b1)
+
+    w = max(0, bx - ax)
+    h = max(0, by - ay)
+    area_x = w * h
+
+    return area_x / (area_n + area_m - area_x)
+
+
+def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes):
+    """Preprocess true boxes to training input format
+    Parameters
+    ----------
+    true_boxes: array, shape=(m, T, 5)
+        Absolute x_min, y_min, x_max, y_max, class_id relative to input_shape.
+    input_shape: array-like, hw, multiples of 32
+    anchors: array, shape=(N, 2), wh
+    num_classes: integer
+    Returns
+    -------
+    y_true: list of array, shape like yolo_outputs, xywh are reletive value
+    """
+    # print(true_boxes[..., 4])
+    # print(num_classes)
+    assert (true_boxes[..., 4] < num_classes).all(), 'class id must be less than num_classes'
+    # default setting
+    num_layers = len(anchors)//3
+    anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
+
+    true_boxes = np.array(true_boxes, dtype='float32')
+    input_shape = np.array(input_shape, dtype='int32')
+    boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2
+    boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2]
+    true_boxes[..., 0:2] = boxes_xy/input_shape[::-1]
+    true_boxes[..., 2:4] = boxes_wh/input_shape[::-1]
+
+    m = true_boxes.shape[0]
+    grid_shapes = [input_shape//{0: 32, 1: 16, 2: 8}[l] for l in range(num_layers)]
+    y_true = [np.zeros((m, grid_shapes[l][0], grid_shapes[l][1],len(anchor_mask[l]), 5+num_classes),
+                       dtype='float32') for l in range(num_layers)]
+
+    # Expand dim to apply broadcasting.
+    anchors = np.expand_dims(anchors, 0)
+    anchor_maxes = anchors / 2.
+    anchor_mins = -anchor_maxes
+    valid_mask = boxes_wh[..., 0] > 0
+
+    for b in range(m):
+        # Discard zero rows.
+        wh = boxes_wh[b, valid_mask[b]]
+        if len(wh) == 0:
+            continue
+        # Expand dim to apply broadcasting.
+        wh = np.expand_dims(wh, -2)
+        box_maxes = wh / 2.
+        box_mins = -box_maxes
+
+        intersect_mins = np.maximum(box_mins, anchor_mins)
+        intersect_maxes = np.minimum(box_maxes, anchor_maxes)
+        intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.)
+        intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
+        box_area = wh[..., 0] * wh[..., 1]
+        anchor_area = anchors[..., 0] * anchors[..., 1]
+        iou = intersect_area / (box_area + anchor_area - intersect_area)
+
+        # Find best anchor for each true box
+        best_anchor = np.argmax(iou, axis=-1)
+
+        for t, n in enumerate(best_anchor):
+            for l in range(num_layers):
+                if n in anchor_mask[l]:
+                    i = np.floor(true_boxes[b,t,0]*grid_shapes[l][1]).astype('int32')
+                    j = np.floor(true_boxes[b,t,1]*grid_shapes[l][0]).astype('int32')
+                    k = anchor_mask[l].index(n)
+                    c = true_boxes[b, t, 4].astype('int32')
+                    y_true[l][b, j, i, k, 0:4] = true_boxes[b, t, 0:4]
+                    y_true[l][b, j, i, k, 4] = 1
+                    y_true[l][b, j, i, k, 5+c] = 1
+
+    return y_true
+
+
+def get_puzzle(shape=(80, 80)):
+    # 创建空图
+    image_pil = Image.new('RGBA', (shape[1], shape[0]), (255, 255, 255, 0))
+    draw = ImageDraw.Draw(image_pil)
+
+    # 居中创建矩形
+    rec_shape = (40, 40)
+    left_up_point = [int((shape[0]-rec_shape[0])/2), int((shape[1]-rec_shape[1])/2)]
+    right_down_point = [left_up_point[0]+rec_shape[0], left_up_point[1]+rec_shape[1]]
+
+    # 透明度
+    flag = random.choice([0, 1])
+    if flag:
+        alpha = random.randint(100, 150)
+    else:
+        alpha = random.randint(160, 255)
+
+    # 背景色
+    if flag:
+        r = random.randint(0, 30)
+    else:
+        r = random.randint(100, 180)
+    # r = random.randint(0, 255)
+    fill_color = (r, r, r, alpha)
+
+    # 边缘色
+    if random.choice([0, 1, 1]):
+        if flag:
+            r = random.randint(140, 170)
+        else:
+            r = random.randint(70, 100)
+        outline_color = (r, r, r)
+    else:
+        outline_color = (fill_color[0], fill_color[1], fill_color[2])
+
+    draw.rectangle((left_up_point[1],
+                    left_up_point[0],
+                    left_up_point[1]+rec_shape[1],
+                    left_up_point[0]+rec_shape[0]),
+                   fill=fill_color,
+                   outline=outline_color)
+
+    # 拼图的圆或半圆
+    radius = random.randint(int(rec_shape[0] / 3 / 2), int(rec_shape[0] / 3 / 1.2))
+    center_list = [[left_up_point[1], int((right_down_point[0]+left_up_point[0])/2), 1],
+                   [right_down_point[1], int((right_down_point[0]+left_up_point[0])/2), 1],
+                   [int((right_down_point[1]+left_up_point[1])/2), left_up_point[0], 0],
+                   [int((right_down_point[1]+left_up_point[1])/2), right_down_point[0], 0]
+                   ]
+    circle_num = random.randint(1, 4)
+    # print("circle_num", circle_num)
+    center_list = random.sample(center_list, circle_num)
+
+    min_w, min_h = left_up_point[1], left_up_point[0]
+    max_w, max_h = right_down_point[1], right_down_point[0]
+    for center in center_list:
+        w, h = center[:2]
+        is_width = center[2]
+
+        # 判断长宽
+        into_ratio = random.randint(int(1/2*radius), int(3/4*radius))
+        if is_width:
+            # 挑选圆是凸还是凹进去
+            if random.choice([0, 1]):
+                center = (center[0]+into_ratio, center[1])
+            else:
+                center = (center[0]-into_ratio, center[1])
+        else:
+            if random.choice([0, 1]):
+                center = (center[0], center[1]+into_ratio)
+            else:
+                center = (center[0], center[1]-into_ratio)
+
+        # 判断透明度
+        color = fill_color
+        if is_width:
+            if left_up_point[1] <= center[0] <= right_down_point[1]:
+                color = (0, 0, 0, 0)
+        else:
+            if left_up_point[0] <= center[1] <= right_down_point[0]:
+                color = (0, 0, 0, 0)
+
+        # print("center, color, alpha", center, color, alpha)
+        draw.ellipse([(center[0]-radius, center[1]-radius),
+                      (center[0]+radius, center[1]+radius)],
+                     fill=color,
+                     outline=outline_color)
+
+        # 修补内部圆的边缘颜色
+        if color[3] == alpha:
+            if is_width:
+                if center[0] < w:
+                    draw.rectangle((w,
+                                    h-radius,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=fill_color)
+                else:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w,
+                                    h+radius),
+                                   fill=fill_color)
+            else:
+                if center[1] < h:
+                    draw.rectangle((w-radius,
+                                    h,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=fill_color)
+                else:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w+radius,
+                                    h),
+                                   fill=fill_color)
+        # 修补外部圆的边缘颜色
+        else:
+            if is_width:
+                if center[0] > w:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w,
+                                    h+radius),
+                                   fill=(0, 0, 0, 0))
+                else:
+                    draw.rectangle((w,
+                                    h-radius,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=(0, 0, 0, 0))
+            else:
+                if center[1] > h:
+                    draw.rectangle((center[0]-radius,
+                                    center[1]-radius,
+                                    w+radius,
+                                    h),
+                                   fill=(0, 0, 0, 0))
+                else:
+                    draw.rectangle((w-radius,
+                                    h,
+                                    center[0]+radius,
+                                    center[1]+radius),
+                                   fill=(0, 0, 0, 0))
+
+        # 新增面积
+        if color[3] == alpha:
+            if center[0]-radius <= min_w:
+                min_w = center[0]-radius
+            if center[0]+radius >= max_w:
+                max_w = center[0]+radius
+            if center[1]-radius <= min_h:
+                min_h = center[1]-radius
+            if center[1]+radius >= max_h:
+                max_h = center[1]+radius
+
+    image_pil = image_pil.crop([min_w, min_h, max_w+1, max_h+1])
+    # image_pil.show("2")
+    return image_pil
+
+
+def puzzle_on_image(image_np, puzzle_shape, image_shape):
+    position_list = []
+    # 获取拼图图片
+    puzzle_image_pil = get_puzzle()
+    puzzle_image_pil = puzzle_image_pil.resize(puzzle_shape)
+    image_np = pil_resize(image_np, image_shape[0], image_shape[1])
+
+    # h, w
+    fg_w, fg_h = puzzle_image_pil.size[:2]
+    bg_h, bg_w = image_np.shape[:2]
+    # 拼图放置的位置
+    position_h = random.randint(0, bg_h-fg_h)
+    position_w = random.randint(0, bg_w-fg_w)
+    position_list.append([position_h, position_w])
+    # for p in position_list:
+    #     cv2.rectangle(image_np, (p[1], p[0]),
+    #                   (p[1]+puzzle_shape[1], p[0]+puzzle_shape[0]),
+    #                   (0, 0, 255), 1)
+    # 拼图添加到背景图上
+    image_np = get_image_paste(image_np, puzzle_image_pil, position_h, position_w)
+    # cv2.imshow("puzzle_on_image", image_np)
+    # cv2.waitKey(0)
+    return image_np, position_list
+
+
+def distort_image(image_np, hue=.1, sat=1.5, val=1.5):
+    """
+    图像失真
+    :return:
+    """
+    def rand(a=0, b=1):
+        return np.random.rand()*(b-a) + a
+
+    # cv2.imshow("distort_image1", image_np)
+    hue = rand(-hue, hue)
+    sat = rand(1, sat) if rand() < .5 else 1/rand(1, sat)
+    val = rand(1, val) if rand() < .5 else 1/rand(1, val)
+
+    image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
+    x = rgb_to_hsv(image_np/255.)
+    x[..., 0] += hue
+    x[..., 0][x[..., 0] > 1] -= 1
+    x[..., 0][x[..., 0] < 0] += 1
+    x[..., 1] *= sat
+    x[..., 2] *= val
+    x[x > 1] = 1
+    x[x < 0] = 0
+    image_np = hsv_to_rgb(x)
+    image_np = cv2.cvtColor(np.uint8(image_np*255), cv2.COLOR_RGB2BGR)
+
+    # cv2.imshow("distort_image2", image_np)
+    # cv2.waitKey(0)
+    return image_np
+
+
+def flip_image(image_np):
+    # cv2.imshow("flip_image1", image_np)
+    if random.choice([0, 1]):
+        # 水平翻转
+        image_np = cv2.flip(image_np, 1)
+    else:
+        # 垂直翻转
+        image_np = cv2.flip(image_np, 0)
+    # cv2.imshow("flip_image2", image_np)
+    # cv2.waitKey(0)
+    return image_np
+
+
+def get_drag_image(image_np):
+    h, w = image_np.shape[:2]
+
+    # 取一定高度图片
+    clip_h = random.randint(int(1/4*h), int(3/4*h))
+    image_clip = image_np[:clip_h, ...]
+
+    # 将图片在一定宽度截断，重新拼接
+    clip_w = random.randint(int(1/6*w), int(5/6*w))
+    image_w1 = image_clip[:, :clip_w, ...]
+    image_w2 = image_clip[:, clip_w:, ...]
+    image_new = np.concatenate([image_w2, image_w1], axis=1)
+
+    # 分割线
+    clip_line = [(image_w2.shape[1], 0), (image_w2.shape[1], clip_h)]
+
+    # show
+    # print(clip_line)
+    # cv2.line(image_new, clip_line[0], clip_line[1], (0, 0, 255), 2)
+    # cv2.imshow("get_drag_image", image_new)
+    # cv2.waitKey(0)
+    return image_new, clip_line
+
+
+def get_real_data_puzzle(shape=(160, 256)):
+    paths = glob("../data/detect2_real/*")
+    i = 10000
+    for p in paths:
+        image = cv2.imread(p)
+        image = pil_resize(image, shape[0], shape[1])
+        cv2.imwrite("../data/detect2_real/"+str(i)+".jpg", image)
+        i += 1
+
+        image = distort_image(image)
+        cv2.imwrite("../data/detect2_real/"+str(i)+".jpg", image)
+        i += 1
+
+        image = flip_image(image)
+        cv2.imwrite("../data/detect2_real/"+str(i)+".jpg", image)
+        i += 1
+
+
+def read_label_puzzle():
+    paths = glob("../data/detect2_real/*.json")
+    map_path = "../data/detect2_real/map.txt"
+    with open(map_path, "a") as f:
+        for p in paths:
+            with open(p, "r") as fp:
+                _dict = json.loads(fp.read())
+            points = _dict.get("shapes")[0].get("points")
+            image_path = _dict.get("imagePath")
+            ps = [str(int(points[0][0])), str(int(points[0][1])),
+                  str(int(points[1][0])), str(int(points[1][1]))]
+            p_str = ",".join(ps)
+            f.write(image_path + " " + p_str + ",0" + "\n")
+
+
+def fix_map_txt():
+    path = "../data/map.txt"
+    with open(path, "r") as f:
+        _list = f.readlines()
+
+    with open("../data/map_new.txt", "w") as f:
+        new_list = []
+        for line in _list:
+            ss = line.split(" ")
+            ps = ss[-1][:-1].split(",")[:-1]
+
+            if random.choice([0, 1, 1, 1]):
+                pix = random.choice([1, 2, 2, 3, 3, 4, 4])
+                for i in range(len(ps)):
+                    if i < 2:
+                        ps[i] = str(int(ps[i]) - pix)
+                    else:
+                        ps[i] = str(int(ps[i]) + pix)
+            new_line = ss[0] + " " + ",".join(ps) + ",0\n"
+            new_list.append(new_line)
+            print("line", line)
+            print("new_line", new_line)
+        f.writelines(new_list)
+
+
+def get_char_map():
+    path = "../data/phrase/phrase3.txt"
+    with open(path, "r") as f:
+        _list = f.readlines()
+    path = "../data/phrase/phrase4.txt"
+    with open(path, "r") as f:
+        _list += f.readlines()
+    path = "../data/phrase/phrase5.txt"
+    with open(path, "r") as f:
+        _list += f.readlines()
+
+    _str = "".join(_list)
+    _str = re.sub("\n", "", _str)
+    _list = list(set([x+"\n" for x in _str]))
+    _list.sort(key=lambda x: x)
+    with open("../data/phrase/char.txt", "w") as f:
+        f.writelines(_list)
+
+
+if __name__ == "__main__":
+    # from click_captcha.utils import get_classes, get_anchors
+    # annotation_path = '../data/detect/map.txt'
+    # log_dir = 'yolo_data/logs/000/'
+    # classes_path = 'yolo_data/my_classes.txt'
+    # anchors_path = 'yolo_data/tiny_yolo_anchors.txt'
+    # class_names = get_classes(classes_path)
+    # num_classes = len(class_names)
+    # anchors = get_anchors(anchors_path)
+    #
+    # with open(annotation_path) as f:
+    #     lines = f.readlines()
+    # random.shuffle(lines)
+    # input_shape = (160, 256)
+    # g = gen_yolo(lines, 10, input_shape, anchors, num_classes)
+    # list(g)
+
+    generate_data_phrase()
+
+    # generate_data_yolo_puzzle()
+    # gen_yolo_puzzle()
+    # _path = "../data/base/0b16d1f1a4e017d4a7ab5779263887f1.jpeg"
+    # get_drag_image(cv2.imread(_path))
+
+    # for ii in range(10):
+    #     im = get_puzzle()
+
+    # with open("../data/chinese.txt", "r") as f:
+    #     _str = f.read()
+    #
+    # _list = [c for c in _str]
+    # _list = list(set(_list))
+    # _str = "".join(_list)
+
+    # with open("../data/chinese.txt", "w") as f:
+    #     f.write(_str)