update

format_convert/convert_image.py

@@ -8,23 +8,19 @@ sys.path.append(os.path.dirname(__file__) + "/../")
 import traceback
 import cv2
 from format_convert import get_memory_info
-from format_convert.utils import judge_error_code, add_div, LineTable
+from format_convert.utils import judge_error_code, add_div, LineTable, get_table_html
 from format_convert.table_correct import get_rotated_image
 from format_convert.convert_need_interface import from_otr_interface, from_ocr_interface
 
 
 def image_preprocess(image_np, image_path, use_ocr=True):
+    from format_convert.convert_tree import _Table, _Sentence
     logging.info("into image_preprocess")
     try:
-        # 长 宽
-        # resize_size = (1024, 768)
-        # 限制图片大小
-        # resize_image(image_path, resize_size)
-
         # 图片倾斜校正，写入原来的图片路径
         g_r_i = get_rotated_image(image_np, image_path)
         if g_r_i == [-1]:
-            return [-1], [], [], 0
+            return [-1]
 
         # otr需要图片resize, 写入另一个路径
         image_np = cv2.imread(image_path)
@@ -37,109 +33,59 @@ def image_preprocess(image_np, image_path, use_ocr=True):
         # 调用otr模型接口
         with open(image_resize_path, "rb") as f:
             image_bytes = f.read()
-        points, split_lines, bboxes, outline_points, lines = from_otr_interface(image_bytes)
-        if judge_error_code(points):
-            return points, [], [], 0
+        list_line = from_otr_interface(image_bytes)
+        if judge_error_code(list_line):
+            return list_line, [], [], 0
 
         # 将resize后得到的bbox根据比例还原
         ratio = (image_np.shape[0]/best_h, image_np.shape[1]/best_w)
-        for i in range(len(bboxes)):
-            bbox = bboxes[i]
-            bboxes[i] = [(int(bbox[0][0]*ratio[1]), int(bbox[0][1]*ratio[0])),
-                         (int(bbox[1][0]*ratio[1]), int(bbox[1][1]*ratio[0]))]
-        for i in range(len(split_lines)):
-            line = split_lines[i]
-            split_lines[i] = [(int(line[0][0]*ratio[1]), int(line[0][1]*ratio[0])),
-                              (int(line[1][0]*ratio[1]), int(line[1][1]*ratio[0]))]
-        for i in range(len(points)):
-            point = points[i]
-            points[i] = (int(point[0]*ratio[1]), int(point[1]*ratio[0]))
-
-        for i in range(len(outline_points)):
-            point = outline_points[i]
-            outline_points[i] = [(int(point[0][0]*ratio[1]), int(point[0][1]*ratio[0])),
-                                 (int(point[1][0]*ratio[1]), int(point[1][1]*ratio[0]))]
-
-        for i in range(len(lines)):
-            point = lines[i]
-            lines[i] = [int(point[0]*ratio[1]), int(point[1]*ratio[0]),
-                        int(point[2]*ratio[1]), int(point[3]*ratio[0])]
-
-        # 查看是否能输出正确框
-        for box in bboxes:
-            cv2.rectangle(image_np, box[0], box[1], (0, 255, 0), 2)
-            # cv2.namedWindow('bbox', 0)
-            # cv2.imshow("bbox", image_np)
-            # cv2.waitKey(0)
+        for i in range(len(list_line)):
+            point = list_line[i]
+            list_line[i] = [int(point[0]*ratio[1]), int(point[1]*ratio[0]),
+                            int(point[2]*ratio[1]), int(point[3]*ratio[0])]
 
         # 调用ocr模型接口
         with open(image_path, "rb") as f:
             image_bytes = f.read()
-        # 有表格
-        if len(bboxes) >= 2:
-            text_list, bbox_list = from_ocr_interface(image_bytes, True)
-            if judge_error_code(text_list):
-                return text_list, [], [], 0
-
-            # for i in range(len(text_list)):
-            #     print(text_list[i], bbox_list[i])
-            # 查看是否能输出正确框
-
-            # for box in bbox_list:
-            #     cv2.rectangle(image_np, (int(box[0][0]), int(box[0][1])),
-            #                   (int(box[2][0]), int(box[2][1])), (255, 0, 0), 1)
-            #     cv2.namedWindow('bbox', 0)
-            #     cv2.imshow("bbox", image_np)
-            #     cv2.waitKey(0)
-
-            # text, column_list = get_formatted_table(text_list, bbox_list, bboxes, split_lines)
-            # 调用现成方法形成表格
-            try:
-                from format_convert.convert_tree import TableLine
-                list_lines = []
-                for line in lines:
-                    list_lines.append(LTLine(1, (line[0], line[1]), (line[2], line[3])))
-                from format_convert.convert_tree import TextBox
-                list_text_boxes = []
-                print("=============1")
-                for i in range(len(bbox_list)):
-                    bbox = bbox_list[i]
-                    b_text = text_list[i]
-                    print("text:",b_text,"bbox:",bbox)
-                    list_text_boxes.append(TextBox([bbox[0][0], bbox[0][1],
-                                                    bbox[2][0], bbox[2][1]], b_text))
-
-                lt = LineTable()
-                tables, obj_in_table, _ = lt.recognize_table(list_text_boxes, list_lines,False)
-                text = [tables, obj_in_table]
-                column_list = []
-            except:
-                traceback.print_exc()
-                text = [-8]
-                column_list = []
-
-            if judge_error_code(text):
-                return text, [], [], 0
-            is_table = 1
-            return text, column_list, outline_points, is_table
-
-        # 无表格
-        else:
-            if use_ocr:
-                text = from_ocr_interface(image_bytes)
-                if judge_error_code(text):
-                    return text, [], [], 0
-
-                is_table = 0
-                return text, [], [], is_table
-            else:
-                is_table = 0
-                return None, [], [], is_table
+        text_list, bbox_list = from_ocr_interface(image_bytes, True)
+        if judge_error_code(text_list):
+            return text_list, [], [], 0
+
+        # 调用现成方法形成表格
+        try:
+            from format_convert.convert_tree import TableLine
+            list_lines = []
+            for line in list_line:
+                list_lines.append(LTLine(1, (line[0], line[1]), (line[2], line[3])))
+            from format_convert.convert_tree import TextBox
+            list_text_boxes = []
+            print("=============1")
+            for i in range(len(bbox_list)):
+                bbox = bbox_list[i]
+                b_text = text_list[i]
+                print("text:",b_text,"bbox:",bbox)
+                list_text_boxes.append(TextBox([bbox[0][0], bbox[0][1],
+                                                bbox[2][0], bbox[2][1]], b_text))
+            lt = LineTable()
+            tables, obj_in_table, _ = lt.recognize_table(list_text_boxes, list_lines, False)
+            text = [tables, obj_in_table]
+            column_list = []
+
+            obj_list = []
+            for table in tables:
+                obj_list.append(_Table(table["table"], table["bbox"]))
+            for text_box in list_text_boxes:
+                if text_box not in obj_in_table:
+                    obj_list.append(_Sentence(text_box.get_text(), text_box.bbox))
+            return obj_list
+        except:
+            traceback.print_exc()
+            return [-8]
 
     except Exception as e:
         logging.info("image_preprocess error")
         print("image_preprocess", traceback.print_exc())
-        return [-1], [], [], 0
+        return [-1]
 
 
 @get_memory_info.memory_decorator

format_convert/convert_need_interface.py

@@ -91,7 +91,7 @@ def from_ocr_interface(image_stream, is_table=False):
 
 
 @get_memory_info.memory_decorator
-def from_otr_interface(image_stream):
+def from_otr_interface2(image_stream):
     logging.info("into from_otr_interface")
     try:
         base64_stream = base64.b64encode(image_stream)
@@ -131,4 +131,32 @@ def from_otr_interface(image_stream):
     except Exception as e:
         logging.info("from_otr_interface error!")
         print("from_otr_interface", traceback.print_exc())
-        return [-1], [-1], [-1], [-1], [-1]
+        return [-1], [-1], [-1], [-1], [-1]
+
+
+def from_otr_interface(image_stream):
+    logging.info("into from_otr_interface")
+    try:
+        base64_stream = base64.b64encode(image_stream)
+
+        # 调用接口
+        try:
+            if globals().get("global_otr_model") is None:
+                globals().update({"global_otr_model": OtrModels().get_model()})
+                print("=========== init otr model ===========")
+            r = otr(data=base64_stream, otr_model=globals().get("global_otr_model"))
+        except TimeoutError:
+            return [-5]
+        except requests.exceptions.ConnectionError as e:
+            logging.info("from_otr_interface")
+            print("from_otr_interface", traceback.print_exc())
+            return [-2]
+
+        # 处理结果
+        _dict = r
+        list_line = eval(_dict.get("list_line"))
+        return list_line
+    except Exception as e:
+        logging.info("from_otr_interface error!")
+        print("from_otr_interface", traceback.print_exc())
+        return [-1]

format_convert/convert_pdf.py

@@ -713,9 +713,9 @@ class PDFConvert:
                 list_sentences = ParseUtils.recognize_sentences(lt_text_list, filter_objs,
                                                                 layout.bbox, page_no)
                 for sentence in list_sentences:
-                    _sen = _Sentence(sentence.text)
-                    _sen.x = sentence.x0
-                    _sen.y = sentence.y0
+                    _sen = _Sentence(sentence.text, sentence.bbox)
+                    # _sen.x = sentence.x0
+                    # _sen.y = sentence.y0
                     # self._page.children.append(_sen)
                     self._page.add_child(_sen)
             except:
@@ -748,9 +748,9 @@ class PDFConvert:
                         self._page.add_child(_image)
                     return
                 else:
-                    _sen = _Sentence(object_text)
-                    _sen.x = x.bbox[0]
-                    _sen.y = x.bbox[1]
+                    _sen = _Sentence(object_text, x.bbox)
+                    # _sen.x = x.bbox[0]
+                    # _sen.y = x.bbox[1]
                     self._page.add_child(_sen)
 
             # 图表对象

format_convert/convert_tree.py

@@ -97,7 +97,6 @@ class _Image:
     def get_html(self):
         # 将Image转为Sentence,table
         self.convert()
-        print("Image", self.error_code)
         if self.error_code is not None:
             return ""
 
@@ -121,22 +120,12 @@ class _Image:
         # 二进制转numpy
         image_np = Image.open(io.BytesIO(self.content))
         image_np = cv2.cvtColor(np.asarray(image_np), cv2.COLOR_RGB2BGR)
-        text, column_list, outline_points, is_table = image_preprocess(image_np,
-                                                                       self.path,
-                                                                       use_ocr=True)
-        print("is_table", is_table)
-        for t in text:
-            print(t)
-        if judge_error_code(text):
-            self.error_code = text
+        obj_list = image_preprocess(image_np, self.path, use_ocr=True)
+        if judge_error_code(obj_list):
+            self.error_code = obj_list
             return
-        if is_table:
-            tables, in_objs = text
-            self.in_table_objs = in_objs
-            for table in tables:
-                self.add_child(_Table(table["table"], table["bbox"]))
-        else:
-            self.add_child(_Sentence(text))
+        for obj in obj_list:
+            self.add_child(obj)
 
 
 class _Table:
@@ -158,16 +147,18 @@ class _Table:
 
 
 class _Sentence:
-    def __init__(self, content):
+    def __init__(self, content, bbox):
         self.content = content
         # 位置
-        self.x = 0
-        self.y = 0
+        self.bbox = bbox
+        self.x = bbox[0]
+        self.y = bbox[1]
         self.error_code = None
 
     def get_html(self):
         if self.error_code is not None:
             return ""
+        print("===========_Sentence get_html", self.content)
         return add_div(self.content)
 
 

format_convert/utils.py

@@ -37,8 +37,8 @@ def add_div(text):
     if text == "" or text is None:
         return text
 
-    if get_platform() == "Windows":
-        print("add_div", text)
+    # if get_platform() == "Windows":
+    #     print("add_div", text)
     if re.findall("<div>", text):
         return text
 
@@ -503,7 +503,7 @@ def slash_replace(_str, reverse=False):
     return _str
 
 
-class LineTable():
+class LineTable:
     def recognize_table(self,list_textbox, list_line,sourceP_LB=True):
         self.list_line = list_line
         self.list_crosspoints = self.recognize_crosspoints(list_line)
@@ -1066,7 +1066,7 @@ def sort_object(obj_list):
     if len(obj_list) == 0:
         return obj_list
     if isinstance(obj_list[0], (_Table, _Sentence, _Image)):
-        obj_list.sort(key=lambda x: x.y, reverse=True)
+        obj_list.sort(key=lambda x: x.y)
         return obj_list
     elif isinstance(obj_list[0], _Page):
         obj_list.sort(key=lambda x: x.page_no)

otr/otr_interface.py

@@ -31,14 +31,14 @@ def otr(data, otr_model):
     try:
         img_data = base64.b64decode(data)
         # points_and_lines = pool.apply(table_detect, (img_data,))
-        points_and_lines = table_detect(img_data, otr_model)
-        return points_and_lines
+        list_lines = line_detect(img_data, otr_model)
+        return list_lines
     except TimeoutError:
         raise TimeoutError
 
 flag = 0
 # model_path = "models/table-line.h5"
-def table_detect(img_data, otr_model):
+def table_detect2(img_data, otr_model):
     logging.info("into otr_interface table_detect")
     start_time = time.time()
     try:
@@ -263,6 +263,40 @@ def table_detect(img_data, otr_model):
                 "outline_points": str([]), "lines": str([])}
 
 
+def line_detect(img_data, otr_model):
+    logging.info("into otr_interface table_detect")
+    start_time = time.time()
+    try:
+        start_time1 = time.time()
+        # 二进制数据流转np.ndarray [np.uint8: 8位像素]
+        img = cv2.imdecode(np.frombuffer(img_data, np.uint8), cv2.IMREAD_COLOR)
+        # logging.info("into otr_interface table_detect 1")
+        # cv2.imwrite("111111.jpg", img)
+
+        # 将bgr转为rbg
+        image_np = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
+        # logging.info("into otr_interface table_detect 2")
+
+        # 选择与图片最接近分辨率，以防失真
+        # best_h, best_w = get_best_predict_size(img)
+        print("image_np.shape", image_np.shape)
+        best_h, best_w, _ = image_np.shape
+        logging.info("otr preprocess time: " + str(round(float(time.time()-start_time1), 4)) + "s")
+
+        # 调用模型
+        # rows, cols = table_line(image_np, otr_model)
+        start_time1 = time.time()
+        list_line = table_line(image_np, otr_model, size=(best_w, best_h), hprob=0.5, vprob=0.5)
+        return {"list_line": str(list_line)}
+    except TimeoutError:
+        raise TimeoutError
+    except Exception as e:
+        logging.info("otr_interface cannot detected table!")
+        print("otr_interface cannot detected table!", traceback.print_exc())
+        logging.info("otr postprocess time: " + str(round(float(time.time()-start_time1), 4)) + "s")
+        return {"list_line": str([])}
+
+
 class OtrModels:
     def __init__(self):
         # python文件所在目录

otr/table_line.py

@@ -197,7 +197,7 @@ def table_net(input_shape=(1152, 896, 3), num_classes=1):
 model = table_net((None, None, 3), 2)
 
 
-def drawpixel(pred):
+def draw_pixel(pred):
     import matplotlib.pyplot as plt
     _array = []
     for _h in range(len(pred)):
@@ -214,7 +214,9 @@ def drawpixel(pred):
     plt.imshow(np.array(_array))
     plt.show()
 
-def points2lines(pred,sourceP_LB=True,prob=0.2,line_width=7,padding=3,min_len=10,cell_width=13):
+
+def points2lines(pred, sourceP_LB=True, prob=0.2, line_width=7, padding=3, min_len=10,
+                 cell_width=13):
 
     def inBbox(bbox,point,line_width):
         x,y = point
@@ -360,13 +362,23 @@ def points2lines(pred,sourceP_LB=True,prob=0.2,line_width=7,padding=3,min_len=10
     #     plt.plot([x0,x1],[y0,y1])
     # for point in list_crosspoints:
     #     plt.scatter(point.get("point")[0],point.get("point")[1])
-    plt.show()
+    # plt.show()
+    return list_line
+
 
+def table_line(img, model, size=(512, 1024), hprob=0.5, vprob=0.5, row=50, col=30, alph=15):
+    sizew, sizeh = size
+    img_new = cv2.resize(img, (sizew, sizeh), interpolation=cv2.INTER_AREA)
 
+    pred = model.predict(np.array([img_new]))
+    pred = pred[0]
 
+    _time = time.time()
+    list_line = points2lines(pred, False)
+    return list_line
 
 
-def table_line(img, model, size=(512, 1024), hprob=0.5, vprob=0.5, row=50, col=30, alph=15):
+def table_line2(img, model, size=(512, 1024), hprob=0.5, vprob=0.5, row=50, col=30, alph=15):
     sizew, sizeh = size
     # [..., ::-1] 最后一维内部反向输出
     # inputBlob, fx, fy = letterbox_image(img[..., ::-1], (sizew, sizeh))
@@ -379,7 +391,7 @@ def table_line(img, model, size=(512, 1024), hprob=0.5, vprob=0.5, row=50, col=3
     # logging.info("into table_line 2")
     pred = pred[0]
 
-    drawpixel(pred)
+    draw_pixel(pred)
     _time = time.time()
     points2lines(pred)
     logging.info("points2lines takes %ds"%(time.time()-_time))

result.html

@@ -1,40 +1,45 @@
-<!DOCTYPE HTML><head><meta charset="UTF-8"></head><body><table border="1">
-<tr>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-</tr>
+<!DOCTYPE HTML><head><meta charset="UTF-8"></head><body><div>华池县柔远镇李庄肉牛养殖场建设项目配</div>
+<div>套设备购置政府采购公开招标中标公告</div>
+<div>、项目编号</div>
+<div>HCZC2021-0001</div>
+<div>二、项目名称</div>
+<div>华池县柔远镇李庄肉牛养殖场建设项目配套设备购置</div>
+<div>三、中标（成交）信息</div>
+<table border="1">
 <tr>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
+<td colspan=1 rowspan=1>供应商名称</td>
+<td colspan=1 rowspan=1>供应商联系地址</td>
+<td colspan=1 rowspan=1>中标金额（万元）</td>
 </tr>
 <tr>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
+<td colspan=1 rowspan=1>华池县卓泰机械设备租赁有限公司</td>
+<td colspan=1 rowspan=1>甘肃省庆阳市华池县柔远镇张川村</td>
+<td colspan=1 rowspan=1>72.3500</td>
 </tr>
 </table>
+<div>四、主要标的信息</div>
+<div>货物类</div>
 <table border="1">
 <tr>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
+<td colspan=1 rowspan=1>供应商名称</td>
+<td colspan=1 rowspan=1>名称</td>
+<td colspan=1 rowspan=1>品牌</td>
+<td colspan=1 rowspan=1>数量</td>
+<td colspan=1 rowspan=1>单价</td>
+<td colspan=1 rowspan=1>规格型号</td>
 </tr>
 <tr>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
-<td colspan=1 rowspan=1></td>
+<td colspan=1 rowspan=1>华池县卓泰机械设备租赁有限公司</td>
+<td colspan=1 rowspan=1>华池县柔远镇李庄肉牛养殖场建设项目配套设备购置</td>
+<td colspan=1 rowspan=1>详见附件</td>
+<td colspan=1 rowspan=1>详见附件</td>
+<td colspan=1 rowspan=1>详见附件</td>
+<td colspan=1 rowspan=1>详见附件</td>
 </tr>
 </table>
+<div>五、评审专家（单一来源采购人员）名单：</div>
+<div>王正刚、段海龙、李鑫、刘翠平、张武峰</div>
+<div>六、代理服务收费标准及金额：</div>
 <div>收费标准：无</div>
 <div>收费金额：0万元</div>
 <div>七、公告期限</div>
@@ -54,4 +59,4 @@
 <div>项目联系人：孙治江</div>
 <div>电话：18793418165</div>
 <div>2</div>
-<div></div></body>
+</body>