convert_image.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047
  1. # encoding=utf8
  2. import copy
  3. import inspect
  4. import io
  5. import logging
  6. import os
  7. import re
  8. import sys
  9. import time
  10. from glob import glob
  11. import requests
  12. import numpy as np
  13. from PIL import Image
  14. sys.path.append(os.path.dirname(__file__) + "/../")
  15. from pdfminer.layout import LTLine
  16. import traceback
  17. import cv2
  18. from isr.pre_process import count_red_pixel
  19. from format_convert.utils import judge_error_code, add_div, LineTable, get_table_html, get_logger, log, \
  20. memory_decorator, pil_resize, np2bytes, ocr_cant_read
  21. from format_convert.convert_need_interface import from_otr_interface, from_ocr_interface, from_gpu_interface_redis, \
  22. from_idc_interface, from_isr_interface
  23. from format_convert.table_correct import get_rotated_image
  24. from botr.extract_table import get_table
  25. def image_process(image_np, image_path, is_from_pdf=False, is_from_docx=False,
  26. b_table_from_text=False, pdf_obj_list=[], pdf_layout_size=()):
  27. from format_convert.convert_tree import _Table, _Sentence
  28. def get_cluster(t_list, b_list, axis):
  29. zip_list = list(zip(t_list, b_list))
  30. if len(zip_list) == 0:
  31. return t_list, b_list
  32. if len(zip_list[0]) > 0:
  33. zip_list.sort(key=lambda x: x[1][axis][1])
  34. cluster_list = []
  35. margin = 5
  36. for text, bbox in zip_list:
  37. _find = 0
  38. for cluster in cluster_list:
  39. if abs(cluster[1] - bbox[axis][1]) <= margin:
  40. cluster[0].append([text, bbox])
  41. cluster[1] = bbox[axis][1]
  42. _find = 1
  43. break
  44. if not _find:
  45. cluster_list.append([[[text, bbox]], bbox[axis][1]])
  46. new_text_list = []
  47. new_bbox_list = []
  48. for cluster in cluster_list:
  49. # print("=============convert_image")
  50. # print("cluster_list", cluster)
  51. center_y = 0
  52. for text, bbox in cluster[0]:
  53. center_y += bbox[axis][1]
  54. center_y = int(center_y / len(cluster[0]))
  55. for text, bbox in cluster[0]:
  56. bbox[axis][1] = center_y
  57. new_text_list.append(text)
  58. new_bbox_list.append(bbox)
  59. # print("cluster_list", cluster)
  60. return new_text_list, new_bbox_list
  61. def merge_textbox(textbox_list, in_objs):
  62. delete_obj = []
  63. threshold = 5
  64. textbox_list.sort(key=lambda x:x.bbox[0])
  65. for k in range(len(textbox_list)):
  66. tb1 = textbox_list[k]
  67. if tb1 not in in_objs and tb1 not in delete_obj:
  68. for m in range(k+1, len(textbox_list)):
  69. tb2 = textbox_list[m]
  70. if tb2 in in_objs:
  71. continue
  72. if abs(tb1.bbox[1]-tb2.bbox[1]) <= threshold \
  73. and abs(tb1.bbox[3]-tb2.bbox[3]) <= threshold:
  74. if tb1.bbox[0] <= tb2.bbox[0]:
  75. tb1.text = tb1.text + tb2.text
  76. else:
  77. tb1.text = tb2.text + tb1.text
  78. tb1.bbox[0] = min(tb1.bbox[0], tb2.bbox[0])
  79. tb1.bbox[2] = max(tb1.bbox[2], tb2.bbox[2])
  80. delete_obj.append(tb2)
  81. for _obj in delete_obj:
  82. if _obj in textbox_list:
  83. textbox_list.remove(_obj)
  84. return textbox_list
  85. def idc_process(_image_np):
  86. # 图片倾斜校正,写入原来的图片路径
  87. # print("image_process", image_path)
  88. # g_r_i = get_rotated_image(_image_np, image_path)
  89. # if judge_error_code(g_r_i):
  90. # if is_from_docx:
  91. # return []
  92. # else:
  93. # return g_r_i
  94. # _image_np = cv2.imread(image_path)
  95. # if _image_np is None:
  96. # return []
  97. # return _image_np
  98. # if _image_np is None:
  99. # return []
  100. # idc模型实现图片倾斜校正
  101. h, w = get_best_predict_size2(_image_np, 1080)
  102. image_resize = pil_resize(_image_np, h, w)
  103. # image_resize_path = image_path.split(".")[0] + "_resize_idc." + image_path.split(".")[-1]
  104. # cv2.imwrite(image_resize_path, image_resize)
  105. # with open(image_resize_path, "rb") as f:
  106. # image_bytes = f.read()
  107. image_bytes = np2bytes(image_resize)
  108. angle = from_idc_interface(image_bytes)
  109. if judge_error_code(angle):
  110. if is_from_docx:
  111. return []
  112. else:
  113. return angle
  114. # 根据角度旋转
  115. image_pil = Image.fromarray(_image_np)
  116. _image_np = np.array(image_pil.rotate(angle, expand=1))
  117. # 写入
  118. # idc_path = image_path.split(".")[0] + "_idc." + image_path.split(".")[-1]
  119. # cv2.imwrite(idc_path, image_np)
  120. return _image_np
  121. def isr_process(_image_np):
  122. log("isr_process image shape " + str(_image_np.shape))
  123. image_np_copy = copy.deepcopy(_image_np)
  124. # isr模型去除印章
  125. _isr_time = time.time()
  126. if count_red_pixel(_image_np):
  127. # 红色像素达到一定值才过模型
  128. image_bytes = np2bytes(_image_np)
  129. _image_np = from_isr_interface(image_bytes)
  130. if judge_error_code(_image_np):
  131. if is_from_docx:
  132. return []
  133. else:
  134. return _image_np
  135. # [1]代表检测不到印章,直接返回
  136. if isinstance(_image_np, list) and _image_np == [1]:
  137. log("no seals detected!")
  138. _image_np = image_np_copy
  139. log("isr total time "+str(time.time()-_isr_time))
  140. return _image_np
  141. def ocr_process(_image_np, _threshold=2048):
  142. log("ocr_process image shape " + str(_image_np.shape))
  143. # ocr图片过大内存溢出,需resize
  144. # 大图按比例缩小,小图维持不变;若统一拉伸成固定大小如1024会爆显存
  145. ratio = (1, 1)
  146. if _image_np.shape[0] > _threshold or _image_np.shape[1] > _threshold:
  147. best_h, best_w = get_best_predict_size2(_image_np, _threshold)
  148. _image_np = pil_resize(_image_np, best_h, best_w)
  149. log("ocr_process image resize " + str(_image_np.shape))
  150. ratio = (image_np.shape[0]/best_h, image_np.shape[1]/best_w)
  151. # 大图片ocr加锁,防止爆显存
  152. # if _image_np.shape[0] >= 1024 and _image_np.shape[1] >= 1024:
  153. # file_lock = True
  154. # else:
  155. # file_lock = False
  156. # 调用ocr模型接口
  157. image_bytes = np2bytes(_image_np)
  158. text_list, bbox_list = from_ocr_interface(image_bytes, is_table=True)
  159. if judge_error_code(text_list):
  160. return text_list, text_list
  161. for i in range(len(bbox_list)):
  162. point = bbox_list[i]
  163. bbox_list[i] = [[int(point[0][0]*ratio[0]), int(point[0][1]*ratio[1])],
  164. [int(point[1][0]*ratio[0]), int(point[1][1]*ratio[1])],
  165. [int(point[2][0]*ratio[0]), int(point[2][1]*ratio[1])],
  166. [int(point[3][0]*ratio[0]), int(point[3][1]*ratio[1])]]
  167. return text_list, bbox_list
  168. def otr_process(_image_np):
  169. log("otr_process image shape " + str(_image_np.shape))
  170. # otr模型识别表格,需要图片resize成模型所需大小, 写入另一个路径
  171. best_h, best_w = get_best_predict_size(_image_np)
  172. image_resize = pil_resize(_image_np, best_h, best_w)
  173. # image_resize_path = image_path.split(".")[0] + "_resize_otr." + image_path.split(".")[-1]
  174. # cv2.imwrite(image_resize_path, image_resize)
  175. # 调用otr模型接口
  176. # with open(image_resize_path, "rb") as f:
  177. # image_bytes = f.read()
  178. image_bytes = np2bytes(image_resize)
  179. list_line = from_otr_interface(image_bytes, is_from_pdf)
  180. if judge_error_code(list_line):
  181. if is_from_docx:
  182. return []
  183. else:
  184. return list_line
  185. # otr resize后得到的bbox根据比例还原
  186. start_time = time.time()
  187. ratio = (_image_np.shape[0]/best_h, _image_np.shape[1]/best_w)
  188. for i in range(len(list_line)):
  189. point = list_line[i]
  190. list_line[i] = [int(point[0]*ratio[1]), int(point[1]*ratio[0]),
  191. int(point[2]*ratio[1]), int(point[3]*ratio[0])]
  192. log("otr resize bbox recover " + str(time.time()-start_time))
  193. return list_line
  194. def botr_process(_image_np, table_list2, text_list2, box_list2, text_box_list2, obj_in_table_list2,
  195. from_pdf=False, pdf_obj_list=[], pdf_layout_size=()):
  196. if from_pdf:
  197. # 交叉验证 ocr结果与pdf obj,暂时使用pdf提取的
  198. h_ratio = _image_np.shape[0] / pdf_layout_size[1]
  199. w_ratio = _image_np.shape[1] / pdf_layout_size[0]
  200. pdf_text_list = []
  201. pdf_box_list = []
  202. for obj in pdf_obj_list:
  203. # pdf坐标是上下颠倒的
  204. obj.bbox = (obj.bbox[0], pdf_layout_size[1]-obj.bbox[1],
  205. obj.bbox[2], pdf_layout_size[1]-obj.bbox[3])
  206. # 根据两个页面大小比例调整坐标
  207. obj.bbox = (obj.bbox[0]*w_ratio, obj.bbox[1]*h_ratio,
  208. obj.bbox[2]*w_ratio, obj.bbox[3]*h_ratio)
  209. # 剔除水印字
  210. text = re.sub('[\n ]', '', obj.get_text())
  211. if len(text) == 1 and abs(obj.bbox[0] - obj.bbox[2]) >= 70:
  212. continue
  213. pdf_box_list.append([[int(obj.bbox[0]), int(obj.bbox[3])],
  214. [],
  215. [int(obj.bbox[2]), int(obj.bbox[1])],
  216. []
  217. ])
  218. pdf_text_list.append(text)
  219. pdf_text_box_list = get_text_box_obj(pdf_text_list, pdf_box_list)
  220. text_list2 = pdf_text_list
  221. box_list2 = pdf_box_list
  222. text_box_list2 = pdf_text_box_list
  223. _text_box_list, _table_list, _obj_in_table_list = get_table(_image_np, table_list2, text_list2, box_list2, text_box_list2)
  224. # print('_text_box_list', len(_text_box_list))
  225. # print('_obj_in_table_list', len(_obj_in_table_list))
  226. # print('text_box_list2', len(text_box_list2))
  227. # print('obj_in_table_list2', len(obj_in_table_list2))
  228. # 保存无边框表格文件
  229. if _table_list:
  230. save_b_table(_image_np, text_box_list2, from_pdf)
  231. text_box_list2 += _text_box_list
  232. text_box_list2 = list(set(text_box_list2))
  233. table_list2 += _table_list
  234. obj_in_table_list2 = obj_in_table_list2.union(_obj_in_table_list)
  235. # print('text_box_list2', len(text_box_list2))
  236. # print('obj_in_table_list2', len(obj_in_table_list2))
  237. return text_box_list2, table_list2, obj_in_table_list2
  238. def table_process(list_line, list_text_boxes):
  239. # 调用现成方法形成表格
  240. try:
  241. if list_line:
  242. from format_convert.convert_tree import TableLine
  243. list_lines = []
  244. for line in list_line:
  245. list_lines.append(LTLine(1, (line[0], line[1]), (line[2], line[3])))
  246. lt = LineTable()
  247. tables, obj_in_table, _ = lt.recognize_table(list_text_boxes, list_lines, False)
  248. if not tables:
  249. return list_text_boxes, tables, obj_in_table
  250. # 合并同一行textbox
  251. # list_text_boxes = merge_textbox(list_text_boxes, obj_in_table)
  252. return list_text_boxes, tables, obj_in_table
  253. else:
  254. return list_text_boxes, [], set()
  255. except:
  256. traceback.print_exc()
  257. return [-8], [-8], [-8]
  258. def get_text_box_obj(_text_list, _bbox_list):
  259. from format_convert.convert_tree import TextBox
  260. _text_box_list = []
  261. for i in range(len(_bbox_list)):
  262. bbox = _bbox_list[i]
  263. b_text = _text_list[i]
  264. _text_box_list.append(TextBox([bbox[0][0], bbox[0][1],
  265. bbox[2][0], bbox[2][1]], b_text))
  266. return _text_box_list
  267. def save_b_table(image_np2, text_box_list2, from_pdf=False):
  268. _start_time = time.time()
  269. _path = '/data/fangjiasheng/format_conversion_maxcompute/save_b_table'
  270. # _path = 'D:/Project/format_conversion_maxcompute/save_b_table'
  271. max_index = 20000
  272. if os.path.exists(_path):
  273. file_list = glob(_path + '/*')
  274. if file_list:
  275. file_index_list = [int(re.split('[/.\\\\-]', x)[-2]) for x in file_list]
  276. file_index_list.sort(key=lambda x: x)
  277. index = file_index_list[-1] + 1
  278. else:
  279. index = 0
  280. if index > max_index:
  281. return
  282. # 文件md5
  283. from format_convert import _global
  284. _md5 = _global.get("md5")
  285. _image_path = _path + '/' + str(_md5) + '-' + str(index) + '.png'
  286. cv2.imwrite(_image_path, image_np2)
  287. log('save b_table image success!')
  288. if from_pdf:
  289. _file_path = _path + '/' + str(_md5) + '-' + str(index) + '.txt'
  290. new_text_box_list2 = [str(x) + '\n' for x in text_box_list2]
  291. with open(_file_path, 'w') as f:
  292. f.writelines(new_text_box_list2)
  293. log('save b_table txt success!')
  294. log('save_b_table cost: ' + str(time.time()-_start_time))
  295. log("into image_preprocess")
  296. try:
  297. if image_np is None:
  298. return []
  299. if image_np.shape[0] <= 20 or image_np.shape[1] <= 20:
  300. return []
  301. if not b_table_from_text:
  302. # 判断是否需要长图分割
  303. slice_flag = need_image_slice(image_np)
  304. log("need_image_slice " + str(slice_flag) + " " + str(image_np.shape))
  305. idc_flag = False
  306. image_np_list = [image_np]
  307. if slice_flag:
  308. # 方向分类
  309. image_np = idc_process(image_np)
  310. idc_flag = True
  311. if isinstance(image_np, list):
  312. return image_np
  313. # 再判断
  314. if need_image_slice(image_np):
  315. # 长图分割
  316. image_np_list = image_slice_new(image_np)
  317. if len(image_np_list) < 1:
  318. log("image_slice failed!")
  319. image_np_list = [image_np]
  320. # return [-10]
  321. all_obj_list = []
  322. _add_y = 0
  323. for image_np in image_np_list:
  324. # print("sub image shape", image_np.shape)
  325. # 整体分辨率限制
  326. threshold = 2048
  327. if image_np.shape[0] > threshold or image_np.shape[1] > threshold:
  328. h, w = get_best_predict_size2(image_np, threshold=threshold)
  329. log("global image resize " + str(image_np.shape[:2]) + " -> " + str(h) + "," + str(w))
  330. image_np = pil_resize(image_np, h, w)
  331. # 印章去除
  332. image_np = isr_process(image_np)
  333. if isinstance(image_np, list):
  334. return image_np
  335. # 文字识别
  336. text_list, box_list = ocr_process(image_np)
  337. if judge_error_code(text_list):
  338. return text_list
  339. # 判断ocr识别是否正确
  340. if ocr_cant_read(text_list, box_list) and not idc_flag:
  341. # 方向分类
  342. image_np = idc_process(image_np)
  343. # cv2.imshow("idc_process", image_np)
  344. # cv2.waitKey(0)
  345. if isinstance(image_np, list):
  346. return image_np
  347. # 文字识别
  348. text_list1, box_list_1 = ocr_process(image_np)
  349. if judge_error_code(text_list1):
  350. return text_list1
  351. # 比较字数
  352. # print("ocr process", len("".join(text_list)), len("".join(text_list1)))
  353. if len("".join(text_list)) < len("".join(text_list1)):
  354. text_list = text_list1
  355. box_list = box_list_1
  356. # 表格识别
  357. line_list = otr_process(image_np)
  358. if judge_error_code(line_list):
  359. return line_list
  360. # 生成TextBox对象
  361. text_box_list = get_text_box_obj(text_list, box_list)
  362. # 表格生成
  363. text_box_list, table_list, obj_in_table_list = table_process(line_list, text_box_list)
  364. if judge_error_code(table_list):
  365. return table_list
  366. # 无边框表格识别
  367. start_time = time.time()
  368. text_box_list, table_list, obj_in_table_list = botr_process(image_np, table_list,
  369. text_list, box_list,
  370. text_box_list,
  371. obj_in_table_list,
  372. b_table_from_text,
  373. pdf_obj_list,
  374. pdf_layout_size,
  375. )
  376. log('botr process cost: ' + str(time.time()-start_time))
  377. # 合并非表格的同一行TextBox
  378. text_box_list = merge_textbox(text_box_list, obj_in_table_list)
  379. # 对象生成
  380. obj_list = []
  381. for table in table_list:
  382. obj_list.append(_Table(table["table"], table["bbox"]))
  383. for text_box in text_box_list:
  384. if text_box not in obj_in_table_list:
  385. obj_list.append(_Sentence(text_box.get_text(), text_box.bbox))
  386. # 修正y
  387. if len(image_np_list) > 1:
  388. list_y = []
  389. for obj in obj_list:
  390. obj.y += _add_y
  391. list_y.append(obj.y)
  392. if len(list_y) > 0:
  393. _add_y = max(list_y)
  394. # 合并
  395. all_obj_list += obj_list
  396. else:
  397. all_obj_list = []
  398. table_list = []
  399. text_list = []
  400. box_list = []
  401. text_box_list = []
  402. obj_in_table_list = set()
  403. # 表格识别
  404. line_list = otr_process(image_np)
  405. if judge_error_code(line_list):
  406. return line_list
  407. # 生成TextBox对象
  408. text_box_list = get_text_box_obj(text_list, box_list)
  409. # 表格生成
  410. text_box_list, table_list, obj_in_table_list = table_process(line_list, text_box_list)
  411. if judge_error_code(table_list):
  412. return table_list
  413. # 无边框表格识别
  414. start_time = time.time()
  415. text_box_list, table_list, obj_in_table_list = botr_process(image_np, table_list,
  416. text_list, box_list,
  417. text_box_list,
  418. obj_in_table_list,
  419. b_table_from_text,
  420. pdf_obj_list,
  421. pdf_layout_size,
  422. )
  423. log('botr process cost: ' + str(time.time()-start_time))
  424. # 合并非表格的同一行TextBox
  425. text_box_list = merge_textbox(text_box_list, obj_in_table_list)
  426. # 对象生成
  427. obj_list = []
  428. for table in table_list:
  429. obj_list.append(_Table(table["table"], table["bbox"]))
  430. for text_box in text_box_list:
  431. if text_box not in obj_in_table_list:
  432. obj_list.append(_Sentence(text_box.get_text(), text_box.bbox))
  433. # 合并
  434. all_obj_list += obj_list
  435. return all_obj_list
  436. except Exception as e:
  437. log("image_preprocess error")
  438. traceback.print_exc()
  439. return [-1]
  440. @memory_decorator
  441. def picture2text(path, html=False):
  442. log("into picture2text")
  443. try:
  444. # 判断图片中表格
  445. img = cv2.imread(path)
  446. if img is None:
  447. return [-3]
  448. text = image_process(img, path)
  449. if judge_error_code(text):
  450. return text
  451. if html:
  452. text = add_div(text)
  453. return [text]
  454. except Exception as e:
  455. log("picture2text error!")
  456. print("picture2text", traceback.print_exc())
  457. return [-1]
  458. def get_best_predict_size(image_np, times=64):
  459. sizes = []
  460. for i in range(1, 100):
  461. if i*times <= 1300:
  462. sizes.append(i*times)
  463. sizes.sort(key=lambda x: x, reverse=True)
  464. min_len = 10000
  465. best_height = sizes[0]
  466. for height in sizes:
  467. if abs(image_np.shape[0] - height) < min_len:
  468. min_len = abs(image_np.shape[0] - height)
  469. best_height = height
  470. min_len = 10000
  471. best_width = sizes[0]
  472. for width in sizes:
  473. if abs(image_np.shape[1] - width) < min_len:
  474. min_len = abs(image_np.shape[1] - width)
  475. best_width = width
  476. return best_height, best_width
  477. def get_best_predict_size2(image_np, threshold=3000):
  478. h, w = image_np.shape[:2]
  479. scale = threshold / max(h, w)
  480. h = int(h * scale)
  481. w = int(w * scale)
  482. return h, w
  483. def image_slice(image_np):
  484. """
  485. slice the image if the height is to large
  486. :return:
  487. """
  488. _sum = np.average(image_np, axis=1)
  489. list_white_line = []
  490. list_ave = list(_sum)
  491. for _i in range(len(list_ave)):
  492. if (list_ave[_i] > 250).all():
  493. list_white_line.append(_i)
  494. set_white_line = set(list_white_line)
  495. width = image_np.shape[1]
  496. height = image_np.shape[0]
  497. list_images = []
  498. _begin = 0
  499. _end = 0
  500. while 1:
  501. if _end > height:
  502. break
  503. _end += width
  504. while 1:
  505. if _begin in set_white_line:
  506. break
  507. if _begin > height:
  508. break
  509. _begin += 1
  510. _image = image_np[_begin:_end, ...]
  511. list_images.append(_image)
  512. _begin = _end
  513. log("image_slice into %d parts" % (len(list_images)))
  514. return list_images
  515. def image_slice_new(image_np):
  516. """
  517. 长图分割
  518. :return:
  519. """
  520. height, width = image_np.shape[:2]
  521. image_origin = copy.deepcopy(image_np)
  522. # 去除黑边
  523. image_np = remove_black_border(image_np)
  524. # 1. 转化成灰度图
  525. image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2GRAY)
  526. # 2. 二值化
  527. ret, binary = cv2.threshold(image_np, 125, 255, cv2.THRESH_BINARY_INV)
  528. # 3. 膨胀和腐蚀操作的核函数
  529. kernal = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
  530. # 4. 膨胀一次,让轮廓突出
  531. dilation = cv2.dilate(binary, kernal, iterations=1)
  532. # dilation = np.add(np.int0(np.full(dilation.shape, 255)), -1 * np.int0(dilation))
  533. # dilation = np.uint8(dilation)
  534. # cv2.namedWindow("dilation", 0)
  535. # cv2.resizeWindow("dilation", 1000, 800)
  536. # cv2.imshow("dilation", dilation)
  537. # cv2.waitKey(0)
  538. # cv2.imwrite("error.jpg", dilation)
  539. # 按行求平均
  540. width_avg = np.average(np.float32(dilation), axis=1)
  541. zero_index = np.where(width_avg == 0.)[0]
  542. # print(height, width)
  543. # print(width_avg)
  544. # print(width_avg.shape)
  545. # print(zero_index)
  546. # print(zero_index.shape)
  547. # zero_index.sort(key=lambda x: x)
  548. # 截取范围内寻找分割点
  549. max_distance = int(width / 2)
  550. image_list = []
  551. last_h = 0
  552. for i in range(height // width + 1):
  553. h = last_h + width
  554. # 前后的分割点
  555. zero_h_after = zero_index[np.where(zero_index >= h)]
  556. zero_h_before = zero_index[np.where(zero_index <= h)]
  557. # print("last_h, h", last_h, h)
  558. # print("last_h, h", last_h, h)
  559. # print(zero_index.shape)
  560. # print("zero_h_after.shape", zero_h_after.shape)
  561. if zero_h_after.shape[0] == 0:
  562. # 最后一截
  563. last_image = image_origin[last_h:, :, :]
  564. if last_image.shape[0] <= max_distance:
  565. image_list[-1] = np.concatenate([image_list[-1], last_image], axis=0)
  566. else:
  567. image_list.append(last_image)
  568. break
  569. # 分割点距离不能太远
  570. cut_h = zero_h_after.tolist()[0]
  571. print("cut_h", cut_h)
  572. if abs(h - cut_h) <= max_distance:
  573. image_list.append(image_origin[last_h:cut_h, :, :])
  574. last_h = cut_h
  575. # 后面找不到往前找
  576. else:
  577. cut_h = zero_h_before.tolist()[-1]
  578. if abs(cut_h - h) <= max_distance:
  579. image_list.append(image_origin[last_h:cut_h, :, :])
  580. last_h = cut_h
  581. # i = 0
  582. # for im in image_list:
  583. # print(im.shape)
  584. # cv2.imwrite("error" + str(i) + ".jpg", im)
  585. # i += 1
  586. # cv2.namedWindow("im", 0)
  587. # cv2.resizeWindow("im", 1000, 800)
  588. # cv2.imshow("im", im)
  589. # cv2.waitKey(0)
  590. log("image_slice into %d parts" % (len(image_list)))
  591. return image_list
  592. def need_image_slice(image_np):
  593. h, w = image_np.shape[:2]
  594. # if h > 3000 and w < 2000:
  595. # return True
  596. if 2. <= h / w and w >= 100:
  597. return True
  598. return False
  599. def remove_black_border(img_np):
  600. try:
  601. # 阈值
  602. threshold = 100
  603. # 转换为灰度图像
  604. gray = cv2.cvtColor(img_np, cv2.COLOR_RGB2GRAY)
  605. # 获取图片尺寸
  606. h, w = gray.shape[:2]
  607. # 无法区分黑色区域超过一半的情况
  608. rowc = gray[:, int(1/2*w)]
  609. colc = gray[int(1/2*h), :]
  610. rowflag = np.argwhere(rowc > threshold)
  611. colflag = np.argwhere(colc > threshold)
  612. left, bottom, right, top = rowflag[0, 0], colflag[-1, 0], rowflag[-1, 0], colflag[0, 0]
  613. # cv2.imshow('remove_black_border', img_np[left:right, top:bottom, :])
  614. # cv2.waitKey()
  615. return img_np[left:right, top:bottom, :]
  616. except:
  617. return img_np
  618. class ImageConvert:
  619. def __init__(self, path, unique_type_dir):
  620. from format_convert.convert_tree import _Document
  621. self._doc = _Document(path)
  622. self.path = path
  623. self.unique_type_dir = unique_type_dir
  624. def init_package(self):
  625. # 各个包初始化
  626. try:
  627. with open(self.path, "rb") as f:
  628. self.image = f.read()
  629. except:
  630. log("cannot open image!")
  631. traceback.print_exc()
  632. self._doc.error_code = [-3]
  633. def convert(self):
  634. from format_convert.convert_tree import _Page, _Image
  635. self.init_package()
  636. if self._doc.error_code is not None:
  637. return
  638. _page = _Page(None, 0)
  639. _image = _Image(self.image, self.path)
  640. _page.add_child(_image)
  641. self._doc.add_child(_page)
  642. def get_html(self):
  643. try:
  644. self.convert()
  645. except:
  646. traceback.print_exc()
  647. self._doc.error_code = [-1]
  648. if self._doc.error_code is not None:
  649. return self._doc.error_code
  650. return self._doc.get_html()
  651. def image_process_old(image_np, image_path, is_from_pdf=False, is_from_docx=False, use_ocr=True):
  652. from format_convert.convert_tree import _Table, _Sentence
  653. def get_cluster(t_list, b_list, axis):
  654. zip_list = list(zip(t_list, b_list))
  655. if len(zip_list) == 0:
  656. return t_list, b_list
  657. if len(zip_list[0]) > 0:
  658. zip_list.sort(key=lambda x: x[1][axis][1])
  659. cluster_list = []
  660. margin = 5
  661. for text, bbox in zip_list:
  662. _find = 0
  663. for cluster in cluster_list:
  664. if abs(cluster[1] - bbox[axis][1]) <= margin:
  665. cluster[0].append([text, bbox])
  666. cluster[1] = bbox[axis][1]
  667. _find = 1
  668. break
  669. if not _find:
  670. cluster_list.append([[[text, bbox]], bbox[axis][1]])
  671. new_text_list = []
  672. new_bbox_list = []
  673. for cluster in cluster_list:
  674. # print("=============convert_image")
  675. # print("cluster_list", cluster)
  676. center_y = 0
  677. for text, bbox in cluster[0]:
  678. center_y += bbox[axis][1]
  679. center_y = int(center_y / len(cluster[0]))
  680. for text, bbox in cluster[0]:
  681. bbox[axis][1] = center_y
  682. new_text_list.append(text)
  683. new_bbox_list.append(bbox)
  684. # print("cluster_list", cluster)
  685. return new_text_list, new_bbox_list
  686. def merge_textbox(textbox_list, in_objs):
  687. delete_obj = []
  688. threshold = 5
  689. textbox_list.sort(key=lambda x:x.bbox[0])
  690. for k in range(len(textbox_list)):
  691. tb1 = textbox_list[k]
  692. if tb1 not in in_objs and tb1 not in delete_obj:
  693. for m in range(k+1, len(textbox_list)):
  694. tb2 = textbox_list[m]
  695. if tb2 in in_objs:
  696. continue
  697. if abs(tb1.bbox[1]-tb2.bbox[1]) <= threshold \
  698. and abs(tb1.bbox[3]-tb2.bbox[3]) <= threshold:
  699. if tb1.bbox[0] <= tb2.bbox[0]:
  700. tb1.text = tb1.text + tb2.text
  701. else:
  702. tb1.text = tb2.text + tb1.text
  703. tb1.bbox[0] = min(tb1.bbox[0], tb2.bbox[0])
  704. tb1.bbox[2] = max(tb1.bbox[2], tb2.bbox[2])
  705. delete_obj.append(tb2)
  706. for _obj in delete_obj:
  707. if _obj in textbox_list:
  708. textbox_list.remove(_obj)
  709. return textbox_list
  710. log("into image_preprocess")
  711. try:
  712. if image_np is None:
  713. return []
  714. # 整体分辨率限制
  715. if image_np.shape[0] > 2000 or image_np.shape[1] > 2000:
  716. h, w = get_best_predict_size2(image_np, threshold=2000)
  717. log("global image resize " + str(image_np.shape[:2]) + " -> " + str(h) + "," + str(w))
  718. image_np = pil_resize(image_np, h, w)
  719. # 图片倾斜校正,写入原来的图片路径
  720. # print("image_process", image_path)
  721. g_r_i = get_rotated_image(image_np, image_path)
  722. if judge_error_code(g_r_i):
  723. if is_from_docx:
  724. return []
  725. else:
  726. return g_r_i
  727. image_np = cv2.imread(image_path)
  728. image_np_copy = copy.deepcopy(image_np)
  729. if image_np is None:
  730. return []
  731. # if image_np is None:
  732. # return []
  733. #
  734. # # idc模型实现图片倾斜校正
  735. # image_resize = pil_resize(image_np, 640, 640)
  736. # image_resize_path = image_path.split(".")[0] + "_resize_idc." + image_path.split(".")[-1]
  737. # cv2.imwrite(image_resize_path, image_resize)
  738. #
  739. # with open(image_resize_path, "rb") as f:
  740. # image_bytes = f.read()
  741. # angle = from_idc_interface(image_bytes)
  742. # if judge_error_code(angle):
  743. # if is_from_docx:
  744. # return []
  745. # else:
  746. # return angle
  747. # # 根据角度旋转
  748. # image_pil = Image.fromarray(image_np)
  749. # image_np = np.array(image_pil.rotate(angle, expand=1))
  750. # # 写入
  751. # idc_path = image_path.split(".")[0] + "_idc." + image_path.split(".")[-1]
  752. # cv2.imwrite(idc_path, image_np)
  753. # isr模型去除印章
  754. _isr_time = time.time()
  755. if count_red_pixel(image_np):
  756. # 红色像素达到一定值才过模型
  757. with open(image_path, "rb") as f:
  758. image_bytes = f.read()
  759. image_np = from_isr_interface(image_bytes)
  760. if judge_error_code(image_np):
  761. if is_from_docx:
  762. return []
  763. else:
  764. return image_np
  765. # [1]代表检测不到印章,直接返回
  766. if isinstance(image_np, list) and image_np == [1]:
  767. log("no seals detected!")
  768. image_np = image_np_copy
  769. else:
  770. isr_path = image_path.split(".")[0] + "_isr." + image_path.split(".")[-1]
  771. cv2.imwrite(isr_path, image_np)
  772. log("isr total time "+str(time.time()-_isr_time))
  773. # otr模型识别表格,需要图片resize成模型所需大小, 写入另一个路径
  774. best_h, best_w = get_best_predict_size(image_np)
  775. # image_resize = cv2.resize(image_np, (best_w, best_h), interpolation=cv2.INTER_AREA)
  776. image_resize = pil_resize(image_np, best_h, best_w)
  777. image_resize_path = image_path.split(".")[0] + "_resize_otr." + image_path.split(".")[-1]
  778. cv2.imwrite(image_resize_path, image_resize)
  779. # 调用otr模型接口
  780. with open(image_resize_path, "rb") as f:
  781. image_bytes = f.read()
  782. list_line = from_otr_interface(image_bytes, is_from_pdf)
  783. if judge_error_code(list_line):
  784. return list_line
  785. # # 预处理
  786. # if is_from_pdf:
  787. # prob = 0.2
  788. # else:
  789. # prob = 0.5
  790. # with open(image_resize_path, "rb") as f:
  791. # image_bytes = f.read()
  792. # img_new, inputs = table_preprocess(image_bytes, prob)
  793. # if type(img_new) is list and judge_error_code(img_new):
  794. # return img_new
  795. # log("img_new.shape " + str(img_new.shape))
  796. #
  797. # # 调用模型运行接口
  798. # _dict = {"inputs": inputs, "md5": _global.get("md5")}
  799. # result = from_gpu_interface(_dict, model_type="otr", predictor_type="")
  800. # if judge_error_code(result):
  801. # logging.error("from_gpu_interface failed! " + str(result))
  802. # raise requests.exceptions.RequestException
  803. #
  804. # pred = result.get("preds")
  805. # gpu_time = result.get("gpu_time")
  806. # log("otr model predict time " + str(gpu_time))
  807. #
  808. # # # 解压numpy
  809. # # decompressed_array = io.BytesIO()
  810. # # decompressed_array.write(pred)
  811. # # decompressed_array.seek(0)
  812. # # pred = np.load(decompressed_array, allow_pickle=True)['arr_0']
  813. # # log("inputs.shape" + str(pred.shape))
  814. #
  815. # 调用gpu共享内存处理
  816. # _dict = {"inputs": inputs, "md5": _global.get("md5")}
  817. # result = from_gpu_share_memory(_dict, model_type="otr", predictor_type="")
  818. # if judge_error_code(result):
  819. # logging.error("from_gpu_interface failed! " + str(result))
  820. # raise requests.exceptions.RequestException
  821. #
  822. # pred = result.get("preds")
  823. # gpu_time = result.get("gpu_time")
  824. # log("otr model predict time " + str(gpu_time))
  825. #
  826. # # 后处理
  827. # list_line = table_postprocess(img_new, pred, prob)
  828. # log("len(list_line) " + str(len(list_line)))
  829. # if judge_error_code(list_line):
  830. # return list_line
  831. # otr resize后得到的bbox根据比例还原
  832. start_time = time.time()
  833. ratio = (image_np.shape[0]/best_h, image_np.shape[1]/best_w)
  834. for i in range(len(list_line)):
  835. point = list_line[i]
  836. list_line[i] = [int(point[0]*ratio[1]), int(point[1]*ratio[0]),
  837. int(point[2]*ratio[1]), int(point[3]*ratio[0])]
  838. log("otr resize bbox recover " + str(time.time()-start_time))
  839. # ocr图片过大内存溢出,需resize
  840. start_time = time.time()
  841. threshold = 3000
  842. ocr_resize_flag = 0
  843. if image_np.shape[0] >= threshold or image_np.shape[1] >= threshold:
  844. ocr_resize_flag = 1
  845. best_h, best_w = get_best_predict_size2(image_np, threshold)
  846. # image_resize = cv2.resize(image_np, (best_w, best_h), interpolation=cv2.INTER_AREA)
  847. image_resize = pil_resize(image_np, best_h, best_w)
  848. log("ocr_process image resize " + str(image_resize.shape))
  849. image_resize_path = image_path.split(".")[0] + "_resize_ocr." + image_path.split(".")[-1]
  850. cv2.imwrite(image_resize_path, image_resize)
  851. log("ocr resize before " + str(time.time()-start_time))
  852. # 调用ocr模型接口
  853. with open(image_resize_path, "rb") as f:
  854. image_bytes = f.read()
  855. text_list, bbox_list = from_ocr_interface(image_bytes, is_table=True)
  856. if judge_error_code(text_list):
  857. return text_list
  858. # # PaddleOCR内部包括预处理,调用模型运行接口,后处理
  859. # paddle_ocr = PaddleOCR(use_angle_cls=True, lang="ch")
  860. # results = paddle_ocr.ocr(image_resize, det=True, rec=True, cls=True)
  861. # # 循环每张图片识别结果
  862. # text_list = []
  863. # bbox_list = []
  864. # for line in results:
  865. # # print("ocr_interface line", line)
  866. # text_list.append(line[-1][0])
  867. # bbox_list.append(line[0])
  868. # if len(text_list) == 0:
  869. # return []
  870. # ocr resize后的bbox还原
  871. if ocr_resize_flag:
  872. ratio = (image_np.shape[0]/best_h, image_np.shape[1]/best_w)
  873. else:
  874. ratio = (1, 1)
  875. for i in range(len(bbox_list)):
  876. point = bbox_list[i]
  877. bbox_list[i] = [[int(point[0][0]*ratio[1]), int(point[0][1]*ratio[0])],
  878. [int(point[1][0]*ratio[1]), int(point[1][1]*ratio[0])],
  879. [int(point[2][0]*ratio[1]), int(point[2][1]*ratio[0])],
  880. [int(point[3][0]*ratio[1]), int(point[3][1]*ratio[0])]]
  881. # 调用现成方法形成表格
  882. try:
  883. from format_convert.convert_tree import TableLine
  884. list_lines = []
  885. for line in list_line:
  886. list_lines.append(LTLine(1, (line[0], line[1]), (line[2], line[3])))
  887. from format_convert.convert_tree import TextBox
  888. list_text_boxes = []
  889. for i in range(len(bbox_list)):
  890. bbox = bbox_list[i]
  891. b_text = text_list[i]
  892. list_text_boxes.append(TextBox([bbox[0][0], bbox[0][1],
  893. bbox[2][0], bbox[2][1]], b_text))
  894. # for _textbox in list_text_boxes:
  895. # print("==",_textbox.get_text())
  896. lt = LineTable()
  897. tables, obj_in_table, _ = lt.recognize_table(list_text_boxes, list_lines, False)
  898. # 合并同一行textbox
  899. list_text_boxes = merge_textbox(list_text_boxes, obj_in_table)
  900. obj_list = []
  901. for table in tables:
  902. obj_list.append(_Table(table["table"], table["bbox"]))
  903. for text_box in list_text_boxes:
  904. if text_box not in obj_in_table:
  905. obj_list.append(_Sentence(text_box.get_text(), text_box.bbox))
  906. return obj_list
  907. except:
  908. traceback.print_exc()
  909. return [-8]
  910. except Exception as e:
  911. log("image_preprocess error")
  912. traceback.print_exc()
  913. return [-1]
  914. if __name__ == "__main__":
  915. image_slice_new(cv2.imread("C:/Users/Administrator/Desktop/test_image/error28.jpg"))