convert.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749
  1. #-*- coding: utf-8 -*-
  2. import gc
  3. import json
  4. import sys
  5. import os
  6. sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
  7. # 强制tf使用cpu
  8. os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
  9. from format_convert.utils import judge_error_code, request_post, get_intranet_ip, get_ip_port, get_logger, log, \
  10. set_flask_global, get_md5_from_bytes, memory_decorator
  11. from format_convert.convert_doc import doc2text, DocConvert
  12. from format_convert.convert_docx import docx2text, DocxConvert
  13. from format_convert.convert_image import picture2text, ImageConvert
  14. from format_convert.convert_pdf import pdf2text, PDFConvert
  15. from format_convert.convert_rar import rar2text, RarConvert
  16. from format_convert.convert_swf import swf2text, SwfConvert
  17. from format_convert.convert_txt import txt2text, TxtConvert
  18. from format_convert.convert_xls import xls2text, XlsConvert
  19. from format_convert.convert_xlsx import xlsx2text, XlsxConvert
  20. from format_convert.convert_zip import zip2text, ZipConvert
  21. import hashlib
  22. from format_convert.judge_platform import get_platform
  23. from ocr import ocr_interface
  24. from otr import otr_interface
  25. import re
  26. import shutil
  27. import base64
  28. import time
  29. import uuid
  30. import logging
  31. from bs4 import BeautifulSoup
  32. from flask import Flask, request, g
  33. import inspect
  34. logging.getLogger("pdfminer").setLevel(logging.WARNING)
  35. from format_convert.table_correct import *
  36. import logging
  37. from format_convert.wrapt_timeout_decorator import *
  38. from format_convert import _global
  39. port_num = [0]
  40. def choose_port():
  41. process_num = 4
  42. if port_num[0] % process_num == 0:
  43. _url = local_url + ":15011"
  44. elif port_num[0] % process_num == 1:
  45. _url = local_url + ":15012"
  46. elif port_num[0] % process_num == 2:
  47. _url = local_url + ":15013"
  48. elif port_num[0] % process_num == 3:
  49. _url = local_url + ":15014"
  50. port_num[0] = port_num[0] + 1
  51. return _url
  52. @memory_decorator
  53. def getText(_type, path_or_stream):
  54. @timeout(300, timeout_exception=TimeoutError, use_signals=False)
  55. def get_html_1(_class):
  56. return _class.get_html()
  57. @timeout(600, timeout_exception=TimeoutError, use_signals=False)
  58. def get_html_2(_class):
  59. return _class.get_html()
  60. log("file type - " + _type)
  61. try:
  62. ss = path_or_stream.split(".")
  63. unique_type_dir = ss[-2] + "_" + ss[-1] + os.sep
  64. except:
  65. unique_type_dir = path_or_stream + "_" + _type + os.sep
  66. if _type == "pdf":
  67. # return PDFConvert(path_or_stream, unique_type_dir).get_html()
  68. return get_html_1(PDFConvert(path_or_stream, unique_type_dir))
  69. if _type == "docx":
  70. # return DocxConvert(path_or_stream, unique_type_dir).get_html()
  71. return get_html_1(DocxConvert(path_or_stream, unique_type_dir))
  72. if _type == "zip":
  73. return ZipConvert(path_or_stream, unique_type_dir).get_html()
  74. # return get_html_2(ZipConvert(path_or_stream, unique_type_dir))
  75. if _type == "rar":
  76. return RarConvert(path_or_stream, unique_type_dir).get_html()
  77. # return get_html_2(RarConvert(path_or_stream, unique_type_dir))
  78. if _type == "xlsx":
  79. # return XlsxConvert(path_or_stream, unique_type_dir).get_html()
  80. return get_html_1(XlsxConvert(path_or_stream, unique_type_dir))
  81. if _type == "xls":
  82. # return XlsConvert(path_or_stream, unique_type_dir).get_html()
  83. return get_html_1(XlsConvert(path_or_stream, unique_type_dir))
  84. if _type == "doc":
  85. # return DocConvert(path_or_stream, unique_type_dir).get_html()
  86. return get_html_1(DocConvert(path_or_stream, unique_type_dir))
  87. if _type == "jpg" or _type == "png" or _type == "jpeg":
  88. # return ImageConvert(path_or_stream, unique_type_dir).get_html()
  89. return get_html_1(ImageConvert(path_or_stream, unique_type_dir))
  90. if _type == "swf":
  91. # return SwfConvert(path_or_stream, unique_type_dir).get_html()
  92. return get_html_1(SwfConvert(path_or_stream, unique_type_dir))
  93. if _type == "txt":
  94. # return TxtConvert(path_or_stream, unique_type_dir).get_html()
  95. return get_html_1(TxtConvert(path_or_stream, unique_type_dir))
  96. return [""]
  97. def to_html(path, text):
  98. with open(path, 'w',encoding="utf8") as f:
  99. f.write("<!DOCTYPE HTML>")
  100. f.write('<head><meta charset="UTF-8"></head>')
  101. f.write("<body>")
  102. f.write(text)
  103. f.write("</body>")
  104. def resize_image(image_path, size):
  105. try:
  106. image_np = cv2.imread(image_path)
  107. # print(image_np.shape)
  108. width = image_np.shape[1]
  109. height = image_np.shape[0]
  110. h_w_rate = height / width
  111. # width_standard = 900
  112. # height_standard = 1400
  113. width_standard = size[1]
  114. height_standard = size[0]
  115. width_new = int(height_standard / h_w_rate)
  116. height_new = int(width_standard * h_w_rate)
  117. if width > width_standard:
  118. image_np = cv2.resize(image_np, (width_standard, height_new))
  119. elif height > height_standard:
  120. image_np = cv2.resize(image_np, (width_new, height_standard))
  121. cv2.imwrite(image_path, image_np)
  122. # print("resize_image", image_np.shape)
  123. return
  124. except Exception as e:
  125. log("resize_image")
  126. print("resize_image", e, global_type)
  127. return
  128. def remove_red_seal(image_np):
  129. """
  130. 去除红色印章
  131. """
  132. # 获得红色通道
  133. blue_c, green_c, red_c = cv2.split(image_np)
  134. # 多传入一个参数cv2.THRESH_OTSU,并且把阈值thresh设为0,算法会找到最优阈值
  135. thresh, ret = cv2.threshold(red_c, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
  136. # print("remove_red_seal thresh", thresh)
  137. # 实测调整为95%效果好一些
  138. filter_condition = int(thresh * 0.98)
  139. thresh1, red_thresh = cv2.threshold(red_c, filter_condition, 255, cv2.THRESH_BINARY)
  140. # 把图片转回 3 通道
  141. image_and = np.expand_dims(red_thresh, axis=2)
  142. image_and = np.concatenate((image_and, image_and, image_and), axis=-1)
  143. # print(image_and.shape)
  144. # 膨胀
  145. gray = cv2.cvtColor(image_and, cv2.COLOR_RGB2GRAY)
  146. kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (2, 2))
  147. erode = cv2.erode(gray, kernel)
  148. cv2.imshow("erode", erode)
  149. cv2.waitKey(0)
  150. image_and = np.bitwise_and(cv2.bitwise_not(blue_c), cv2.bitwise_not(erode))
  151. result_img = cv2.bitwise_not(image_and)
  152. cv2.imshow("remove_red_seal", result_img)
  153. cv2.waitKey(0)
  154. return result_img
  155. def remove_underline(image_np):
  156. """
  157. 去除文字下划线
  158. """
  159. # 灰度化
  160. gray = cv2.cvtColor(image_np, cv2.COLOR_BGR2GRAY)
  161. # 二值化
  162. binary = cv2.adaptiveThreshold(~gray, 255,
  163. cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY,
  164. 15, 10)
  165. # Sobel
  166. kernel_row = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]], np.float32)
  167. kernel_col = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], np.float32)
  168. # binary = cv2.filter2D(binary, -1, kernel=kernel)
  169. binary_row = cv2.filter2D(binary, -1, kernel=kernel_row)
  170. binary_col = cv2.filter2D(binary, -1, kernel=kernel_col)
  171. cv2.imshow("custom_blur_demo", binary)
  172. cv2.waitKey(0)
  173. rows, cols = binary.shape
  174. # 识别横线
  175. scale = 5
  176. kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (cols // scale, 1))
  177. erodedcol = cv2.erode(binary_row, kernel, iterations=1)
  178. cv2.imshow("Eroded Image", erodedcol)
  179. cv2.waitKey(0)
  180. dilatedcol = cv2.dilate(erodedcol, kernel, iterations=1)
  181. cv2.imshow("dilate Image", dilatedcol)
  182. cv2.waitKey(0)
  183. return
  184. def getMDFFromFile(path):
  185. _length = 0
  186. try:
  187. _md5 = hashlib.md5()
  188. with open(path, "rb") as ff:
  189. while True:
  190. data = ff.read(4096)
  191. if not data:
  192. break
  193. _length += len(data)
  194. _md5.update(data)
  195. return _md5.hexdigest(), _length
  196. except Exception as e:
  197. traceback.print_exc()
  198. return None, _length
  199. def add_html_format(text_list):
  200. new_text_list = []
  201. for t in text_list:
  202. html_t = "<!DOCTYPE HTML>\n"
  203. html_t += '<head><meta charset="UTF-8"></head>\n'
  204. html_t += "<body>\n"
  205. html_t += t
  206. html_t += "\n</body>\n"
  207. new_text_list.append(html_t)
  208. return new_text_list
  209. if get_platform() == "Windows":
  210. globals().update({"time_out": 1000})
  211. else:
  212. globals().update({"time_out": 300})
  213. # @timeout_decorator.timeout(100, timeout_exception=TimeoutError)
  214. # @timeout(globals().get("time_out"), timeout_exception=TimeoutError, use_signals=False)
  215. def unique_temp_file_process(stream, _type, _md5):
  216. if get_platform() == "Windows":
  217. _global._init()
  218. globals().update({"md5": _md5})
  219. _global.update({"md5": _md5})
  220. log("into unique_temp_file_process")
  221. try:
  222. # 每个调用在temp中创建一个唯一空间
  223. uid1 = uuid.uuid1().hex
  224. unique_space_path = _path + os.sep + "temp" + os.sep + uid1 + os.sep
  225. # unique_space_path = "/mnt/fangjiasheng/" + "temp/" + uid1 + "/"
  226. # 判断冲突
  227. if not os.path.exists(unique_space_path):
  228. if not os.path.exists(_path + os.sep + "temp"):
  229. os.mkdir(_path + os.sep + "temp" + os.sep)
  230. os.mkdir(unique_space_path)
  231. else:
  232. uid2 = uuid.uuid1().hex
  233. if not os.path.exists(_path + os.sep + "temp"):
  234. os.mkdir(_path + os.sep + "temp" + os.sep)
  235. os.mkdir(_path + os.sep + "temp" + os.sep + uid2 + os.sep)
  236. # os.mkdir("/mnt/" + "temp/" + uid2 + "/")
  237. # 在唯一空间中,对传入的文件也保存为唯一
  238. uid3 = uuid.uuid1().hex
  239. file_path = unique_space_path + uid3 + "." + _type
  240. with open(file_path, "wb") as ff:
  241. ff.write(stream)
  242. text = getText(_type, file_path)
  243. # 获取swf转换的图片
  244. swf_images = []
  245. if _type == "swf":
  246. image_name_list = []
  247. for root, dirs, files in os.walk(unique_space_path, topdown=False):
  248. for name in files:
  249. if name[-4:] == ".png" and "resize" not in name:
  250. image_name_list.append(name)
  251. image_name_list.sort(key=lambda x: x)
  252. for name in image_name_list:
  253. with open(os.path.join(unique_space_path, name), "rb") as f:
  254. img_bytes = f.read()
  255. swf_images.append(base64.b64encode(img_bytes))
  256. log("unique_temp_file_process len(swf_images) " + str(len(swf_images)))
  257. return text, swf_images
  258. except TimeoutError:
  259. return [-5], []
  260. except Exception as e:
  261. log("unique_temp_file_process failed!")
  262. traceback.print_exc()
  263. return [-1], []
  264. finally:
  265. print("======================================")
  266. try:
  267. if get_platform() == "Linux":
  268. # log("not delete temp file")
  269. # 删除该唯一空间下所有文件
  270. if os.path.exists(unique_space_path):
  271. shutil.rmtree(unique_space_path)
  272. except Exception as e:
  273. log("Delete Files Failed!")
  274. def cut_str(text_list, only_text_list, max_bytes_length=2000000):
  275. log("into cut_str")
  276. try:
  277. # 计算有格式总字节数
  278. bytes_length = 0
  279. for text in text_list:
  280. bytes_length += len(bytes(text, encoding='utf-8'))
  281. # print("text_list", bytes_length)
  282. # 小于直接返回
  283. if bytes_length < max_bytes_length:
  284. print("return text_list no cut")
  285. return text_list
  286. # 全部文件连接,重新计算无格式字节数
  287. all_text = ""
  288. bytes_length = 0
  289. for text in only_text_list:
  290. bytes_length += len(bytes(text, encoding='utf-8'))
  291. all_text += text
  292. # print("only_text_list", bytes_length)
  293. # 小于直接返回
  294. if bytes_length < max_bytes_length:
  295. print("return only_text_list no cut")
  296. return only_text_list
  297. # 截取字符
  298. all_text = all_text[:int(max_bytes_length/3)]
  299. # print("text bytes ", len(bytes(all_text, encoding='utf-8')))
  300. # print("return only_text_list has cut")
  301. return [all_text]
  302. except Exception as e:
  303. log("cut_str " + str(e))
  304. return ["-1"]
  305. @memory_decorator
  306. def convert_maxcompute(data, ocr_model, otr_model):
  307. """
  308. 接口返回值:
  309. {[str], 1}: 处理成功
  310. {[-1], 0}: 逻辑处理错误
  311. {[-2], 0}: 接口调用错误
  312. {[-3], 1}: 文件格式错误,无法打开
  313. {[-4], 0}: 各类文件调用第三方包读取超时
  314. {[-5], 0}: 整个转换过程超时
  315. {[-6], 0}: 阿里云UDF队列超时
  316. {[-7], 1}: 文件需密码,无法打开
  317. :return: {"result_html": str([]), "result_text":str([]) "is_success": int}
  318. """
  319. # 控制内存
  320. # soft, hard = resource.getrlimit(resource.RLIMIT_AS)
  321. # resource.setrlimit(resource.RLIMIT_AS, (15 * 1024 ** 3, hard))
  322. log("into convert")
  323. start_time = time.time()
  324. _md5 = "1000000"
  325. try:
  326. # 模型加入全局变量
  327. globals().update({"global_ocr_model": ocr_model})
  328. globals().update({"global_otr_model": otr_model})
  329. stream = base64.b64decode(data.get("file"))
  330. _type = data.get("type")
  331. _md5 = get_md5_from_bytes(stream)
  332. if get_platform() == "Windows":
  333. # 解除超时装饰器,直接访问原函数
  334. origin_unique_temp_file_process = unique_temp_file_process.__wrapped__
  335. text, swf_images = origin_unique_temp_file_process(stream, _type, _md5)
  336. else:
  337. # Linux 通过装饰器设置整个转换超时时间
  338. try:
  339. text, swf_images = unique_temp_file_process(stream, _type, _md5)
  340. except TimeoutError:
  341. log("convert time out! 1200 sec")
  342. text = [-5]
  343. swf_images = []
  344. error_code = [[-x] for x in range(1, 9)]
  345. still_success_code = [[-3], [-7]]
  346. if text in error_code:
  347. if text in still_success_code:
  348. print({"failed result": text, "is_success": 1}, time.time() - start_time)
  349. return {"result_html": [str(text[0])], "result_text": [str(text[0])],
  350. "is_success": 1}
  351. else:
  352. print({"failed result": text, "is_success": 0}, time.time() - start_time)
  353. return {"result_html": [str(text[0])], "result_text": [str(text[0])],
  354. "is_success": 0}
  355. # 结果保存result.html
  356. if get_platform() == "Windows":
  357. text_str = ""
  358. for t in text:
  359. text_str += t
  360. to_html("../result.html", text_str)
  361. # 取纯文本
  362. only_text = []
  363. for t in text:
  364. new_t = BeautifulSoup(t, "lxml").get_text()
  365. new_t = re.sub("\n", "", new_t)
  366. only_text.append(new_t)
  367. # 判断长度,过长截取
  368. text = cut_str(text, only_text)
  369. only_text = cut_str(only_text, only_text)
  370. if len(only_text) == 0:
  371. only_text = [""]
  372. if only_text[0] == '' and len(only_text) <= 1:
  373. print({"md5: ": str(_md5), "finished result": ["", 0], "is_success": 1}, time.time() - start_time)
  374. else:
  375. print("md5: " + str(_md5), {"finished result": [str(only_text)[:20], len(str(text))],
  376. "is_success": 1}, time.time() - start_time)
  377. return {"result_html": text, "result_text": only_text, "is_success": 1}
  378. except Exception as e:
  379. print({"md5: ": str(_md5), "failed result": [-1], "is_success": 0}, time.time() - start_time)
  380. print("convert", traceback.print_exc())
  381. return {"result_html": ["-1"], "result_text": ["-1"], "is_success": 0}
  382. # 接口配置
  383. app = Flask(__name__)
  384. @app.route('/convert', methods=['POST'])
  385. def _convert():
  386. """
  387. 接口返回值:
  388. {[str], 1}: 处理成功
  389. {[-1], 0}: 逻辑处理错误
  390. {[-2], 0}: 接口调用错误
  391. {[-3], 1}: 文件格式错误,无法打开
  392. {[-4], 0}: 各类文件调用第三方包读取超时
  393. {[-5], 0}: 整个转换过程超时
  394. {[-6], 0}: 阿里云UDF队列超时
  395. {[-7], 1}: 文件需密码,无法打开
  396. :return: {"result_html": str([]), "result_text":str([]) "is_success": int}
  397. """
  398. # log("growth start" + str(objgraph.growth()))
  399. # log("most_common_types start" + str(objgraph.most_common_types(20)))
  400. # tracemalloc.start(25)
  401. # snapshot = tracemalloc.take_snapshot()
  402. _global._init()
  403. _global.update({"md5": "1"+"0"*15})
  404. set_flask_global()
  405. # _global.update({"port": str(port)})
  406. log("into convert")
  407. start_time = time.time()
  408. _md5 = _global.get("md5")
  409. _type = None
  410. try:
  411. _time = time.time()
  412. data = request.form
  413. if not data:
  414. log("convert no data!")
  415. raise ConnectionError
  416. file_path = data.get("file_path")
  417. if file_path is None:
  418. stream = base64.b64decode(data.get("file"))
  419. log("get bytes from file " + str(time.time()-_time))
  420. # 有路径则直接取路径打开文件
  421. else:
  422. with open(file_path, "rb") as f:
  423. stream = f.read()
  424. log("get bytes from file_path " + str(time.time()-_time))
  425. _type = data.get("type")
  426. _md5 = get_md5_from_bytes(stream)
  427. _md5 = _md5[0]
  428. _global.update({"md5": _md5})
  429. if get_platform() == "Windows":
  430. # 解除超时装饰器,直接访问原函数
  431. # origin_unique_temp_file_process = unique_temp_file_process.__wrapped__
  432. # text, swf_images = origin_unique_temp_file_process(stream, _type)
  433. try:
  434. text, swf_images = unique_temp_file_process(stream, _type, _md5)
  435. except TimeoutError:
  436. log("convert time out! 300 sec")
  437. text = [-5]
  438. swf_images = []
  439. else:
  440. # Linux 通过装饰器设置整个转换超时时间
  441. try:
  442. text, swf_images = unique_temp_file_process(stream, _type, _md5)
  443. except TimeoutError:
  444. log("convert time out! 300 sec")
  445. text = [-5]
  446. swf_images = []
  447. still_success_code = [-3, -4, -7]
  448. if judge_error_code(text):
  449. if judge_error_code(text, still_success_code):
  450. is_success = 1
  451. else:
  452. is_success = 0
  453. log("md5: " + str(_md5)
  454. + " finished result: " + str(text)
  455. + " is_success: " + str(is_success) + " "
  456. + str(_type) + " "
  457. + " " + str(time.time() - start_time))
  458. return json.dumps({"result_html": [str(text[0])], "result_text": [str(text[0])],
  459. "is_success": is_success, "swf_images": str(swf_images)})
  460. # 结果保存result.html
  461. # if get_platform() == "Windows":
  462. text_str = ""
  463. for t in text:
  464. text_str += t
  465. to_html(os.path.dirname(os.path.abspath(__file__)) + "/../result.html", text_str)
  466. # 取纯文本
  467. only_text = []
  468. for t in text:
  469. new_t = BeautifulSoup(t, "lxml").get_text()
  470. new_t = re.sub("\n", "", new_t)
  471. only_text.append(new_t)
  472. # 判断长度,过长截取
  473. text = cut_str(text, only_text)
  474. only_text = cut_str(only_text, only_text)
  475. if len(only_text) == 0:
  476. only_text = [""]
  477. if only_text[0] == '' and len(only_text) <= 1:
  478. print({"finished result": ["", 0], "is_success": 1}, time.time() - start_time)
  479. log("md5: " + str(_md5) + " "
  480. + " finished result: ['', 0] is_success: 1 "
  481. + str(_type) + " "
  482. + str(time.time() - start_time))
  483. else:
  484. log("md5: " + str(_md5) +
  485. " finished result: " + str(only_text)[:20] + " "
  486. + str(len(str(text))) + " is_success: 1 "
  487. + str(_type) + " "
  488. + str(time.time() - start_time))
  489. # log("growth end" + str(objgraph.growth()))
  490. # log("most_common_types end" + str(objgraph.most_common_types(20)))
  491. return json.dumps({"result_html": text, "result_text": only_text,
  492. "is_success": 1, "swf_images": str(swf_images)})
  493. except ConnectionError:
  494. log("convert post has no data!" + " failed result: [-2] is_success: 0 "
  495. + str(time.time() - start_time))
  496. return json.dumps({"result_html": ["-2"], "result_text": ["-2"],
  497. "is_success": 0, "swf_images": str([])})
  498. except Exception as e:
  499. log("md5: " + str(_md5) + " failed result: [-1] is_success: 0 "
  500. + str(_type) + " " +
  501. str(time.time() - start_time))
  502. traceback.print_exc()
  503. return json.dumps({"result_html": ["-1"], "result_text": ["-1"],
  504. "is_success": 0, "swf_images": str([])})
  505. finally:
  506. # _global._del()
  507. # gc.collect()
  508. log("finally")
  509. # snapshot1 = tracemalloc.take_snapshot()
  510. # top_stats = snapshot1.compare_to(snapshot, 'lineno')
  511. # log("[ Top 20 differences ]")
  512. # for stat in top_stats[:20]:
  513. # if stat.size_diff < 0:
  514. # continue
  515. # log(stat)
  516. # gth = objgraph.growth(limit=10)
  517. # for gt in gth:
  518. # log("growth type:%s, count:%s, growth:%s" % (gt[0], gt[1], gt[2]))
  519. # # if gt[2] > 100 or gt[1] > 300:
  520. # # continue
  521. # if gt[2] < 5:
  522. # continue
  523. # _p = os.path.dirname(os.path.abspath(__file__))
  524. # objgraph.show_backrefs(objgraph.by_type(gt[0])[0], max_depth=10, too_many=5,
  525. # filename=_p + "/dots/%s_%s_backrefs.dot" % (_md5, gt[0]))
  526. # objgraph.show_refs(objgraph.by_type(gt[0])[0], max_depth=10, too_many=5,
  527. # filename=_p + "/dots/%s_%s_refs.dot" % (_md5, gt[0]))
  528. # objgraph.show_chain(
  529. # objgraph.find_backref_chain(objgraph.by_type(gt[0])[0], objgraph.is_proper_module),
  530. # filename=_p + "/dots/%s_%s_chain.dot" % (_md5, gt[0])
  531. # )
  532. def test_more(_dir, process_no=None):
  533. file_path_list = []
  534. for root, dirs, files in os.walk(_dir, topdown=False):
  535. for name in files:
  536. file_path_list.append(os.path.join(root, name))
  537. start_time = time.time()
  538. i = 0
  539. for p in file_path_list:
  540. if i % 10 == 0:
  541. if process_no is not None:
  542. print("Process", process_no, i, time.time()-start_time)
  543. else:
  544. print("Loop", i, time.time()-start_time)
  545. test_one(p, from_remote=True)
  546. i += 1
  547. def test_one(p, from_remote=False):
  548. with open(p, "rb") as f:
  549. file_bytes = f.read()
  550. file_base64 = base64.b64encode(file_bytes)
  551. data = {"file": file_base64, "type": p.split(".")[-1], "filemd5": 100}
  552. if from_remote:
  553. ocr_model = None
  554. otr_model = None
  555. _url = 'http://127.0.0.1:15010/convert'
  556. # _url = 'http://192.168.2.102:15010/convert'
  557. # _url = 'http://172.16.160.65:15010/convert'
  558. result = json.loads(request_post(_url, data, time_out=10000))
  559. if p.split(".")[-1] == "swf":
  560. swf_images = eval(result.get("swf_images"))
  561. print(type(swf_images))
  562. # for img in swf_images:
  563. # img_bytes = base64.b64decode(img)
  564. # img = cv2.imdecode(np.frombuffer(img_bytes, np.uint8), cv2.IMREAD_COLOR)
  565. # cv2.imshow("swf_images", img)
  566. # cv2.waitKey(0)
  567. else:
  568. ocr_model = ocr_interface.OcrModels().get_model()
  569. otr_model = otr_interface.OtrModels().get_model()
  570. result = convert_maxcompute(data, ocr_model, otr_model)
  571. print("result_text", result.get("result_text")[0][:20])
  572. print("is_success", result.get("is_success"))
  573. def test_duplicate(path_list, process_no=None):
  574. start_time = time.time()
  575. for i in range(500):
  576. if i % 10 == 0:
  577. if process_no is not None:
  578. print("Process", process_no, i*len(path_list), time.time()-start_time)
  579. else:
  580. print("Loop", i*len(path_list), time.time()-start_time)
  581. for p in path_list:
  582. test_one(p, from_remote=True)
  583. global_type = ""
  584. local_url = "http://127.0.0.1"
  585. if get_platform() == "Windows":
  586. _path = os.path.abspath(os.path.dirname(__file__))
  587. else:
  588. _path = "/home/admin"
  589. if not os.path.exists(_path):
  590. _path = os.path.dirname(os.path.abspath(__file__))
  591. if __name__ == '__main__':
  592. # convert interface
  593. if len(sys.argv) == 2:
  594. port = int(sys.argv[1])
  595. else:
  596. port = 15010
  597. globals().update({"md5": "1"+"0"*15})
  598. globals().update({"port": str(port)})
  599. # _global._init()
  600. # _global.update({"md5": "1"+"0"*15})
  601. # _global.update({"port": str(port)})
  602. # ip = get_intranet_ip()
  603. # log("my ip"+str(ip))
  604. # ip = "http://" + ip
  605. ip_port_dict = get_ip_port()
  606. ip = "http://127.0.0.1"
  607. processes = ip_port_dict.get(ip).get("convert_processes")
  608. set_flask_global()
  609. if get_platform() == "Windows":
  610. app.run(host='0.0.0.0', port=port, processes=1, threaded=False, debug=False)
  611. else:
  612. # app.run(host='0.0.0.0', port=port, processes=processes, threaded=False, debug=False)
  613. app.run(port=15011)
  614. # if get_platform() == "Windows":
  615. # # file_path = "C:/Users/Administrator/Desktop/error7.jpg"
  616. # # file_path = "D:/BIDI_DOC/比地_文档/2022/Test_Interface/20210609202634853485.xlsx"
  617. # # file_path = "D:/BIDI_DOC/比地_文档/2022/Test_ODPS/1624325845476.pdf"
  618. # file_path = "C:/Users/Administrator/Downloads/1650967920520.pdf"
  619. # else:
  620. # file_path = "test1.doc"
  621. # test_one(file_path, from_remote=True)
  622. # if get_platform() == "Windows":
  623. # file_dir = "D:/BIDI_DOC/比地_文档/table_images/"
  624. # else:
  625. # file_dir = "../table_images/"
  626. #
  627. # for j in range(10):
  628. # p = Process(target=test_more, args=(file_dir, j, ))
  629. # p.start()
  630. # p.join()
  631. # if get_platform() == "Windows":
  632. # # file_path_list = ["D:/BIDI_DOC/比地_文档/2022/Test_Interface/1623328459080.doc",
  633. # # "D:/BIDI_DOC/比地_文档/2022/Test_Interface/94961e1987d1090e.xls",
  634. # # "D:/BIDI_DOC/比地_文档/2022/Test_Interface/11111111.rar"]
  635. # file_path_list = ["D:/BIDI_DOC/比地_文档/2022/Test_Interface/1623328459080.doc",
  636. # "D:/BIDI_DOC/比地_文档/2022/Test_Interface/94961e1987d1090e.xls"]
  637. # # file_path_list = ["D:/BIDI_DOC/比地_文档/2022/Test_Interface/1623328459080.doc"]
  638. #
  639. # else:
  640. # file_path_list = ["test1.pdf"]
  641. # for j in range(10):
  642. # p = Process(target=test_duplicate, args=(file_path_list, j, ))
  643. # p.start()
  644. # p.join()