', h[:first_table_start])) <= 0 and \
len(re.findall('\d', col_text_list[0])) == len(col_text_list[0]):
connect_flag2 = True
# if not connect_flag2 and len(re.findall('
', h[:first_table_start])) <= 0 and len(re.findall('[\u4e00-\u9fff]', h[:first_table_start])) <= 25:
# connect_flag2 = True
connect_flag_list.append([i, connect_flag2, connect_flag1])
print('connect_flag_list', connect_flag_list)
# 根据条件1合并需连接页码，形成组
connect_pages_list = []
if connect_flag_list:
temp_list = [connect_flag_list[0]]
for i in range(1, len(connect_flag_list)):
c = connect_flag_list[i]
if c[1] and temp_list[-1][2]:
temp_list.append(c)
else:
if temp_list:
connect_pages_list.append(temp_list)
temp_list = [c]
# connect_pages_list.append([c])
if temp_list:
connect_pages_list.append(temp_list)
print('connect_pages_list', connect_pages_list)
# 判断后续条件：判断组内列数是否相同
connect_pages_list2 = []
for c_list in connect_pages_list:
if len(c_list) == 1:
connect_pages_list2.append(c_list)
else:
col_cnt_list = []
# 单元格可能被复制了，相同的合并当做一列
merge_col_cnt_list = []
for c in c_list:
soup = soup_list[c[0]]
table1 = soup.findAll('table')[-1]
table2 = soup.findAll('table')[0]
tr1 = table1.findAll('tr')
tr2 = table2.findAll('tr')
td1 = tr1[-1].findAll('td')
td2 = tr2[0].findAll('td')
col_cnt_list.append([len(td2), len(td1)])
# # 计算合并重复文本格子后的列数
# last_text = td1[0].text
# merge_td1 = [last_text]
# for td in td1:
# if td.text == last_text:
# continue
# else:
# merge_td1.append(td.text)
# last_text = td.text
# last_text = td2[0].text
# merge_td2 = [last_text]
# for td in td2:
# if td.text == last_text:
# continue
# else:
# merge_td2.append(td.text)
# last_text = td.text
# merge_col_cnt_list.append([len(merge_td2), len(merge_td1)])
# 判断
new_c_list = [c_list[0]]
# print('col_cnt_list', col_cnt_list)
for i in range(len(col_cnt_list) - 1):
if col_cnt_list[i][1] != col_cnt_list[i + 1][0]:
# and merge_col_cnt_list[i][1] != merge_col_cnt_list[i + 1][0]:
connect_pages_list2.append(new_c_list)
new_c_list = [c_list[i + 1]]
else:
new_c_list.append(c_list[i + 1])
if new_c_list:
connect_pages_list2.append(new_c_list)
print('connect_pages_list2', connect_pages_list2)
# 符合连接条件的拼接表格
new_html_list = []
for c_list in connect_pages_list2:
if len(c_list) == 1:
new_html_list.append(html_list[c_list[0][0]])
continue
new_html = ''
for c in c_list:
match = re.finditer('', new_html)
last_table_index = None
for m in match:
last_table_index = m.span()[0]
new_html += html_list[c[0]]
# print('html_list[c[0]]', html_list[c[0]])
if last_table_index is None:
continue
match = re.finditer('
', new_html[last_table_index:])
first_table_index = None
for m in match:
first_table_index = last_table_index + m.span()[1]
break
if first_table_index is None:
continue
# print('re', re.findall('
.*?
', new_html[last_table_index:first_table_index]))
# 非贪婪匹配
new_html_sub = re.sub('
.*?
',
'| #@#@# |
',
new_html[last_table_index:first_table_index])
new_html = new_html[:last_table_index] + new_html_sub + new_html[first_table_index:]
# print('new_html', new_html)
# new_html = new_html[:-5]
# ([-/第页0-9]||
)*
# 非贪婪匹配
# match = re.finditer('
.*?
', new_html)
# for m in match:
# if '#@#@#' in m.group():
#
# new_html = re.sub('
.*#@#@#.*?
',
# '| #@#@# |
',
# new_html)
# print('new_html', new_html)
soup = BeautifulSoup(new_html, 'lxml')
trs = soup.findAll('tr')
decompose_trs = []
for i in range(len(trs)):
if trs[i].get_text() == '#@#@#':
td1 = trs[i - 1].findAll('td')
td2 = trs[i + 1].findAll('td')
if td2[0].get_text() == '':
# 解决连续多页是一行表格，该行会被去掉问题
find_father = False
for father, son in decompose_trs:
# print('son', son)
# print('td1', trs[i - 1])
if father != '' and son == trs[i - 1]:
td_father = father.findAll('td')
for j in range(len(td_father)):
# print('td_father[j].string3', td_father[j].string)
td_father[j].string = td_father[j].get_text() + td2[j].get_text()
# print('td_father[j].string4', td_father[j].string)
find_father = True
decompose_trs.append([father, trs[i + 1]])
break
if not find_father:
for j in range(len(td1)):
# print('td1[j].string1', td1[j].string)
td1[j].string = td1[j].get_text() + td2[j].get_text()
# print('td1[j].string2', td1[j].string)
decompose_trs.append([trs[i - 1], trs[i + 1]])
# print('trs[i + 1]', trs[i + 1])
# trs[i + 1].decompose()
# print('trs[i-1]', trs[i-1])
# trs[i].decompose()
decompose_trs.append(['', trs[i]])
# print('decompose_trs', decompose_trs)
# for father, son in decompose_trs:
# print('father', father)
# print('son', son)
# print('len(decompose_trs)', len(decompose_trs))
for father, son in decompose_trs:
for tr in trs:
if tr == son:
tr.decompose()
break
new_html = str(soup)
new_html_list.append(new_html)
html_str = ''
for h in new_html_list:
html_str += h
return [html_str]
def get_html(self):
if self._doc.error_code is not None:
return self._doc.error_code
self.convert()
if self._doc.error_code is not None:
return self._doc.error_code
html = self._doc.get_html(return_list=True)
# 表格连接
try:
html = self.connect_table(html)
except:
traceback.print_exc()
return [-12]
return html
def delete_water_mark(self, lt_text_list, page_bbox, times=5):
# 删除过多重复字句，为水印
duplicate_dict = {}
for _obj in lt_text_list:
t = _obj.get_text()
if t in duplicate_dict.keys():
duplicate_dict[t][0] += 1
duplicate_dict[t][1].append(_obj)
else:
duplicate_dict[t] = [1, [_obj]]
delete_text = []
for t in duplicate_dict.keys():
if duplicate_dict[t][0] >= times:
obj_list = duplicate_dict[t][1]
obj_list.sort(key=lambda x: x.bbox[3])
obj_distance_h = abs(obj_list[-1].bbox[3] - obj_list[0].bbox[1])
obj_list.sort(key=lambda x: x.bbox[2])
obj_distance_w = abs(obj_list[-1].bbox[2] - obj_list[0].bbox[0])
if obj_distance_h >= abs(page_bbox[1] - page_bbox[3]) * 0.7 \
and obj_distance_w >= abs(page_bbox[0] - page_bbox[2]) * 0.7:
delete_text.append(t)
temp_text_list = []
for _obj in lt_text_list:
t = _obj.get_text()
if t not in delete_text:
temp_text_list.append(_obj)
return temp_text_list
def resize_image(self, img_path, max_size=2000):
_img = cv2.imread(img_path)
if _img.shape[0] <= max_size or _img.shape[1] <= max_size:
return
else:
resize_axis = 0 if _img.shape[0] >= _img.shape[1] else 1
ratio = max_size / _img.shape[resize_axis]
new_shape = [0, 0]
new_shape[resize_axis] = max_size
new_shape[1 - resize_axis] = int(_img.shape[1 - resize_axis] * ratio)
_img = cv2.resize(_img, (new_shape[1], new_shape[0]))
cv2.imwrite(img_path, _img)
def get_single_pdf(self, path, page_no):
log("into get_single_pdf")
try:
pdf_origin = copy.deepcopy(self.doc_pypdf2)
pdf_new = copy.deepcopy(self.doc_pypdf2_new)
pdf_new.addPage(pdf_origin.getPage(page_no))
path_new = path.split(".")[0] + "_split.pdf"
with open(path_new, "wb") as ff:
pdf_new.write(ff)
return path_new
except PyPDF2.utils.PdfReadError as e:
return [-3]
except Exception as e:
log("get_single_pdf error! page " + str(page_no))
return [-3]
def get_text_font():
def flags_decomposer(flags):
"""Make font flags human readable."""
l = []
if flags & 2 ** 0:
l.append("superscript")
if flags & 2 ** 1:
l.append("italic")
if flags & 2 ** 2:
l.append("serifed")
else:
l.append("sans")
if flags & 2 ** 3:
l.append("monospaced")
else:
l.append("proportional")
if flags & 2 ** 4:
l.append("bold")
return ", ".join(l)
def get_underlined_textLines(page):
"""
获取某页pdf上的所有下划线文本信息
:param page: fitz中的一页
:return: list of tuples,每个tuple都是一个完整的下划线覆盖的整体：[(下划线句, 所在blk_no, 所在line_no), ...]
"""
paths = page.get_drawings() # get drawings on the current page
# 获取该页内所有的height很小的bbox。因为下划线其实大多是这种矩形
# subselect things we may regard as lines
lines = []
for p in paths:
for item in p["items"]:
if item[0] == "l": # an actual line
p1, p2 = item[1:]
if p1.y == p2.y:
lines.append((p1, p2))
elif item[0] == "re": # a rectangle: check if height is small
r = item[1]
if r.width > r.height and r.height <= 2:
lines.append((r.tl, r.tr)) # take top left / right points
# 获取该页的`max_lineheight`，用于下面比较距离使用
blocks = page.get_text("dict", flags=11)["blocks"]
max_lineheight = 0
for b in blocks:
for l in b["lines"]:
bbox = fitz.Rect(l["bbox"])
if bbox.height > max_lineheight:
max_lineheight = bbox.height
underlined_res = []
# 开始对下划线内容进行查询
# make a list of words
words = page.get_text("words")
# if underlined, the bottom left / right of a word
# should not be too far away from left / right end of some line:
for wdx, w in enumerate(words): # w[4] is the actual word string
r = fitz.Rect(w[:4]) # first 4 items are the word bbox
for p1, p2 in lines: # check distances for start / end points
if abs(r.bl - p1) <= max_lineheight: # 当前word的左下满足下划线左下
if abs(r.br - p2) <= max_lineheight: # 当前word的右下满足下划线右下（单个词，无空格）
print(f"Word '{w[4]}' is underlined! Its block-line number is {w[-3], w[-2]}")
underlined_res.append((w[4], w[-3], w[-2])) # 分别是(下划线词，所在blk_no，所在line_no)
break # don't check more lines
else: # 继续寻找同line右侧的有缘人，因为有些下划线覆盖的词包含多个词，多个词之间有空格
curr_line_num = w[-2] # line nunmber
for right_wdx in range(wdx + 1, len(words), 1):
_next_w = words[right_wdx]
if _next_w[-2] != curr_line_num: # 当前遍历到的右侧word已经不是当前行的了（跨行是不行的）
break
_r_right = fitz.Rect(_next_w[:4]) # 获取当前同行右侧某word的方框4点
if abs(_r_right.br - p2) <= max_lineheight: # 用此word右下点和p2(目标下划线右上点)算距离，距离要小于max_lineheight
print(
f"Word '{' '.join([_one_word[4] for _one_word in words[wdx:right_wdx + 1]])}' is underlined! " +
f"Its block-line number is {w[-3], w[-2]}")
underlined_res.append(
(' '.join([_one_word[4] for _one_word in words[wdx:right_wdx + 1]]),
w[-3], w[-2])
) # 分别是(下划线词，所在blk_no，所在line_no)
break # don't check more lines
return underlined_res
_p = r'C:\Users\Administrator\Desktop\test_pdf\error2-2.pdf'
doc_pymupdf = read_pymupdf(_p)
page = doc_pymupdf[0]
blocks = page.get_text("dict", flags=11)["blocks"]
for b in blocks: # iterate through the text blocks
for l in b["lines"]: # iterate through the text lines
for s in l["spans"]: # iterate through the text spans
print("")
font_properties = "Font: '%s' (%s), size %g, color #%06x" % (
s["font"], # font name
flags_decomposer(s["flags"]), # readable font flags
s["size"], # font size
s["color"], # font color
)
print(s)
print("Text: '%s'" % s["text"]) # simple print of text
print(font_properties)
get_underlined_textLines(page)
# 以下为现成pdf单页解析接口
class ParseSentence:
def __init__(self, bbox, fontname, fontsize, _text, _title, title_text, _pattern, title_degree, is_outline,
outline_location, page_no):
(x0, y0, x1, y1) = bbox
self.x0 = x0
self.y0 = y0
self.x1 = x1
self.y1 = y1
self.bbox = bbox
self.fontname = fontname
self.fontsize = fontsize
self.text = _text
self.title = _title
self.title_text = title_text
self.groups = _pattern
self.title_degree = title_degree
self.is_outline = is_outline
self.outline_location = outline_location
self.page_no = page_no
def __repr__(self):
return "%s,%s,%s,%d,%s" % (self.text, self.title, self.is_outline, self.outline_location, str(self.bbox))
class ParseUtils:
@staticmethod
def getFontinfo(_page):
for _obj in _page._objs:
if isinstance(_obj, (LTTextBoxHorizontal, LTTextBoxVertical)):
for textline in _obj._objs:
done = False
for lchar in textline._objs:
if isinstance(lchar, (LTChar)):
_obj.fontname = lchar.fontname
_obj.fontsize = lchar.size
done = True
break
if done:
break
@staticmethod
def recognize_sentences(list_textbox, filter_objs, page_bbox, page_no,
remove_space=True, sourceP_LB=True):
list_textbox.sort(key=lambda x: x.bbox[0])
list_textbox.sort(key=lambda x: x.bbox[3], reverse=sourceP_LB)
cluster_textbox = []
for _textbox in list_textbox:
if _textbox in filter_objs:
continue
_find = False
for _ct in cluster_textbox:
if abs(_ct["y"] - _textbox.bbox[1]) < 5:
_find = True
_ct["textbox"].append(_textbox)
if not _find:
cluster_textbox.append({"y": _textbox.bbox[1], "textbox": [_textbox]})
cluster_textbox.sort(key=lambda x: x["y"], reverse=sourceP_LB)
list_sentences = []
for _line in cluster_textbox:
_textboxs = _line["textbox"]
_textboxs.sort(key=lambda x: x.bbox[0])
_linetext = _textboxs[0].get_text()
for _i in range(1, len(_textboxs)):
if abs(_textboxs[_i].bbox[0] - _textboxs[_i - 1].bbox[2]) > 60:
if _linetext and _linetext[-1] not in (",", "，", "。", ".", "、", "；"):
_linetext += "=，="
_linetext += _textboxs[_i].get_text()
_linetext = re.sub("[\s\r\n]", "", _linetext)
_bbox = (_textboxs[0].bbox[0], _textboxs[0].bbox[1],
_textboxs[-1].bbox[2], _textboxs[-1].bbox[3])
_title = None
_pattern_groups = None
title_text = ""
if not _title:
_groups = ParseUtils.find_title_by_pattern(_textboxs[0].get_text())
if _groups:
_title = _groups[0][0]
title_text = _groups[0][1]
_pattern_groups = _groups
if not _title:
_groups = ParseUtils.find_title_by_pattern(_linetext)
if _groups:
_title = _groups[0][0]
title_text = _groups[0][1]
_pattern_groups = _groups
if not _title:
_title = ParseUtils.rec_incenter(_bbox, page_bbox)
title_degree = 2
if not _title:
_linetext = _linetext.replace("=，=", "，")
else:
_linetext = _linetext.replace("=，=", "")
title_degree = int(_title.split("_")[1])
# 页码
if ParseUtils.rec_incenter(_bbox, page_bbox) and re.search("^\d+$", _linetext) is not None:
continue
if _linetext == "" or re.search("^，+$", _linetext) is not None:
continue
is_outline = False
outline_location = -1
_search = re.search("(?P.+?)\.{5,}(?P\d+)$", _linetext)
if _search is not None:
is_outline = True
_linetext = _search.group("text")
outline_location = int(_search.group("nums"))
list_sentences.append(
ParseSentence(_bbox, _textboxs[-1].__dict__.get("fontname"), _textboxs[-1].__dict__.get("fontsize"),
_linetext, _title, title_text, _pattern_groups, title_degree, is_outline,
outline_location, page_no))
# for _sen in list_sentences:
# print(_sen.__dict__)
return list_sentences
@staticmethod
def find_title_by_pattern(_text,
_pattern="(?P(?P^第?)(?P[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P[、章]))|" \
"(?P^(?P[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+))|" \
"(?P^(?P第?)(?P[一二三四五六七八九十]+)(?P[节]))|" \
"(?P^(?P\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P\d{1,2})(?P[\.．、\s\-]))|" \
"(?P^(?P\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P\d{1,2})(?P[\.．、\s\-]))|" \
"(?P^(?P\d{1,2}[\.．、\s\-])(?P\d{1,2})(?P[\.．、\s\-]))|" \
"(?P^(?P\d{1,2})(?P[\.．、\s\-]))|" \
"(?P^(?P（?)(?P\d{1,2})(?P）))|" \
"(?P^(?P（?)(?P[a-zA-Z]+)(?P）))|"
"(?P^(?P（?)(?P[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P）))|" \
):
_se = re.search(_pattern, _text)
groups = []
if _se is not None:
_gd = _se.groupdict()
for k, v in _gd.items():
if v is not None:
groups.append((k, v))
if len(groups):
groups.sort(key=lambda x: x[0])
return groups
return None
@staticmethod
def rec_incenter(o_bbox, p_bbox):
p_width = p_bbox[2] - p_bbox[0]
l_space = (o_bbox[0] - p_bbox[0]) / p_width
r_space = (p_bbox[2] - o_bbox[2]) / p_width
if abs((l_space - r_space)) < 0.1 and l_space > 0.2:
return "title_2"
@staticmethod
def is_first_title(_title):
if _title is None:
return False
if re.search("^\d+$", _title) is not None:
if int(_title) == 1:
return True
return False
if re.search("^[一二三四五六七八九十百]+$", _title) is not None:
if _title == "一":
return True
return False
if re.search("^[a-z]+$", _title) is not None:
if _title == "a":
return True
return False
if re.search("^[A-Z]+$", _title) is not None:
if _title == "A":
return True
return False
if re.search("^[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]$", _title) is not None:
if _title == "Ⅰ":
return True
return False
return False
@staticmethod
def get_next_title(_title):
if re.search("^\d+$", _title) is not None:
return str(int(_title) + 1)
if re.search("^[一二三四五六七八九十百]+$", _title) is not None:
_next_title = ParseUtils.make_increase(['一', '二', '三', '四', '五', '六', '七', '八', '九', '十'],
re.sub("[十百]", '', _title))
_next_title = list(_next_title)
_next_title.reverse()
if _next_title[-1] != "十":
if len(_next_title) >= 2:
_next_title.insert(-1, '十')
if len(_next_title) >= 4:
_next_title.insert(-3, '百')
if _title[0] == "十":
if _next_title == "十":
_next_title = ["二", "十"]
_next_title.insert(0, "十")
_next_title = "".join(_next_title)
return _next_title
if re.search("^[a-z]+$", _title) is not None:
_next_title = ParseUtils.make_increase([chr(i + ord('a')) for i in range(26)], _title)
_next_title = list(_next_title)
_next_title.reverse()
return "".join(_next_title)
if re.search("^[A-Z]+$", _title) is not None:
_next_title = ParseUtils.make_increase([chr(i + ord('A')) for i in range(26)], _title)
_next_title = list(_next_title)
_next_title.reverse()
return "".join(_next_title)
if re.search("^[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]$", _title) is not None:
_sort = ["Ⅰ", "Ⅱ", "Ⅲ", "Ⅳ", "Ⅴ", "Ⅵ", "Ⅶ", "Ⅷ", "Ⅸ", "Ⅹ", "Ⅺ", "Ⅻ"]
_index = _sort.index(_title)
if _index < len(_sort) - 1:
return _sort[_index + 1]
return None
@staticmethod
def make_increase(_sort, _title, _add=1):
if len(_title) == 0 and _add == 0:
return ""
if len(_title) == 0 and _add == 1:
return _sort[0]
_index = _sort.index(_title[-1])
next_index = (_index + _add) % len(_sort)
next_chr = _sort[next_index]
if _index == len(_sort) - 1:
_add = 1
else:
_add = 0
return next_chr + ParseUtils.make_increase(_sort, _title[:-1], _add)
@staticmethod
def rec_serial(_text, o_bbox, p_bbox, fontname, _pattern="(?P^[一二三四五六七八九十]+[、])|" \
"(?P^\d+[\.、\s])|" \
"(?P^\d+\.\d+[\.、\s])|" \
"(?P^\d+\.\d+\.\d+[\.、\s])|" \
"(?P^\d+\.\d+\.\d+\.\d+[\.、\s])"):
# todo :recog the serial of the sentence
_se = re.search(_pattern, _text)
if _se is not None:
_gd = _se.groupdict()
for k, v in _gd.items():
if v is not None:
return k
return None
if __name__ == '__main__':
# get_text_font()
PDFConvert(r"C:/Users/Administrator/Downloads/1651896704621.pdf", "C:/Users/Administrator/Downloads/1").get_html()
# print(b'\x10')