|
|
@@ -115,17 +115,19 @@ class ParseDocument():
|
|
|
|
|
|
# self.soup = BeautifulSoup(self.html,"lxml")
|
|
|
# self.soup = BeautifulSoup(self.html,"html.parser")
|
|
|
+ self.auto_merge_table = auto_merge_table
|
|
|
|
|
|
self.soup = BeautifulSoup(self.html,"html5lib")
|
|
|
_body = self.soup.find("body")
|
|
|
if _body is not None:
|
|
|
self.soup = _body
|
|
|
- list_obj = self.soup.find_all(recursive=False)
|
|
|
+ self.list_obj = self.soup.find_all(recursive=False)
|
|
|
|
|
|
- for obj in list_obj:
|
|
|
- print("obj",obj.get_text()[:20])
|
|
|
+ # for obj in self.list_obj:
|
|
|
+ # print("obj",obj.get_text()[:20])
|
|
|
+
|
|
|
+ self.tree = self.buildParsetree(self.list_obj,[],auto_merge_table)
|
|
|
|
|
|
- self.tree = self.buildParsetree(list_obj,auto_merge_table)
|
|
|
|
|
|
# #识别目录树
|
|
|
# for _page in self.childs:
|
|
|
@@ -137,6 +139,11 @@ class ParseDocument():
|
|
|
# if self.parseTree:
|
|
|
# self.parseTree.printParseTree()
|
|
|
|
|
|
+ def fix_tree(self,_product):
|
|
|
+ products = extract_products(self.tree,_product)
|
|
|
+ if len(products)>0:
|
|
|
+ self.tree = self.buildParsetree(self.list_obj,products,self.auto_merge_table)
|
|
|
+
|
|
|
def print_tree(self,tree,append=""):
|
|
|
if append=="":
|
|
|
self.set_tree_id = set()
|
|
|
@@ -145,7 +152,7 @@ class ParseDocument():
|
|
|
if _id in self.set_tree_id:
|
|
|
continue
|
|
|
self.set_tree_id.add(_id)
|
|
|
- print(append,t["text"][:50],t["sentence_title"])
|
|
|
+ print(append,t["text"][:50],t["sentence_title"],t["title_before"],t["title_after"])
|
|
|
childs = t["child_title"]
|
|
|
self.print_tree(childs,append=append+" ")
|
|
|
|
|
|
@@ -154,17 +161,18 @@ class ParseDocument():
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
- def find_title_by_pattern(self,_text,_pattern="(^|★|:|：|\s+)(?P<title_1>(?P<title_1_index_0_0>第?)(?P<title_1_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_1_index_2_0>[、章册部\.:：]))|" \
|
|
|
- "([\s★\*]*)(?P<title_3>(?P<title_3_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?)(?P<title_3_index_0_1>[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_3_index_0_2>))|" \
|
|
|
- "([\s★\*]*)(?P<title_4>(?P<title_4_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?第?)(?P<title_4_index_1_1>[一二三四五六七八九十]+)(?P<title_4_index_2_0>[节章册部\.:：、、]))|" \
|
|
|
- "([\s★\*]*)(?P<title_12>(?P<title_12_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_12_index_1_1>\d{1,2})(?P<title_12_index_2_0>[\.．、\s\-]?))|"\
|
|
|
- "([\s★\*]*)(?P<title_11>(?P<title_11_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_11_index_1_1>\d{1,2})(?P<title_11_index_2_0>[\.．、\s\-]?))|" \
|
|
|
- "([\s★\*]*)(?P<title_10>(?P<title_10_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_10_index_1_1>\d{1,2})(?P<title_10_index_2_0>[\.．、\s\-]?))|" \
|
|
|
- "([\s★\*]*)(?P<title_7>(?P<title_7_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-])(?P<title_7_index_1_1>\d{1,2})(?P<title_7_index_2_0>[\.．、\s\-]?))|" \
|
|
|
- "([\s★\*]*)(?P<title_6>(?P<title_6_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?包?)(?P<title_6_index_0_1>\d{1,2})(?P<title_6_index_2_0>[\.．、\s\-]?))|" \
|
|
|
- "([\s★\*]*)(?P<title_15>(?P<title_15_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_15_index_1_1>\d{1,2})(?P<title_15_index_2_0>[)）]))|" \
|
|
|
- "([\s★\*]*)(?P<title_17>(?P<title_17_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_17_index_1_1>[a-wA-W]+)(?P<title_17_index_2_0>[)）]))|" \
|
|
|
- "([\s★\*]*)(?P<title_19>(?P<title_19_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_19_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_19_index_2_0>[)）]))" \
|
|
|
+ def find_title_by_pattern(self,_text,_pattern="(^|★|▲|:|：|\s+)(?P<title_1>(?P<title_1_index_0_0>第?)(?P<title_1_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_1_index_2_0>[、章册包部\.:：]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_3>(?P<title_3_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?)(?P<title_3_index_0_1>[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_3_index_0_2>))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_4>(?P<title_4_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?第?)(?P<title_4_index_1_1>[一二三四五六七八九十]+)(?P<title_4_index_2_0>[节章册部\.:：、、]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_5>(?P<title_5_index_0_0>^)(?P<title_5_index_1_1>[一二三四五六七八九十]+)(?P<title_5_index_2_0>)[^一二三四五六七八九十节章册部\.:：、、])|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_12>(?P<title_12_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_12_index_1_1>\d{1,2})(?P<title_12_index_2_0>[\.．、\s\-]?))|"\
|
|
|
+ "([\s★▲\*]*)(?P<title_11>(?P<title_11_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_11_index_1_1>\d{1,2})(?P<title_11_index_2_0>[\.．、\s\-]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_10>(?P<title_10_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_10_index_1_1>\d{1,2})(?P<title_10_index_2_0>[\.．、\s\-]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_7>(?P<title_7_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．\s\-])(?P<title_7_index_1_1>\d{1,2})(?P<title_7_index_2_0>[\.．包:：、\s\-]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_6>(?P<title_6_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?包?)(?P<title_6_index_0_1>\d{1,2})(?P<title_6_index_2_0>[\.．、\s\-包]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_15>(?P<title_15_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_15_index_1_1>\d{1,2})(?P<title_15_index_2_0>[)）]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_17>(?P<title_17_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_17_index_1_1>[a-wA-W]+)(?P<title_17_index_2_0>[)）]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_19>(?P<title_19_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_19_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_19_index_2_0>[)）]))" \
|
|
|
):
|
|
|
_se = re.search(_pattern,_text)
|
|
|
groups = []
|
|
|
@@ -252,7 +260,7 @@ class ParseDocument():
|
|
|
for obj_i in range(len(list_obj)):
|
|
|
obj = list_obj[obj_i]
|
|
|
_type = "sentence"
|
|
|
- _text = obj.text
|
|
|
+ _text = obj.text.strip()
|
|
|
if obj.name=="table":
|
|
|
_type = "table"
|
|
|
_text = str(obj)
|
|
|
@@ -274,8 +282,21 @@ class ParseDocument():
|
|
|
dict_before[title_before] += 1
|
|
|
return dict_before
|
|
|
|
|
|
+ def is_page_no(self,sentence):
|
|
|
+ if len(sentence)<10:
|
|
|
+ if re.search("\d+页|^\-\d+\-$",sentence) is not None:
|
|
|
+ return True
|
|
|
+
|
|
|
+ def block_tree(self,childs):
|
|
|
+ for child in childs:
|
|
|
+
|
|
|
+ if not child["block"]:
|
|
|
+ child["block"] = True
|
|
|
+ childs2 = child["child_title"]
|
|
|
+ self.block_tree(childs2)
|
|
|
|
|
|
- def buildParsetree(self,list_obj,auto_merge_table=True):
|
|
|
+
|
|
|
+ def buildParsetree(self,list_obj,products=[],auto_merge_table=True):
|
|
|
|
|
|
self.parseTree = None
|
|
|
trees = []
|
|
|
@@ -299,7 +320,7 @@ class ParseDocument():
|
|
|
for obj_i in range(len(list_obj)):
|
|
|
obj = list_obj[obj_i]
|
|
|
_type = "sentence"
|
|
|
- _text = obj.text
|
|
|
+ _text = standard_product(obj.text)
|
|
|
if obj.name=="table":
|
|
|
_type = "table"
|
|
|
_text = str(obj)
|
|
|
@@ -315,24 +336,41 @@ class ParseDocument():
|
|
|
title_next = None
|
|
|
childs = []
|
|
|
|
|
|
-
|
|
|
list_table = None
|
|
|
-
|
|
|
+ block = False
|
|
|
|
|
|
if _type=="sentence":
|
|
|
- sentence_groups = self.find_title_by_pattern(_text[:10])
|
|
|
- if sentence_groups:
|
|
|
- # c062f53cf83401e671822003d63c1828print("sentence_groups",sentence_groups)
|
|
|
- title_before = sentence_groups[1][1].replace("（","(").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".").replace("．",".")
|
|
|
- if title_before in dict_before and dict_before[title_before]>1:
|
|
|
- sentence_title = sentence_groups[0][0]
|
|
|
- sentence_title_text = sentence_groups[0][1]
|
|
|
- title_index = sentence_groups[-2][1]
|
|
|
-
|
|
|
- title_after = sentence_groups[-1][1].replace("）",")").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".").replace("．",".")
|
|
|
- next_index = self.get_next_title(title_index)
|
|
|
- else:
|
|
|
- title_before = None
|
|
|
+ if self.is_page_no(_text):
|
|
|
+ continue
|
|
|
+
|
|
|
+
|
|
|
+ _fix = False
|
|
|
+ for p in products:
|
|
|
+ if re.sub("^(\d[.、]?)+","",_text.strip())==p:
|
|
|
+ title_before = "=产品"
|
|
|
+ sentence_title = "title_0"
|
|
|
+ sentence_title_text = p
|
|
|
+ title_index = "0"
|
|
|
+ title_after = "产品="
|
|
|
+ next_index = "0"
|
|
|
+ _fix = True
|
|
|
+ break
|
|
|
+
|
|
|
+ if not _fix:
|
|
|
+ sentence_groups = self.find_title_by_pattern(_text[:10])
|
|
|
+ if sentence_groups:
|
|
|
+ # c062f53cf83401e671822003d63c1828print("sentence_groups",sentence_groups)
|
|
|
+ title_before = standard_title_context(sentence_groups[1][1])
|
|
|
+ if title_before in dict_before and dict_before[title_before]>1:
|
|
|
+ sentence_title = sentence_groups[0][0]
|
|
|
+ sentence_title_text = sentence_groups[0][1]
|
|
|
+ title_index = sentence_groups[-2][1]
|
|
|
+
|
|
|
+ title_after = sentence_groups[-1][1]
|
|
|
+ next_index = self.get_next_title(title_index)
|
|
|
+ else:
|
|
|
+ title_before = None
|
|
|
+
|
|
|
|
|
|
if _type=="sentence":
|
|
|
if sentence_title is None and len(list_data)>0 and list_data[-1]["sentence_title"] is not None and list_data[-1]["line_width"]>=max_length*0.6:
|
|
|
@@ -375,7 +413,8 @@ class ParseDocument():
|
|
|
if not _append:
|
|
|
_data = {"type":_type, "text":_text,"list_table":list_table,"line_width":len(_text),"sentence_title":sentence_title,"title_index":title_index,
|
|
|
"sentence_title_text":sentence_title_text,"sentence_groups":sentence_groups,"parent_title":parent_title,
|
|
|
- "child_title":childs,"title_before":title_before,"title_after":title_after,"title_next":title_next,"next_index":next_index}
|
|
|
+ "child_title":childs,"title_before":title_before,"title_after":title_after,"title_next":title_next,"next_index":next_index,
|
|
|
+ "block":block}
|
|
|
|
|
|
if _type=="table":
|
|
|
last_table = _data
|
|
|
@@ -399,9 +438,43 @@ class ParseDocument():
|
|
|
break
|
|
|
_d = list_data[-i]
|
|
|
if _d.get("sentence_title")==sentence_title and title_before==_d["title_before"] and title_after==_d["title_after"]:
|
|
|
- if _d["next_index"]==title_index and _d["title_next"] is None:
|
|
|
+ if _d["next_index"]==title_index and _d["title_next"] is None and not _d["block"]:
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if i==1 and not _d["block"] and _d.get("sentence_title")==sentence_title and title_before==_d["title_before"] and title_after==_d["title_after"]:
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ title_before = standard_title_context(title_before)
|
|
|
+ title_after = standard_title_context(title_after)
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if _d.get("sentence_title")==sentence_title and title_before==standard_title_context(_d["title_before"]) and title_after==standard_title_context(_d["title_after"]):
|
|
|
+ if _d["next_index"]==title_index and _d["title_next"] is None and not _d["block"]:
|
|
|
_data["parent_title"] = _d["parent_title"]
|
|
|
_d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
if _d["parent_title"] is not None:
|
|
|
_d["parent_title"]["child_title"].append(_data)
|
|
|
_find = True
|
|
|
@@ -410,15 +483,33 @@ class ParseDocument():
|
|
|
if _find:
|
|
|
break
|
|
|
_d = list_data[-i]
|
|
|
- if i==1 and _d.get("sentence_title")==sentence_title and title_before==_d["title_before"] and title_after==_d["title_after"]:
|
|
|
+ if not _d["block"] and _d.get("sentence_title")==sentence_title and title_before==standard_title_context(_d["title_before"]) and title_after==standard_title_context(_d["title_after"]):
|
|
|
_data["parent_title"] = _d["parent_title"]
|
|
|
_d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ for i in range(1,min(len(list_data)+1,20)):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if not _d["block"] and _d.get("sentence_title")==sentence_title and title_before==standard_title_context(_d["title_before"]):
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
if _d["parent_title"] is not None:
|
|
|
_d["parent_title"]["child_title"].append(_data)
|
|
|
_find = True
|
|
|
break
|
|
|
|
|
|
if not _find:
|
|
|
+ print("not found",_text)
|
|
|
if len(list_data)>0:
|
|
|
for i in range(1,len(list_data)+1):
|
|
|
_d = list_data[-i]
|
|
|
@@ -441,8 +532,14 @@ class ParseDocument():
|
|
|
|
|
|
return list_data
|
|
|
|
|
|
+def standard_title_context(_title_context):
|
|
|
+ return _title_context.replace("（","(").replace("）",")").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".").replace("．",".")
|
|
|
+
|
|
|
+def standard_product(sentence):
|
|
|
+ return sentence.replace("（","(").replace("）",")")
|
|
|
|
|
|
def extract_products(list_data,_product,_param_pattern = "产品名称|采购内存|标的名称|采购内容|(标的|维修|系统|报价构成|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品?|采购|物装|配件|资产|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|品目|^品名|气体|标项|分项|项目|计划|包组|标段|[分子]?包|子目|服务|招标|中标|成交|工程|招标内容)[\)）的]?([、\w]{,4}名称|内容|描述)|标的|标项|项目$|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品|物装|配件|资产|招标内容|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|菜名|^品目$|^品名$|^名称|^内容$"):
|
|
|
+ _product = standard_product(_product)
|
|
|
list_result = []
|
|
|
for _data_i in range(len(list_data)):
|
|
|
_data = list_data[_data_i]
|
|
|
@@ -467,10 +564,10 @@ def extract_products(list_data,_product,_param_pattern = "产品名称|采购内
|
|
|
for cell_i in range(len(line)):
|
|
|
cell = line[cell_i]
|
|
|
cell_text = cell[0]
|
|
|
- if len(cell_text)<10 and re.search(_param_pattern,cell_text) is not None and re.search("单价|数量|总价|规格|品牌|型号|用途|要求|采购量",line_text) is not None:
|
|
|
+ if len(cell_text)<10 and re.search(_param_pattern,cell_text) is not None and re.search("单价|数量|预算|限价|总价|品牌|规格|型号|用途|要求|采购量",line_text) is not None:
|
|
|
_begin_index = line_i+1
|
|
|
-
|
|
|
list_head_index.append(cell_i)
|
|
|
+
|
|
|
for line_i in range(len(list_table)):
|
|
|
line = list_table[line_i]
|
|
|
for cell_i in list_head_index:
|
|
|
@@ -495,17 +592,26 @@ def extract_products(list_data,_product,_param_pattern = "产品名称|采购内
|
|
|
if len(list_head_index)>0:
|
|
|
for line_i in range(_begin_index,len(list_table)):
|
|
|
line = list_table[line_i]
|
|
|
+ has_number = False
|
|
|
+ for cell_i in range(len(line)):
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if re.search("^\d+$",cell_text) is not None:
|
|
|
+ has_number = True
|
|
|
+
|
|
|
for cell_i in list_head_index:
|
|
|
if cell_i>=len(line):
|
|
|
continue
|
|
|
cell = line[cell_i]
|
|
|
- _text = cell[0]
|
|
|
- if re.search(_param_pattern,_text) is None:
|
|
|
- table_products.append(_text)
|
|
|
+ cell_text = cell[0]
|
|
|
+ if re.search(_param_pattern,cell_text) is None or has_number:
|
|
|
+ table_products.append(cell_text)
|
|
|
+
|
|
|
if len(table_products)>0:
|
|
|
- if min([len(x) for x in table_products])>0:
|
|
|
+ # print("table_products",table_products,_text)
|
|
|
+ if min([len(x) for x in table_products])>0 and max([len(x) for x in table_products])<=20:
|
|
|
list_result.extend(table_products)
|
|
|
- list_result = [a for a in list_result if len(a)>1 and len(a)<20]
|
|
|
+ list_result = list(set([a for a in list_result if len(a)>1 and len(a)<20 and re.search("预算|合计|金额|万元|运费",a) is None]))
|
|
|
return list_result
|
|
|
|
|
|
|
|
|
@@ -544,49 +650,62 @@ def get_correct_product(product,products):
|
|
|
for _d in list_data:
|
|
|
is_sim = _d["is_sim"]
|
|
|
if is_sim:
|
|
|
+ if len(_d["product"])>len(product) and _d["product"].find(product)>=0:
|
|
|
+ return _product
|
|
|
return _d["product"]
|
|
|
return product
|
|
|
|
|
|
-def get_childs_text(childs,_product,products,is_end=False):
|
|
|
+def get_childs_text(childs,_product,products,is_begin=False,is_end=False):
|
|
|
_text = ""
|
|
|
|
|
|
for _child in childs:
|
|
|
+
|
|
|
child_text = _child.get("text")
|
|
|
|
|
|
+ if child_text.find(_product)>=0:
|
|
|
+ is_begin = True
|
|
|
+
|
|
|
+ print("get_childs_text",child_text[:10],is_begin)
|
|
|
+
|
|
|
for p in products:
|
|
|
|
|
|
if child_text.find(_product)<0 and child_text.find(p)>=0:
|
|
|
- is_end = True
|
|
|
+ if is_begin:
|
|
|
+ is_end = True
|
|
|
break
|
|
|
- if is_end:
|
|
|
- break
|
|
|
+ if re.search(end_pattern,child_text) is not None:
|
|
|
+ if is_begin:
|
|
|
+ is_end = True
|
|
|
|
|
|
+ if is_begin and is_end:
|
|
|
+ break
|
|
|
|
|
|
- _text += _child.get("text")+"\n"
|
|
|
+ if is_begin:
|
|
|
+ _text += _child.get("text")+"\r\n"
|
|
|
childs2 = _child.get("child_title",[])
|
|
|
|
|
|
-
|
|
|
if len(childs2)>0:
|
|
|
for _child2 in childs2:
|
|
|
- child_text,is_end = get_childs_text([_child2],_product,products)
|
|
|
- if is_end:
|
|
|
+ child_text,is_begin,is_end = get_childs_text([_child2],_product,products,is_begin)
|
|
|
+ if is_begin and is_end:
|
|
|
break
|
|
|
else:
|
|
|
- _text += child_text
|
|
|
- return _text,is_end
|
|
|
+ if is_begin:
|
|
|
+ _text += child_text
|
|
|
+ return _text,is_begin,is_end
|
|
|
|
|
|
def extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result,):
|
|
|
_data = list_data[_data_i]
|
|
|
childs = _data.get("child_title",[])
|
|
|
if len(childs)>0:
|
|
|
- child_text,_ = get_childs_text([_data],_product,products)
|
|
|
- list_result.append(child_text)
|
|
|
- return True
|
|
|
+ child_text,_,_ = get_childs_text([parent_title],_product,products)
|
|
|
+ if len(child_text)>0:
|
|
|
+ list_result.append(child_text)
|
|
|
+ return True
|
|
|
if parent_title is not None:
|
|
|
childs = parent_title.get("child_title",[])
|
|
|
if len(childs)>0:
|
|
|
|
|
|
-
|
|
|
print(parent_title["text"])
|
|
|
for c in childs:
|
|
|
print("11",c["text"])
|
|
|
@@ -602,22 +721,24 @@ def extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,
|
|
|
for p in products:
|
|
|
if ptext.find(_product)<0 and ptext.find(p)>=0:
|
|
|
_find = True
|
|
|
- print("p find",p)
|
|
|
break
|
|
|
+ if re.search(end_pattern,ptext) is not None:
|
|
|
+ _find = True
|
|
|
if _find:
|
|
|
- print("======break")
|
|
|
- print(ptext)
|
|
|
break
|
|
|
- p_text += ptext+"\n"
|
|
|
+ p_text += ptext+"\r\n"
|
|
|
if len(p_text)>0:
|
|
|
list_result.append(p_text)
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
-def extract_parameters_by_table(_product,_param_pattern,list_data,_data_i,list_result):
|
|
|
+end_pattern = "商务要求|评分标准|商务条件|商务条件"
|
|
|
+
|
|
|
+def extract_parameters_by_table(_product,products,_param_pattern,list_data,_data_i,list_result):
|
|
|
_data = list_data[_data_i]
|
|
|
_text = _data["text"]
|
|
|
list_table = _data["list_table"]
|
|
|
+ parent_title = _data["parent_title"]
|
|
|
if list_table is not None:
|
|
|
_check = True
|
|
|
max_length = max([len(a) for a in list_table])
|
|
|
@@ -650,51 +771,102 @@ def extract_parameters_by_table(_product,_param_pattern,list_data,_data_i,list_r
|
|
|
list_result.append(cell_text)
|
|
|
if len(cell_text)<len(_product)*10 and str(cell_text).find(_product)>=0:
|
|
|
for _index in list_head_index:
|
|
|
+ if _index>=len(line):
|
|
|
+ continue
|
|
|
_cell = line[_index]
|
|
|
if len(cell[0])>0:
|
|
|
print("add on table",_cell[0])
|
|
|
list_result.append(_cell[0])
|
|
|
+ if not _flag and (re.search(_param_pattern,_text) is not None or (parent_title is not None and re.search(_param_pattern,parent_title["text"]) is not None)) and _text.find(_product)>=0:
|
|
|
+ _soup = BeautifulSoup(_text,"html5lib")
|
|
|
+ _table = _soup.find("table")
|
|
|
+ if _table is not None:
|
|
|
+ trs = getTrs(_table)
|
|
|
+ list_trs = []
|
|
|
+ _find = False
|
|
|
+ for tr in trs:
|
|
|
+ tr_text = tr.get_text()
|
|
|
+ if tr_text.find(_product)>=0:
|
|
|
+ _find = True
|
|
|
+
|
|
|
+ for p in products:
|
|
|
+ if _find and p!=_product and tr_text.find(p)>=0:
|
|
|
+ _find = False
|
|
|
+ break
|
|
|
+ if re.search(end_pattern,tr_text) is not None:
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ if _find:
|
|
|
+ list_trs.append(tr)
|
|
|
+ if len(list_trs)>0:
|
|
|
+ _html = "<table>%s</table>"%("\r\n".join([str(a) for a in list_trs]))
|
|
|
+ print("_html",_html)
|
|
|
+ list_result.append(_html)
|
|
|
+
|
|
|
|
|
|
def extract_product_parameters(list_data,_product):
|
|
|
- _param_pattern = "产品配置|配置要求|技术要求|技术参数|参数指标|具体参数|规格参数|参数要求|技术需求|配置清单|(质量|技术).{,10}要求|明细及参数|验收标准|^参数$"
|
|
|
+
|
|
|
+ _param_pattern = "(产品|技术|清单[及和]?|配置|参数|具体|明细[及和]?|项目|货物)(指标|配置|要求|参数|需求|规格)|配置清单|(质量|技术).{,10}要求|验收标准|^参数$"
|
|
|
list_result = []
|
|
|
- _product = _product.strip()
|
|
|
+ _product = standard_product(_product.strip())
|
|
|
products = extract_products(list_data,_product)
|
|
|
|
|
|
_product = get_correct_product(_product,products)
|
|
|
print("===",_product,products)
|
|
|
+ is_project = False
|
|
|
for _data_i in range(len(list_data)):
|
|
|
_data = list_data[_data_i]
|
|
|
_type = _data["type"]
|
|
|
_text = _data["text"]
|
|
|
if _type=="sentence":
|
|
|
if _text.find(_product)>=0:
|
|
|
- print("_text",_text,_data["sentence_title"])
|
|
|
+ if re.search("项目名称|采购项目",_text) is not None:
|
|
|
+ is_project = True
|
|
|
+ print("_text find",_text,_data["sentence_title"])
|
|
|
parent_title = _data.get("parent_title")
|
|
|
- if re.search(_param_pattern,_text) is not None:
|
|
|
- extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result)
|
|
|
-
|
|
|
- elif parent_title is not None:
|
|
|
+ parent_text = ""
|
|
|
+ parent_parent_title = None
|
|
|
+ parent_parent_text = ""
|
|
|
+ parent_title_index = None
|
|
|
+ parent_parent_title_index = None
|
|
|
+ if parent_title is not None:
|
|
|
parent_text = parent_title.get("text","")
|
|
|
- print("parent_text",parent_text)
|
|
|
- if re.search(_param_pattern,parent_text) is not None:
|
|
|
- extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result)
|
|
|
+ parent_parent_title = parent_title.get("parent_title")
|
|
|
+ parent_title_index = parent_title["title_index"]
|
|
|
+ if parent_parent_title is not None:
|
|
|
+ parent_parent_text = parent_parent_title.get("text","")
|
|
|
+ parent_parent_title_index = parent_parent_title["title_index"]
|
|
|
+
|
|
|
+ _suit = False
|
|
|
+ if re.search(_param_pattern,_text) is not None and (len(_text)<15 or _data["title_index"] is not None):
|
|
|
+ _suit = True
|
|
|
+ if re.search(_param_pattern,parent_text) is not None and (len(parent_text)<15 or parent_title_index is not None):
|
|
|
+ _suit = True
|
|
|
+ if re.search(_param_pattern,parent_parent_text) is not None and (len(parent_parent_text)<15 or parent_parent_title_index is not None):
|
|
|
+ _suit = True
|
|
|
+ if _suit:
|
|
|
+ if not extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result):
|
|
|
+ extract_parameters_by_tree(_product,products,list_data,_data_i,parent_parent_title,list_result)
|
|
|
+
|
|
|
+
|
|
|
+ if re.search(_param_pattern,_text) is not None and (len(_text)<15 or _data["title_index"] is not None):
|
|
|
+ print("re.search(_param_pattern,_text)",_text,is_project)
|
|
|
+ childs = _data["child_title"]
|
|
|
+ if len(childs)>0 and (len(products)>0 or is_project):
|
|
|
+ print("re.search(_param_pattern,_text)",_text,is_project)
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
+ print("re.search(_param_pattern,_text)",extract_text)
|
|
|
+ if len(extract_text)>0:
|
|
|
+ list_result.append(extract_text)
|
|
|
|
|
|
- else:
|
|
|
- parent_title = parent_title.get("parent_title")
|
|
|
- if parent_title is not None:
|
|
|
- parent_text = parent_title.get("text","")
|
|
|
- # print("parent_text",parent_text)
|
|
|
- if re.search(_param_pattern,parent_text) is not None:
|
|
|
- extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result)
|
|
|
|
|
|
elif _type=="table":
|
|
|
- extract_parameters_by_table(_product,_param_pattern,list_data,_data_i,list_result)
|
|
|
+ extract_parameters_by_table(_product,products,_param_pattern,list_data,_data_i,list_result)
|
|
|
|
|
|
|
|
|
# for i in range(len(list_result)):
|
|
|
# print("result%d"%i,list_result[i])
|
|
|
- list_result.sort(key=lambda x:len(re.findall('[^.][0-9a-zA-Z]+[^.]',x)), reverse=True)
|
|
|
+ list_result.sort(key=lambda x:len(re.findall('[^.、][0-9a-zA-Z::：；]+[^.、]?',BeautifulSoup(x,"html5lib").get_text())), reverse=True)
|
|
|
|
|
|
print("+++++++++++++++++++++")
|
|
|
for i in range(len(list_result)):
|
|
|
@@ -703,27 +875,34 @@ def extract_product_parameters(list_data,_product):
|
|
|
|
|
|
for _result in list_result:
|
|
|
_check = True
|
|
|
+ _result_text = BeautifulSoup(_result,"html5lib").get_text()
|
|
|
+ if re.search("项目编号|项目名称|联合体投标",_result) is not None:
|
|
|
+ _check = False
|
|
|
for p in products:
|
|
|
- if _result.find(p)>0 and not is_similar(_product,p,80):
|
|
|
+ if _result_text.find(p)>0 and not (is_similar(_product,p,80) or p.find(_product)>=0 or _product.find(p)>=0):
|
|
|
_check = False
|
|
|
if len(_result)<10:
|
|
|
_check = False
|
|
|
+ if len(_result_text)>5000:
|
|
|
+ _check = False
|
|
|
if _check:
|
|
|
return _result
|
|
|
-
|
|
|
return None
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|
- _html = open("download/107015f8e994683fd88827ad209f0d13.html", "r", encoding="utf8").read()
|
|
|
+ filepath = "download/d5ceaa54bbdc6a28c7b8796ca71930a9.html"
|
|
|
+ _product = "肺功能测试系统"
|
|
|
+
|
|
|
+ _html = open(filepath, "r", encoding="utf8").read()
|
|
|
|
|
|
pd = ParseDocument(_html,True)
|
|
|
|
|
|
+ pd.fix_tree(_product)
|
|
|
list_data = pd.tree
|
|
|
pd.print_tree(list_data)
|
|
|
|
|
|
-
|
|
|
- _text = extract_product_parameters(list_data,"CT")
|
|
|
+ _text = extract_product_parameters(list_data,_product)
|
|
|
print("extract_text",_text)
|
|
|
|
luojiehua