|
|
@@ -0,0 +1,969 @@
|
|
|
+#coding:utf8
|
|
|
+
|
|
|
+import re
|
|
|
+
|
|
|
+from BaseDataMaintenance.maintenance.product.productUtils import *
|
|
|
+import logging
|
|
|
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
|
+
|
|
|
+logger = logging.getLogger(__name__)
|
|
|
+logger.setLevel(logging.DEBUG)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+from bs4 import BeautifulSoup
|
|
|
+import copy
|
|
|
+
|
|
|
+end_pattern = "商务要求|评分标准|商务条件|商务条件"
|
|
|
+_param_pattern = "(产品|技术|清单[及和]?|配置|参数|具体|明细[及和]?|项目|货物|服务)(指标|配置|要求|参数|需求|规格)|配置清单|(质量|技术).{,10}要求|验收标准|^参数$"
|
|
|
+meter_pattern = "角度|容积|色彩|帧率|磁场|强度|允差|噪音|材质|频率|阻抗|浓度|范围|误差|精确|温度|可调|设定值|功能|检测|高度|宽度|模式|尺寸|重量|峰值|容量|寿命|稳定性|高温|电源|电压|功率|压力|压强"
|
|
|
+
|
|
|
+def getTrs(tbody):
|
|
|
+ #获取所有的tr
|
|
|
+ trs = []
|
|
|
+ if tbody.name=="table":
|
|
|
+ body = tbody.find("tbody",recursive=False)
|
|
|
+ if body is not None:
|
|
|
+ tbody = body
|
|
|
+ objs = tbody.find_all(recursive=False)
|
|
|
+ for obj in objs:
|
|
|
+ if obj.name=="tr":
|
|
|
+ trs.append(obj)
|
|
|
+ if obj.name=="tbody" or obj.name=="table":
|
|
|
+ for tr in obj.find_all("tr",recursive=False):
|
|
|
+ trs.append(tr)
|
|
|
+ return trs
|
|
|
+
|
|
|
+def fixSpan(tbody):
|
|
|
+ # 处理colspan, rowspan信息补全问题
|
|
|
+ #trs = tbody.findChildren('tr', recursive=False)
|
|
|
+
|
|
|
+ trs = getTrs(tbody)
|
|
|
+ ths_len = 0
|
|
|
+ ths = list()
|
|
|
+ trs_set = set()
|
|
|
+ #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱
|
|
|
+ # 遍历每一个tr
|
|
|
+
|
|
|
+ for indtr, tr in enumerate(trs):
|
|
|
+ ths_tmp = tr.findChildren('th', recursive=False)
|
|
|
+ #不补全含有表格的tr
|
|
|
+ if len(tr.findChildren('table'))>0:
|
|
|
+ continue
|
|
|
+ if len(ths_tmp) > 0:
|
|
|
+ ths_len = ths_len + len(ths_tmp)
|
|
|
+ for th in ths_tmp:
|
|
|
+ ths.append(th)
|
|
|
+ trs_set.add(tr)
|
|
|
+ # 遍历每行中的element
|
|
|
+ tds = tr.findChildren(recursive=False)
|
|
|
+ for indtd, td in enumerate(tds):
|
|
|
+ # 若有colspan 则补全同一行下一个位置
|
|
|
+ if 'colspan' in td.attrs:
|
|
|
+ if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
|
|
|
+ col = int(re.sub("[^0-9]","",str(td['colspan'])))
|
|
|
+ if col<100 and len(td.get_text())<1000:
|
|
|
+ td['colspan'] = 1
|
|
|
+ for i in range(1, col, 1):
|
|
|
+ td.insert_after(copy.copy(td))
|
|
|
+
|
|
|
+ for indtr, tr in enumerate(trs):
|
|
|
+ ths_tmp = tr.findChildren('th', recursive=False)
|
|
|
+ #不补全含有表格的tr
|
|
|
+ if len(tr.findChildren('table'))>0:
|
|
|
+ continue
|
|
|
+ if len(ths_tmp) > 0:
|
|
|
+ ths_len = ths_len + len(ths_tmp)
|
|
|
+ for th in ths_tmp:
|
|
|
+ ths.append(th)
|
|
|
+ trs_set.add(tr)
|
|
|
+ # 遍历每行中的element
|
|
|
+ tds = tr.findChildren(recursive=False)
|
|
|
+ for indtd, td in enumerate(tds):
|
|
|
+ # 若有rowspan 则补全下一行同样位置
|
|
|
+ if 'rowspan' in td.attrs:
|
|
|
+ if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
|
|
|
+ row = int(re.sub("[^0-9]","",str(td['rowspan'])))
|
|
|
+ td['rowspan'] = 1
|
|
|
+ for i in range(1, row, 1):
|
|
|
+ # 获取下一行的所有td， 在对应的位置插入
|
|
|
+ if indtr+i<len(trs):
|
|
|
+ tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
|
|
|
+ if len(tds1) >= (indtd) and len(tds1)>0:
|
|
|
+ if indtd > 0:
|
|
|
+ tds1[indtd - 1].insert_after(copy.copy(td))
|
|
|
+ else:
|
|
|
+ tds1[0].insert_before(copy.copy(td))
|
|
|
+ elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全
|
|
|
+ tds1[indtd-2].insert_after(copy.copy(td))
|
|
|
+def getTable(tbody):
|
|
|
+ #trs = tbody.findChildren('tr', recursive=False)
|
|
|
+ fixSpan(tbody)
|
|
|
+ trs = getTrs(tbody)
|
|
|
+ inner_table = []
|
|
|
+ for tr in trs:
|
|
|
+ tr_line = []
|
|
|
+ tds = tr.findChildren(['td','th'], recursive=False)
|
|
|
+ if len(tds)==0:
|
|
|
+ tr_line.append([re.sub('\xa0','',tr.get_text()),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
|
|
|
+ for td in tds:
|
|
|
+ tr_line.append([re.sub('\xa0','',td.get_text()),0])
|
|
|
+ #tr_line.append([td.get_text(),0])
|
|
|
+ inner_table.append(tr_line)
|
|
|
+ return inner_table
|
|
|
+
|
|
|
+class ParseDocument():
|
|
|
+
|
|
|
+ def __init__(self,_html,auto_merge_table=True):
|
|
|
+ if _html is None:
|
|
|
+ _html = ""
|
|
|
+ self.html = _html
|
|
|
+
|
|
|
+ # self.soup = BeautifulSoup(self.html,"lxml")
|
|
|
+ # self.soup = BeautifulSoup(self.html,"html.parser")
|
|
|
+ self.auto_merge_table = auto_merge_table
|
|
|
+
|
|
|
+ self.soup = BeautifulSoup(self.html,"html5lib")
|
|
|
+ _body = self.soup.find("body")
|
|
|
+ if _body is not None:
|
|
|
+ self.soup = _body
|
|
|
+ self.list_obj = self.soup.find_all(recursive=False)
|
|
|
+
|
|
|
+ # for obj in self.list_obj:
|
|
|
+ # print("obj",obj.get_text()[:20])
|
|
|
+
|
|
|
+ self.tree = self.buildParsetree(self.list_obj,[],auto_merge_table)
|
|
|
+
|
|
|
+
|
|
|
+ # #识别目录树
|
|
|
+ # if self.parseTree:
|
|
|
+ # self.parseTree.printParseTree()
|
|
|
+
|
|
|
+ def fix_tree(self,_product):
|
|
|
+ products = extract_products(self.tree,_product)
|
|
|
+ if len(products)>0:
|
|
|
+ self.tree = self.buildParsetree(self.list_obj,products,self.auto_merge_table)
|
|
|
+
|
|
|
+ def print_tree(self,tree,append=""):
|
|
|
+ if append=="":
|
|
|
+ self.set_tree_id = set()
|
|
|
+ for t in tree:
|
|
|
+ _id = id(t)
|
|
|
+ if _id in self.set_tree_id:
|
|
|
+ continue
|
|
|
+ self.set_tree_id.add(_id)
|
|
|
+ logger.debug("%s %s %s %s %s"%(append,t["text"][:50],t["sentence_title"],t["title_before"],t["title_after"]))
|
|
|
+ childs = t["child_title"]
|
|
|
+ self.print_tree(childs,append=append+"-|")
|
|
|
+
|
|
|
+ def is_title_first(self,title):
|
|
|
+ if title in ("一","1","Ⅰ","a","A"):
|
|
|
+ return True
|
|
|
+ return False
|
|
|
+
|
|
|
+ def find_title_by_pattern(self,_text,_pattern="(^|★|▲|:|：|\s+)(?P<title_1>(?P<title_1_index_0_0>第?)(?P<title_1_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_1_index_2_0>[、章册包标部\.:：]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_3>(?P<title_3_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?)(?P<title_3_index_0_1>[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_3_index_0_2>))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_4>(?P<title_4_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?第?)(?P<title_4_index_1_1>[一二三四五六七八九十]+)(?P<title_4_index_2_0>[节章册部\.:：、、]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_5>(?P<title_5_index_0_0>^)(?P<title_5_index_1_1>[一二三四五六七八九十]+)(?P<title_5_index_2_0>)[^一二三四五六七八九十节章册部\.:：、、])|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_12>(?P<title_12_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_12_index_1_1>\d{1,2})(?P<title_12_index_2_0>[\.．、\s\-]?))|"\
|
|
|
+ "([\s★▲\*]*)(?P<title_11>(?P<title_11_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_11_index_1_1>\d{1,2})(?P<title_11_index_2_0>[\.．、\s\-]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_10>(?P<title_10_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_10_index_1_1>\d{1,2})(?P<title_10_index_2_0>[\.．、\s\-]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_7>(?P<title_7_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．\s\-])(?P<title_7_index_1_1>\d{1,2})(?P<title_7_index_2_0>[\.．包标:：、\s\-]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_6>(?P<title_6_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?包?)(?P<title_6_index_0_1>\d{1,2})(?P<title_6_index_2_0>[\.．、\s\-包标]?))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_15>(?P<title_15_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_15_index_1_1>\d{1,2})(?P<title_15_index_2_0>[)）包标]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_17>(?P<title_17_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_17_index_1_1>[a-wA-W]+)(?P<title_17_index_2_0>[)）包标]))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_19>(?P<title_19_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_19_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_19_index_2_0>[)）]))" \
|
|
|
+ ):
|
|
|
+ _se = re.search(_pattern,_text)
|
|
|
+ groups = []
|
|
|
+ if _se is not None:
|
|
|
+ _gd = _se.groupdict()
|
|
|
+ for k,v in _gd.items():
|
|
|
+ if v is not None:
|
|
|
+ groups.append((k,v))
|
|
|
+ if len(groups):
|
|
|
+ # groups.sort(key=lambda x:x[0])
|
|
|
+ return groups
|
|
|
+ return None
|
|
|
+
|
|
|
+ def make_increase(self,_sort,_title,_add=1):
|
|
|
+ if len(_title)==0 and _add==0:
|
|
|
+ return ""
|
|
|
+ if len(_title)==0 and _add==1:
|
|
|
+ return _sort[0]
|
|
|
+ _index = _sort.index(_title[-1])
|
|
|
+ next_index = (_index+_add)%len(_sort)
|
|
|
+ next_chr = _sort[next_index]
|
|
|
+ if _index==len(_sort)-1:
|
|
|
+ _add = 1
|
|
|
+ else:
|
|
|
+ _add = 0
|
|
|
+ return next_chr+self.make_increase(_sort,_title[:-1],_add)
|
|
|
+
|
|
|
+
|
|
|
+ def get_next_title(self,_title):
|
|
|
+ if re.search("^\d+$",_title) is not None:
|
|
|
+ return str(int(_title)+1)
|
|
|
+ if re.search("^[一二三四五六七八九十百]+$",_title) is not None:
|
|
|
+ if _title[-1]=="十":
|
|
|
+ return _title+"一"
|
|
|
+ if _title[-1]=="百":
|
|
|
+ return _title+"零一"
|
|
|
+
|
|
|
+ if _title[-1]=="九":
|
|
|
+ if len(_title)==1:
|
|
|
+ return "十"
|
|
|
+ if len(_title)==2:
|
|
|
+ if _title[0]=="十":
|
|
|
+ return "二十"
|
|
|
+ if len(_title)==3:
|
|
|
+ if _title[0]=="九":
|
|
|
+ return "一百"
|
|
|
+ else:
|
|
|
+ _next_title = self.make_increase(['一','二','三','四','五','六','七','八','九','十'],re.sub("[十百]",'',_title[0]))
|
|
|
+ return _next_title+"十"
|
|
|
+
|
|
|
+ _next_title = self.make_increase(['一','二','三','四','五','六','七','八','九','十'],re.sub("[十百]",'',_title))
|
|
|
+ _next_title = list(_next_title)
|
|
|
+ _next_title.reverse()
|
|
|
+ if _next_title[-1]!="十":
|
|
|
+ if len(_next_title)>=2:
|
|
|
+ _next_title.insert(-1,'十')
|
|
|
+ if len(_next_title)>=4:
|
|
|
+ _next_title.insert(-3,'百')
|
|
|
+ if _title[0]=="十":
|
|
|
+ if _next_title=="十":
|
|
|
+ _next_title = ["二","十"]
|
|
|
+ _next_title.insert(0,"十")
|
|
|
+ _next_title = "".join(_next_title)
|
|
|
+ return _next_title
|
|
|
+ if re.search("^[a-z]+$",_title) is not None:
|
|
|
+ _next_title = self.make_increase([chr(i+ord('a')) for i in range(26)],_title)
|
|
|
+ _next_title = list(_next_title)
|
|
|
+ _next_title.reverse()
|
|
|
+ return "".join(_next_title)
|
|
|
+ if re.search("^[A-Z]+$",_title) is not None:
|
|
|
+ _next_title = self.make_increase([chr(i+ord('A')) for i in range(26)],_title)
|
|
|
+ _next_title = list(_next_title)
|
|
|
+ _next_title.reverse()
|
|
|
+ return "".join(_next_title)
|
|
|
+ if re.search("^[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]$",_title) is not None:
|
|
|
+ _sort = ["Ⅰ","Ⅱ","Ⅲ","Ⅳ","Ⅴ","Ⅵ","Ⅶ","Ⅷ","Ⅸ","Ⅹ","Ⅺ","Ⅻ"]
|
|
|
+ _index = _sort.index(_title)
|
|
|
+ if _index<len(_sort)-1:
|
|
|
+ return _sort[_index+1]
|
|
|
+ return None
|
|
|
+
|
|
|
+ def count_title_before(self,list_obj):
|
|
|
+ dict_before = {}
|
|
|
+ dict_sentence_count = {}
|
|
|
+ illegal_sentence = set()
|
|
|
+ for obj_i in range(len(list_obj)):
|
|
|
+ obj = list_obj[obj_i]
|
|
|
+ _type = "sentence"
|
|
|
+ _text = obj.text.strip()
|
|
|
+ if obj.name=="table":
|
|
|
+ _type = "table"
|
|
|
+ _text = str(obj)
|
|
|
+ _append = False
|
|
|
+
|
|
|
+
|
|
|
+ if _type=="sentence":
|
|
|
+ if len(_text)>10 and len(_text)<100:
|
|
|
+ if _text not in dict_sentence_count:
|
|
|
+ dict_sentence_count[_text] = 0
|
|
|
+ dict_sentence_count[_text] += 1
|
|
|
+ if re.search("\d+页",_text) is not None:
|
|
|
+ illegal_sentence.add(_text)
|
|
|
+ elif len(_text)<10:
|
|
|
+ if re.search("第\d+页",_text) is not None:
|
|
|
+ illegal_sentence.add(_text)
|
|
|
+
|
|
|
+ sentence_groups = self.find_title_by_pattern(_text[:10])
|
|
|
+ if sentence_groups:
|
|
|
+ # c062f53cf83401e671822003d63c1828print("sentence_groups",sentence_groups)
|
|
|
+ sentence_title = sentence_groups[0][0]
|
|
|
+ sentence_title_text = sentence_groups[0][1]
|
|
|
+ title_index = sentence_groups[-2][1]
|
|
|
+ title_before = sentence_groups[1][1].replace("（","(").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".")
|
|
|
+ title_after = sentence_groups[-1][1].replace("）",")").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".")
|
|
|
+ next_index = self.get_next_title(title_index)
|
|
|
+ if title_before not in dict_before:
|
|
|
+ dict_before[title_before] = 0
|
|
|
+ dict_before[title_before] += 1
|
|
|
+
|
|
|
+ for k,v in dict_sentence_count.items():
|
|
|
+ if v>10:
|
|
|
+ illegal_sentence.add(k)
|
|
|
+ return dict_before,illegal_sentence
|
|
|
+
|
|
|
+ def is_page_no(self,sentence):
|
|
|
+ if len(sentence)<10:
|
|
|
+ if re.search("\d+页|^\-\d+\-$",sentence) is not None:
|
|
|
+ return True
|
|
|
+
|
|
|
+ def block_tree(self,childs):
|
|
|
+ for child in childs:
|
|
|
+
|
|
|
+ if not child["block"]:
|
|
|
+ child["block"] = True
|
|
|
+ childs2 = child["child_title"]
|
|
|
+ self.block_tree(childs2)
|
|
|
+
|
|
|
+
|
|
|
+ def buildParsetree(self,list_obj,products=[],auto_merge_table=True):
|
|
|
+
|
|
|
+ self.parseTree = None
|
|
|
+ trees = []
|
|
|
+ list_length = []
|
|
|
+ for obj in list_obj[:200]:
|
|
|
+ if obj.name!="table":
|
|
|
+ list_length.append(len(obj.get_text()))
|
|
|
+ if len(list_length)>0:
|
|
|
+ max_length = max(list_length)
|
|
|
+ else:
|
|
|
+ max_length = 40
|
|
|
+
|
|
|
+ logger.debug("%s:%d"%("max_length",max_length))
|
|
|
+
|
|
|
+
|
|
|
+ list_data = []
|
|
|
+ last_table_index = None
|
|
|
+ last_table_columns = None
|
|
|
+ last_table = None
|
|
|
+ dict_before,illegal_sentence = self.count_title_before(list_obj)
|
|
|
+ for obj_i in range(len(list_obj)):
|
|
|
+ obj = list_obj[obj_i]
|
|
|
+ _type = "sentence"
|
|
|
+ _text = standard_product(obj.text)
|
|
|
+ if obj.name=="table":
|
|
|
+ _type = "table"
|
|
|
+ _text = standard_product(str(obj))
|
|
|
+ _append = False
|
|
|
+ sentence_title = None
|
|
|
+ sentence_title_text = None
|
|
|
+ sentence_groups = None
|
|
|
+ title_index = None
|
|
|
+ next_index = None
|
|
|
+ parent_title = None
|
|
|
+ title_before = None
|
|
|
+ title_after = None
|
|
|
+ title_next = None
|
|
|
+ childs = []
|
|
|
+
|
|
|
+ list_table = None
|
|
|
+ block = False
|
|
|
+
|
|
|
+ if _type=="sentence":
|
|
|
+ if _text in illegal_sentence:
|
|
|
+ continue
|
|
|
+
|
|
|
+ _fix = False
|
|
|
+ for p in products:
|
|
|
+ if re.sub("^(\d[.、]?)+","",_text.strip())==p:
|
|
|
+ title_before = "=产品"
|
|
|
+ sentence_title = "title_0"
|
|
|
+ sentence_title_text = p
|
|
|
+ title_index = "0"
|
|
|
+ title_after = "产品="
|
|
|
+ next_index = "0"
|
|
|
+ _fix = True
|
|
|
+ break
|
|
|
+
|
|
|
+ if not _fix:
|
|
|
+ sentence_groups = self.find_title_by_pattern(_text[:10])
|
|
|
+ if sentence_groups:
|
|
|
+ title_before = standard_title_context(sentence_groups[1][1])
|
|
|
+ if title_before in dict_before and dict_before[title_before]>1:
|
|
|
+ sentence_title = sentence_groups[0][0]
|
|
|
+ sentence_title_text = sentence_groups[0][1]
|
|
|
+ title_index = sentence_groups[-2][1]
|
|
|
+
|
|
|
+ title_after = sentence_groups[-1][1]
|
|
|
+ next_index = self.get_next_title(title_index)
|
|
|
+ else:
|
|
|
+ title_before = None
|
|
|
+
|
|
|
+
|
|
|
+ if _type=="sentence":
|
|
|
+ if sentence_title is None and len(list_data)>0 and list_data[-1]["sentence_title"] is not None and list_data[-1]["line_width"]>=max_length*0.6:
|
|
|
+ list_data[-1]["text"] += _text
|
|
|
+ list_data[-1]["line_width"] = len(_text)
|
|
|
+ _append = True
|
|
|
+ elif sentence_title is None and len(list_data)>0 and _type==list_data[-1]["type"]:
|
|
|
+ if list_data[-1]["line_width"]>=max_length*0.7:
|
|
|
+ list_data[-1]["text"] += _text
|
|
|
+ list_data[-1]["line_width"] = len(_text)
|
|
|
+ _append = True
|
|
|
+
|
|
|
+ if _type=="table":
|
|
|
+ _soup = BeautifulSoup(_text,"lxml")
|
|
|
+ _table = _soup.find("table")
|
|
|
+ if _table is not None:
|
|
|
+ list_table = getTable(_table)
|
|
|
+ table_columns = len(list_table[0])
|
|
|
+
|
|
|
+ if auto_merge_table:
|
|
|
+ if last_table_index is not None and abs(obj_i-last_table_index)<=2 and last_table_columns is not None and last_table_columns==table_columns:
|
|
|
+ if last_table is not None:
|
|
|
+ trs = getTrs(_table)
|
|
|
+ last_tbody = BeautifulSoup(last_table["text"],"lxml")
|
|
|
+ _table = last_tbody.find("table")
|
|
|
+ last_trs = getTrs(_table)
|
|
|
+ _append = True
|
|
|
+
|
|
|
+ for _line in list_table:
|
|
|
+ last_table["list_table"].append(_line)
|
|
|
+ if len(last_trs)>0:
|
|
|
+ for _tr in trs:
|
|
|
+ last_trs[-1].insert_after(copy.copy(_tr))
|
|
|
+ last_table["text"] = re.sub("</?html>|</?body>","",str(last_tbody))
|
|
|
+
|
|
|
+ last_table_index = obj_i
|
|
|
+ last_table_columns = len(list_table[-1])
|
|
|
+
|
|
|
+
|
|
|
+ if not _append:
|
|
|
+ _data = {"type":_type, "text":_text,"list_table":list_table,"line_width":len(_text),"sentence_title":sentence_title,"title_index":title_index,
|
|
|
+ "sentence_title_text":sentence_title_text,"sentence_groups":sentence_groups,"parent_title":parent_title,
|
|
|
+ "child_title":childs,"title_before":title_before,"title_after":title_after,"title_next":title_next,"next_index":next_index,
|
|
|
+ "block":block}
|
|
|
+
|
|
|
+ if _type=="table":
|
|
|
+ last_table = _data
|
|
|
+ last_table_index = obj_i
|
|
|
+ if list_table:
|
|
|
+ last_table_columns = last_table_columns = len(list_table[-1])
|
|
|
+
|
|
|
+ if sentence_title is not None:
|
|
|
+ if len(list_data)>0:
|
|
|
+ if self.is_title_first(title_index):
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ _d = list_data[-i]
|
|
|
+ if _d["sentence_title"] is not None:
|
|
|
+ _data["parent_title"] = _d
|
|
|
+ _d["child_title"].append(_data)
|
|
|
+ break
|
|
|
+ else:
|
|
|
+ _find = False
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if _d.get("sentence_title")==sentence_title and title_before==_d["title_before"] and title_after==_d["title_after"]:
|
|
|
+ if _d["next_index"]==title_index and _d["title_next"] is None and not _d["block"]:
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if i==1 and not _d["block"] and _d.get("sentence_title")==sentence_title and title_before==_d["title_before"] and title_after==_d["title_after"]:
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ title_before = standard_title_context(title_before)
|
|
|
+ title_after = standard_title_context(title_after)
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if _d.get("sentence_title")==sentence_title and title_before==standard_title_context(_d["title_before"]) and title_after==standard_title_context(_d["title_after"]):
|
|
|
+ if _d["next_index"]==title_index and _d["title_next"] is None and not _d["block"]:
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if not _d["block"] and _d.get("sentence_title")==sentence_title and title_before==standard_title_context(_d["title_before"]) and title_after==standard_title_context(_d["title_after"]):
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ # self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ for i in range(1,min(len(list_data)+1,20)):
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _d = list_data[-i]
|
|
|
+ if not _d["block"] and _d.get("sentence_title")==sentence_title and title_before==standard_title_context(_d["title_before"]):
|
|
|
+ _data["parent_title"] = _d["parent_title"]
|
|
|
+ _d["title_next"] = _data
|
|
|
+ if len(_d["child_title"])>0:
|
|
|
+ _d["child_title"][-1]["title_next"] = ""
|
|
|
+ # self.block_tree(_d["child_title"])
|
|
|
+ if _d["parent_title"] is not None:
|
|
|
+ _d["parent_title"]["child_title"].append(_data)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+
|
|
|
+ if not _find:
|
|
|
+ if len(list_data)>0:
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ _d = list_data[-i]
|
|
|
+ if _d.get("sentence_title") is not None:
|
|
|
+ _data["parent_title"] = _d
|
|
|
+ _d["child_title"].append(_data)
|
|
|
+ break
|
|
|
+
|
|
|
+
|
|
|
+ else:
|
|
|
+ if len(list_data)>0:
|
|
|
+ for i in range(1,len(list_data)+1):
|
|
|
+ _d = list_data[-i]
|
|
|
+ if _d.get("sentence_title") is not None:
|
|
|
+ _data["parent_title"] = _d
|
|
|
+ _d["child_title"].append(_data)
|
|
|
+ break
|
|
|
+
|
|
|
+ list_data.append(_data)
|
|
|
+
|
|
|
+ return list_data
|
|
|
+
|
|
|
+def standard_title_context(_title_context):
|
|
|
+ return _title_context.replace("（","(").replace("）",")").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".").replace("．",".")
|
|
|
+
|
|
|
+def standard_product(sentence):
|
|
|
+ return sentence.replace("（","(").replace("）",")")
|
|
|
+
|
|
|
+def extract_products(list_data,_product,_param_pattern = "产品名称|采购内存|标的名称|采购内容|(标的|维修|系统|报价构成|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品?|采购|物装|配件|资产|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|品目|^品名|气体|标项|分项|项目|计划|包组|标段|[分子]?包|子目|服务|招标|中标|成交|工程|招标内容)[\)）的]?([、\w]{,4}名称|内容|描述)|标的|标项|项目$|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品|物装|配件|资产|招标内容|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|菜名|^品目$|^品名$|^名称|^内容$"):
|
|
|
+ _product = standard_product(_product)
|
|
|
+ list_result = []
|
|
|
+ for _data_i in range(len(list_data)):
|
|
|
+ _data = list_data[_data_i]
|
|
|
+ _type = _data["type"]
|
|
|
+ _text = _data["text"]
|
|
|
+ table_products = []
|
|
|
+ if _type=="table":
|
|
|
+ list_table = _data["list_table"]
|
|
|
+ if list_table is None:
|
|
|
+ continue
|
|
|
+ _check = True
|
|
|
+ max_length = max([len(a) for a in list_table])
|
|
|
+ min_length = min([len(a) for a in list_table])
|
|
|
+ if min_length<max_length/2:
|
|
|
+ continue
|
|
|
+ list_head_index = []
|
|
|
+ _begin_index = 0
|
|
|
+ head_cell_text = ""
|
|
|
+ for line_i in range(len(list_table[:2])):
|
|
|
+ line = list_table[line_i]
|
|
|
+ line_text = ",".join([cell[0] for cell in line])
|
|
|
+ for cell_i in range(len(line)):
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if len(cell_text)<10 and re.search(_param_pattern,cell_text) is not None and re.search("单价|数量|预算|限价|总价|品牌|规格|型号|用途|要求|采购量",line_text) is not None:
|
|
|
+ _begin_index = line_i+1
|
|
|
+ list_head_index.append(cell_i)
|
|
|
+
|
|
|
+ for line_i in range(len(list_table)):
|
|
|
+ line = list_table[line_i]
|
|
|
+ for cell_i in list_head_index:
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ head_cell_text += cell_text
|
|
|
+
|
|
|
+ # print("===head_cell_text",head_cell_text)
|
|
|
+ if re.search("招标人|采购人|项目编号|项目名称|金额|^\d+$",head_cell_text) is not None:
|
|
|
+ list_head_index = []
|
|
|
+
|
|
|
+
|
|
|
+ for line in list_table:
|
|
|
+ line_text = ",".join([cell[0] for cell in line])
|
|
|
+ for cell_i in range(len(line)):
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if cell_text is not None and _product is not None and len(cell_text)<len(_product)*10 and re.search(_product,cell_text) is not None and re.search("单价|数量|总价|规格|品牌|型号|用途|要求|采购量",line_text) is not None:
|
|
|
+ list_head_index.append(cell_i)
|
|
|
+
|
|
|
+ list_head_index = list(set(list_head_index))
|
|
|
+ if len(list_head_index)>0:
|
|
|
+ for line_i in range(_begin_index,len(list_table)):
|
|
|
+ line = list_table[line_i]
|
|
|
+ has_number = False
|
|
|
+ for cell_i in range(len(line)):
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if re.search("^\d+$",cell_text) is not None:
|
|
|
+ has_number = True
|
|
|
+
|
|
|
+ for cell_i in list_head_index:
|
|
|
+ if cell_i>=len(line):
|
|
|
+ continue
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if re.search(_param_pattern,cell_text) is None or has_number:
|
|
|
+ if re.search("^[\da-zA-Z]+$",cell_text) is None:
|
|
|
+ table_products.append(cell_text)
|
|
|
+
|
|
|
+ if len(table_products)>0:
|
|
|
+ if min([len(x) for x in table_products])>0 and max([len(x) for x in table_products])<=20:
|
|
|
+ list_result.extend(table_products)
|
|
|
+ list_result = list(set([a for a in list_result if len(a)>1 and len(a)<20 and re.search("预算|合计|金额|万元|运费",a) is None]))
|
|
|
+ return list_result
|
|
|
+
|
|
|
+
|
|
|
+def get_childs(childs):
|
|
|
+ list_data = []
|
|
|
+ for _child in childs:
|
|
|
+ list_data.append(_child)
|
|
|
+ childs2 = _child.get("child_title",[])
|
|
|
+
|
|
|
+ if len(childs2)>0:
|
|
|
+ for _child2 in childs2:
|
|
|
+ list_data.extend(get_childs([_child2]))
|
|
|
+ return list_data
|
|
|
+
|
|
|
+def get_range_data_by_childs(list_data,childs):
|
|
|
+ range_data = []
|
|
|
+ list_child = get_childs(childs)
|
|
|
+ list_index = []
|
|
|
+ set_child = set([id(x) for x in list_child])
|
|
|
+ for _data_i in range(len(list_data)):
|
|
|
+ _data = list_data[_data_i]
|
|
|
+ _id = id(_data)
|
|
|
+ if _id in set_child:
|
|
|
+ list_index.append(_data_i)
|
|
|
+ if len(list_index)>0:
|
|
|
+ range_data = list_data[min(list_index):max(list_index)+1]
|
|
|
+ return range_data
|
|
|
+
|
|
|
+def get_correct_product(product,products):
|
|
|
+ list_data = []
|
|
|
+ for p in products:
|
|
|
+ is_sim = is_similar(product,p)
|
|
|
+ _d = {"product":p,"distance":abs(len(product)-len(p)),"is_sim":is_sim}
|
|
|
+ list_data.append(_d)
|
|
|
+ list_data.sort(key=lambda x:x["distance"])
|
|
|
+ for _d in list_data:
|
|
|
+ is_sim = _d["is_sim"]
|
|
|
+ if is_sim:
|
|
|
+ if len(_d["product"])>len(product) and _d["product"].find(product)>=0:
|
|
|
+ return product
|
|
|
+ return _d["product"]
|
|
|
+ return product
|
|
|
+
|
|
|
+def get_childs_text(childs,_product,products,is_begin=False,is_end=False):
|
|
|
+ _text = ""
|
|
|
+
|
|
|
+ for _child in childs:
|
|
|
+
|
|
|
+ child_text = _child.get("text")
|
|
|
+
|
|
|
+ if child_text.find(_product)>=0:
|
|
|
+ is_begin = True
|
|
|
+
|
|
|
+ logger.debug("%s-%s-%s"%("get_childs_text",child_text[:10],str(is_begin)))
|
|
|
+
|
|
|
+ for p in products:
|
|
|
+ if child_text.find(p)>=0 and is_similar(_product,p,90):
|
|
|
+ is_begin = True
|
|
|
+
|
|
|
+ if child_text.find(_product)<0 and child_text.find(p)>=0 and not is_similar(_product,p,80):
|
|
|
+ if is_begin:
|
|
|
+ is_end = True
|
|
|
+ break
|
|
|
+ if re.search(end_pattern,child_text) is not None:
|
|
|
+ if is_begin:
|
|
|
+ is_end = True
|
|
|
+
|
|
|
+ if is_begin and is_end:
|
|
|
+ break
|
|
|
+
|
|
|
+ if is_begin:
|
|
|
+ _text += _child.get("text")+"\r\n"
|
|
|
+ childs2 = _child.get("child_title",[])
|
|
|
+
|
|
|
+ if len(childs2)>0:
|
|
|
+ for _child2 in childs2:
|
|
|
+ child_text,is_begin,is_end = get_childs_text([_child2],_product,products,is_begin)
|
|
|
+ if is_begin and is_end:
|
|
|
+ break
|
|
|
+ else:
|
|
|
+ if is_begin:
|
|
|
+ _text += child_text
|
|
|
+ return _text,is_begin,is_end
|
|
|
+
|
|
|
+def extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result,):
|
|
|
+ _data = list_data[_data_i]
|
|
|
+ childs = _data.get("child_title",[])
|
|
|
+ if len(childs)>0:
|
|
|
+ child_text,_,_ = get_childs_text([parent_title],_product,products)
|
|
|
+ logger.info("extract_parameters_by_tree child_text:%s"%child_text)
|
|
|
+ if len(child_text)>0:
|
|
|
+ list_result.append(child_text)
|
|
|
+ return True
|
|
|
+ if parent_title is not None:
|
|
|
+ childs = parent_title.get("child_title",[])
|
|
|
+ if len(childs)>0:
|
|
|
+
|
|
|
+ range_data = get_range_data_by_childs(list_data[_data_i:],childs)
|
|
|
+ p_text = ""
|
|
|
+ _find = False
|
|
|
+ for pdata in range_data:
|
|
|
+ ptype = _data["type"]
|
|
|
+ ptext = pdata["text"]
|
|
|
+ for p in products:
|
|
|
+ if ptext.find(_product)<0 and ptext.find(p)>=0:
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
+ if re.search(end_pattern,ptext) is not None:
|
|
|
+ _find = True
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ p_text += ptext+"\r\n"
|
|
|
+ if len(p_text)>0:
|
|
|
+ list_result.append(p_text)
|
|
|
+ return True
|
|
|
+ return False
|
|
|
+
|
|
|
+
|
|
|
+def get_table_pieces(_text,_product,products,list_result,_find):
|
|
|
+ _soup = BeautifulSoup(_text,"html5lib")
|
|
|
+ _table = _soup.find("table")
|
|
|
+ if _table is not None:
|
|
|
+ trs = getTrs(_table)
|
|
|
+ list_trs = []
|
|
|
+ for tr in trs:
|
|
|
+ tr_text = tr.get_text()
|
|
|
+ if tr_text.find(_product)>=0:
|
|
|
+ _find = True
|
|
|
+
|
|
|
+ logger.debug("%s-%s"%("table_html_tr",tr_text))
|
|
|
+ for p in products:
|
|
|
+ if _find and p!=_product and tr_text.find(p)>=0:
|
|
|
+ _find = False
|
|
|
+ break
|
|
|
+ if re.search(end_pattern,tr_text) is not None:
|
|
|
+ _find = False
|
|
|
+ break
|
|
|
+ if _find:
|
|
|
+ list_trs.append(tr)
|
|
|
+ if len(list_trs)>0:
|
|
|
+ table_html = "<table>%s</table>"%("\r\n".join([str(a) for a in list_trs]))
|
|
|
+ list_result.append(table_html)
|
|
|
+
|
|
|
+def extract_parameters_by_table(_product,products,_param_pattern,list_data,_data_i,list_result):
|
|
|
+ _data = list_data[_data_i]
|
|
|
+ _text = _data["text"]
|
|
|
+ list_table = _data["list_table"]
|
|
|
+ parent_title = _data["parent_title"]
|
|
|
+ if list_table is not None:
|
|
|
+ _check = True
|
|
|
+ max_length = max([len(a) for a in list_table])
|
|
|
+ min_length = min([len(a) for a in list_table])
|
|
|
+ text_line_first = ",".join(a[0] for a in list_table[0])
|
|
|
+ if min_length<max_length/2:
|
|
|
+ return
|
|
|
+ last_data = list_data[_data_i-1]
|
|
|
+ _flag = False
|
|
|
+ if last_data["type"]=="sentence" and last_data["text"].find(_product)>=0:
|
|
|
+ logger.debug("last sentence find product %s-%s"%(_product,last_data["text"]))
|
|
|
+ _flag = True
|
|
|
+ # print(text_line_first,"text_line_first",re.search(_param_pattern,text_line_first) is not None and text_line_first.find(_product)>=0)
|
|
|
+ if re.search(_param_pattern,text_line_first) is not None and text_line_first.find(_product)>=0:
|
|
|
+ _flag = True
|
|
|
+ if _flag:
|
|
|
+ logger.debug("extract_type add all table %s"%_text)
|
|
|
+ if len(products)==0:
|
|
|
+ list_result.append(_text)
|
|
|
+ else:
|
|
|
+ for p in products:
|
|
|
+ if p!=_product and _text.find(p)>=0:
|
|
|
+ logger.debug("extract_type add all table failed %s-%s"%(_product,p))
|
|
|
+ _flag = False
|
|
|
+ break
|
|
|
+ if _flag:
|
|
|
+ logger.debug("extract_type add all table succeed")
|
|
|
+ get_table_pieces(_text,_product,products,list_result,True)
|
|
|
+ else:
|
|
|
+ list_head_index = []
|
|
|
+ for line in list_table[:2]:
|
|
|
+ for cell_i in range(len(line)):
|
|
|
+ cell = line[cell_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if len(cell_text)<20 and re.search(_param_pattern,cell_text) is not None:
|
|
|
+ list_head_index.append(cell_i)
|
|
|
+ list_head_index = list(set(list_head_index))
|
|
|
+ for line in list_table:
|
|
|
+ for cell in line:
|
|
|
+ cell_text = cell[0]
|
|
|
+ if len(cell_text)>50 and len(re.findall("\d+",cell_text))>10 and cell_text.find(_product)>=0:
|
|
|
+ list_result.append(cell_text)
|
|
|
+ if len(cell_text)<len(_product)*10 and str(cell_text).find(_product)>=0:
|
|
|
+ for _index in list_head_index:
|
|
|
+ if _index>=len(line):
|
|
|
+ continue
|
|
|
+ _cell = line[_index]
|
|
|
+ if len(cell[0])>0:
|
|
|
+ logger.info("%s-%s"%("add on table",_cell[0]))
|
|
|
+ list_result.append(_cell[0])
|
|
|
+ if not _flag and (re.search(_param_pattern,_text) is not None or (parent_title is not None and re.search(_param_pattern,parent_title["text"]) is not None)) and _text.find(_product)>=0:
|
|
|
+ get_table_pieces(_text,_product,products,list_result,False)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+def getBestProductText(list_result,_product,products):
|
|
|
+ list_result.sort(key=lambda x:len(re.findall(meter_pattern+"|"+'[:：;；]|\d+[%A-Za-z]+',BeautifulSoup(x,"html5lib").get_text())), reverse=True)
|
|
|
+
|
|
|
+ logger.debug("+++++++++++++++++++++")
|
|
|
+ for i in range(len(list_result)):
|
|
|
+ logger.debug("result%d %s"%(i,list_result[i]))
|
|
|
+ logger.debug("+++++++++++++++++++++")
|
|
|
+
|
|
|
+ for i in range(len(list_result)):
|
|
|
+ _result = list_result[i]
|
|
|
+ _check = True
|
|
|
+ _result_text = BeautifulSoup(_result,"html5lib").get_text()
|
|
|
+ _search = re.search("项目编号[:：]|项目名称[:：]|联合体投标",_result)
|
|
|
+ if _search is not None:
|
|
|
+ logger.debug("result%d error illegal text %s"%(i,str(_search)))
|
|
|
+ _check = False
|
|
|
+ if not (len(_result_text)<1000 and _result[:6]!="<table"):
|
|
|
+ for p in products:
|
|
|
+ if _result_text.find(p)>0 and not (is_similar(_product,p,80) or p.find(_product)>=0 or _product.find(p)>=0):
|
|
|
+ logger.debug("result%d error product scoss %s"%(i,p))
|
|
|
+ _check = False
|
|
|
+ if len(_result_text)<50:
|
|
|
+ if re.search(meter_pattern,_result_text) is None:
|
|
|
+ logger.debug("result%d error text min count"%(i))
|
|
|
+ _check = False
|
|
|
+ if len(_result_text)>5000:
|
|
|
+ logger.debug("result%d error text max count"%(i))
|
|
|
+ _check = False
|
|
|
+ if _check:
|
|
|
+ return _result
|
|
|
+
|
|
|
+def extract_product_parameters(list_data,_product):
|
|
|
+
|
|
|
+ list_result = []
|
|
|
+ _product = standard_product(_product.strip())
|
|
|
+ products = extract_products(list_data,_product)
|
|
|
+
|
|
|
+ _product = get_correct_product(_product,products)
|
|
|
+ logger.debug("all products %s-%s"%(_product,str(products)))
|
|
|
+ is_project = False
|
|
|
+ _find_count = 0
|
|
|
+ for _data_i in range(len(list_data)):
|
|
|
+ _data = list_data[_data_i]
|
|
|
+ _type = _data["type"]
|
|
|
+ _text = _data["text"]
|
|
|
+ if _type=="sentence":
|
|
|
+ if _text.find(_product)>=0:
|
|
|
+ _find_count += 1
|
|
|
+ if re.search("项目名称|采购项目",_text) is not None:
|
|
|
+ is_project = True
|
|
|
+ if re.search("项目名称|采购项目",_product) is not None:
|
|
|
+ is_project = True
|
|
|
+ parent_title = _data.get("parent_title")
|
|
|
+ parent_text = ""
|
|
|
+ parent_parent_title = None
|
|
|
+ parent_parent_text = ""
|
|
|
+ parent_title_index = None
|
|
|
+ parent_parent_title_index = None
|
|
|
+ childs = get_childs([_data])
|
|
|
+
|
|
|
+
|
|
|
+ child_find = False
|
|
|
+ for c in childs:
|
|
|
+ if re.search(_param_pattern,c["text"]) is not None and len(c["text"])<30:
|
|
|
+ child_find = True
|
|
|
+ break
|
|
|
+
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
+ logger.debug("childs found extract_text %s"%extract_text)
|
|
|
+ if child_find:
|
|
|
+ if len(extract_text)>0:
|
|
|
+ list_result.append(extract_text)
|
|
|
+ else:
|
|
|
+ if len(_text)<len(_product)+10 and _data["sentence_title"] is not None:
|
|
|
+ if re.search(meter_pattern,extract_text) is not None:
|
|
|
+ list_result.append(extract_text)
|
|
|
+
|
|
|
+ if parent_title is not None:
|
|
|
+ parent_text = parent_title.get("text","")
|
|
|
+ parent_parent_title = parent_title.get("parent_title")
|
|
|
+ parent_title_index = parent_title["title_index"]
|
|
|
+ if parent_parent_title is not None:
|
|
|
+ parent_parent_text = parent_parent_title.get("text","")
|
|
|
+ parent_parent_title_index = parent_parent_title["title_index"]
|
|
|
+
|
|
|
+ _suit = False
|
|
|
+ if re.search(_param_pattern,_text) is not None and len(_text)<50:
|
|
|
+ _suit = True
|
|
|
+ if re.search(_param_pattern,parent_text) is not None and len(parent_text)<50:
|
|
|
+ _suit = True
|
|
|
+ if re.search(_param_pattern,parent_parent_text) is not None and len(parent_parent_text)<50:
|
|
|
+ _suit = True
|
|
|
+ if _suit:
|
|
|
+ logger.debug("extract_type sentence %s"%("extract_parameters_by_tree"))
|
|
|
+ if not extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result):
|
|
|
+ logger.debug("extract_type sentence %s"%("extract_parameters_by_tree"))
|
|
|
+ extract_parameters_by_tree(_product,products,list_data,_data_i,parent_parent_title,list_result)
|
|
|
+
|
|
|
+
|
|
|
+ if re.search(_param_pattern,_text) is not None and len(_text)<50:
|
|
|
+ childs = _data["child_title"]
|
|
|
+ if len(childs)>0:
|
|
|
+ logger.debug("extract_type sentence %s"%("re.search(_param_pattern,_text) is not None and len(_text)<50:"))
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
+ if len(extract_text)>0:
|
|
|
+ list_result.append(extract_text)
|
|
|
+ elif is_project:
|
|
|
+ logger.debug("extract_type sentence is_project")
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products,is_begin=True)
|
|
|
+ if len(extract_text)>0 and re.search(meter_pattern,extract_text) is not None:
|
|
|
+ list_result.append(extract_text)
|
|
|
+
|
|
|
+
|
|
|
+ elif _type=="table":
|
|
|
+ if _text.find(_product)>=0:
|
|
|
+ _find_count += 1
|
|
|
+ extract_parameters_by_table(_product,products,_param_pattern,list_data,_data_i,list_result)
|
|
|
+
|
|
|
+
|
|
|
+ return getBestProductText(list_result,_product,products),_find_count
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+
|
|
|
+ filepath = "download/8679fef3a6fff56abcbdaccb1a190c80.html"
|
|
|
+ _product = "移液器"
|
|
|
+
|
|
|
+ _html = open(filepath, "r", encoding="utf8").read()
|
|
|
+
|
|
|
+ pd = ParseDocument(_html,False)
|
|
|
+
|
|
|
+ pd.fix_tree(_product)
|
|
|
+ list_data = pd.tree
|
|
|
+ pd.print_tree(list_data)
|
|
|
+
|
|
|
+ _text,_count = extract_product_parameters(list_data,_product)
|
|
|
+ logger.info("find count:%d"%(_count))
|
|
|
+ logger.info("extract_text %s"%_text)
|
|
|
+
|
