|
|
@@ -2,22 +2,22 @@
|
|
|
|
|
|
import re
|
|
|
|
|
|
-from BaseDataMaintenance.maintenance.product.productUtils import *
|
|
|
+from BaseDataMaintenance.maintenance.product.productUtils import is_similar
|
|
|
import logging
|
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
-logger.setLevel(logging.DEBUG)
|
|
|
-
|
|
|
-
|
|
|
+logger.setLevel(logging.INFO)
|
|
|
|
|
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
import copy
|
|
|
|
|
|
end_pattern = "商务要求|评分标准|商务条件|商务条件"
|
|
|
-_param_pattern = "(产品|技术|清单[及和]?|配置|参数|具体|明细[及和]?|项目|货物|服务)(指标|配置|要求|参数|需求|规格)|配置清单|(质量|技术).{,10}要求|验收标准|^参数$"
|
|
|
-meter_pattern = "角度|容积|色彩|帧率|磁场|强度|允差|噪音|材质|频率|阻抗|浓度|范围|误差|精确|温度|可调|设定值|功能|检测|高度|宽度|模式|尺寸|重量|峰值|容量|寿命|稳定性|高温|电源|电压|功率|压力|压强"
|
|
|
+_param_pattern = "(产品|技术|清单|配置|参数|具体|明细|项目|招标|货物|服务|规格|工作|具体)[及和与]?(指标|配置|条件|要求|参数|需求|规格|条款|名称及要求)|配置清单|(质量|技术).{,10}要求|验收标准|^(参数|功能)$"
|
|
|
+meter_pattern = "[><≤≥±]\d+|\d+(?:[μucmkK微毫千]?[米升LlgGmMΩ]|摄氏度|英寸|度|天|VA|dB|bpm|rpm|kPa|mol|cmH20|%|°|Mpa|Hz|K?HZ|℃|W|min|[*×xX])|[*×xX]\d+|/min|\ds[^a-zA-Z]|GB.{,20}标准|PVC|PP|角度|容积|色彩|自动|流量|外径|轴位|折射率|帧率|柱镜|振幅|磁场|镜片|防漏|强度|允差|心率|倍数|瞳距|底座|色泽|噪音|间距|材质|材料|表面|频率|阻抗|浓度|兼容|防尘|防水|内径|实时|一次性|误差|性能|距离|精确|温度|超温|范围|跟踪|对比度|亮度|[横纵]向|均压|负压|正压|可调|设定值|功能|检测|高度|厚度|宽度|深度|[单双多]通道|效果|指数|模式|尺寸|重量|峰值|谷值|容量|寿命|稳定性|高温|信号|电源|电流|转换率|效率|释放量|转速|离心力|向心力|弯曲|电压|功率|气量|国标|标准协议|灵敏度|最大值|最小值|耐磨|波形|高压|性强|工艺|光源|低压|压力|压强|速度|湿度|重量|毛重|[MLX大中小]+码|净重|颜色|[红橙黄绿青蓝紫]色|不锈钢|输入|输出|噪声|认证|配置"
|
|
|
+not_meter_pattern = "投标报价|中标金额|商务部分|公章|分值构成|业绩|详见|联系人|联系电话|合同价|金额|采购预算|资金来源|费用|质疑|评审因素|评审标准|商务资信|商务评分|专家论证意见|评标方法|代理服务费|售后服务|评分类型|评分项目|预算金额|得\d+分|项目金额|详见招标文件|乙方"
|
|
|
+
|
|
|
|
|
|
def getTrs(tbody):
|
|
|
#获取所有的tr
|
|
|
@@ -128,7 +128,7 @@ class ParseDocument():
|
|
|
_body = self.soup.find("body")
|
|
|
if _body is not None:
|
|
|
self.soup = _body
|
|
|
- self.list_obj = self.soup.find_all(recursive=False)
|
|
|
+ self.list_obj = self.get_soup_objs(self.soup)
|
|
|
|
|
|
# for obj in self.list_obj:
|
|
|
# print("obj",obj.get_text()[:20])
|
|
|
@@ -140,6 +140,18 @@ class ParseDocument():
|
|
|
# if self.parseTree:
|
|
|
# self.parseTree.printParseTree()
|
|
|
|
|
|
+ def get_soup_objs(self,soup,list_obj=None):
|
|
|
+ if list_obj is None:
|
|
|
+ list_obj = []
|
|
|
+ childs = soup.find_all(recursive=False)
|
|
|
+ for _obj in childs:
|
|
|
+ childs1 = _obj.find_all(recursive=False)
|
|
|
+ if len(childs1)==0 or len(_obj.get_text())<40 or _obj.name=="table":
|
|
|
+ list_obj.append(_obj)
|
|
|
+ else:
|
|
|
+ self.get_soup_objs(_obj,list_obj)
|
|
|
+ return list_obj
|
|
|
+
|
|
|
def fix_tree(self,_product):
|
|
|
products = extract_products(self.tree,_product)
|
|
|
if len(products)>0:
|
|
|
@@ -148,12 +160,16 @@ class ParseDocument():
|
|
|
def print_tree(self,tree,append=""):
|
|
|
if append=="":
|
|
|
self.set_tree_id = set()
|
|
|
+
|
|
|
+ # for t in tree:
|
|
|
+ # logger.debug("%s text:%s title:%s title_text:%s before:%s after%s product:%s"%("==>",t["text"][:50],t["sentence_title"],t["sentence_title_text"],t["title_before"],t["title_after"],t["has_product"]))
|
|
|
+
|
|
|
for t in tree:
|
|
|
_id = id(t)
|
|
|
if _id in self.set_tree_id:
|
|
|
continue
|
|
|
self.set_tree_id.add(_id)
|
|
|
- logger.debug("%s %s %s %s %s"%(append,t["text"][:50],t["sentence_title"],t["title_before"],t["title_after"]))
|
|
|
+ logger.debug("%s text:%s title:%s title_text:%s before:%s after%s product:%s"%(append,t["text"][:50],t["sentence_title"],t["sentence_title_text"],t["title_before"],t["title_after"],t["has_product"]))
|
|
|
childs = t["child_title"]
|
|
|
self.print_tree(childs,append=append+"-|")
|
|
|
|
|
|
@@ -162,18 +178,18 @@ class ParseDocument():
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
- def find_title_by_pattern(self,_text,_pattern="(^|★|▲|:|：|\s+)(?P<title_1>(?P<title_1_index_0_0>第?)(?P<title_1_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_1_index_2_0>[、章册包标部\.:：]))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_3>(?P<title_3_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?)(?P<title_3_index_0_1>[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_3_index_0_2>))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_4>(?P<title_4_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?第?)(?P<title_4_index_1_1>[一二三四五六七八九十]+)(?P<title_4_index_2_0>[节章册部\.:：、、]))|" \
|
|
|
+ def find_title_by_pattern(self,_text,_pattern="(^|★|▲|:|：|\s+)(?P<title_1>(?P<title_1_index_0_0>第?)(?P<title_1_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_1_index_2_0>[、章册包标部.:：、、]+))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_3>(?P<title_3_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?)(?P<title_3_index_0_1>[ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_3_index_0_2>[、章册包标部.:：、、]+))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_4>(?P<title_4_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?第?)(?P<title_4_index_1_1>[一二三四五六七八九十]+)(?P<title_4_index_2_0>[节章册部\.:：、、]+))|" \
|
|
|
"([\s★▲\*]*)(?P<title_5>(?P<title_5_index_0_0>^)(?P<title_5_index_1_1>[一二三四五六七八九十]+)(?P<title_5_index_2_0>)[^一二三四五六七八九十节章册部\.:：、、])|" \
|
|
|
"([\s★▲\*]*)(?P<title_12>(?P<title_12_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_12_index_1_1>\d{1,2})(?P<title_12_index_2_0>[\.．、\s\-]?))|"\
|
|
|
"([\s★▲\*]*)(?P<title_11>(?P<title_11_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_11_index_1_1>\d{1,2})(?P<title_11_index_2_0>[\.．、\s\-]?))|" \
|
|
|
"([\s★▲\*]*)(?P<title_10>(?P<title_10_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．、\s\-]\d{1,2}[\.．、\s\-])(?P<title_10_index_1_1>\d{1,2})(?P<title_10_index_2_0>[\.．、\s\-]?))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_7>(?P<title_7_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．\s\-])(?P<title_7_index_1_1>\d{1,2})(?P<title_7_index_2_0>[\.．包标:：、\s\-]?))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_6>(?P<title_6_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?包?)(?P<title_6_index_0_1>\d{1,2})(?P<title_6_index_2_0>[\.．、\s\-包标]?))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_15>(?P<title_15_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_15_index_1_1>\d{1,2})(?P<title_15_index_2_0>[)）包标]))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_17>(?P<title_17_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_17_index_1_1>[a-wA-W]+)(?P<title_17_index_2_0>[)）包标]))|" \
|
|
|
- "([\s★▲\*]*)(?P<title_19>(?P<title_19_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_19_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_19_index_2_0>[)）]))" \
|
|
|
+ "([\s★▲\*]*)(?P<title_7>(?P<title_7_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?\d{1,2}[\.．\s\-])(?P<title_7_index_1_1>\d{1,2})(?P<title_7_index_2_0>[\.．包标:：、\s\-]*))|" \
|
|
|
+ "(^[\s★▲\*]*)(?P<title_6>(?P<title_6_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?包?)(?P<title_6_index_0_1>\d{1,2})(?P<title_6_index_2_0>[\.．、\s\-包标]*))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_15>(?P<title_15_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_15_index_1_1>\d{1,2})(?P<title_15_index_2_0>[)）包标\.．:：、]+))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_17>(?P<title_17_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_17_index_1_1>[a-zA-Z]+)(?P<title_17_index_2_0>[)）包标\.．:：、]+))|" \
|
|
|
+ "([\s★▲\*]*)(?P<title_19>(?P<title_19_index_0_0>[^一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]{,3}?[(（]?)(?P<title_19_index_1_1>[一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)(?P<title_19_index_2_0>[)）]))"
|
|
|
):
|
|
|
_se = re.search(_pattern,_text)
|
|
|
groups = []
|
|
|
@@ -324,6 +340,7 @@ class ParseDocument():
|
|
|
max_length = max(list_length)
|
|
|
else:
|
|
|
max_length = 40
|
|
|
+ max_length = min(max_length,40)
|
|
|
|
|
|
logger.debug("%s:%d"%("max_length",max_length))
|
|
|
|
|
|
@@ -335,6 +352,9 @@ class ParseDocument():
|
|
|
dict_before,illegal_sentence = self.count_title_before(list_obj)
|
|
|
for obj_i in range(len(list_obj)):
|
|
|
obj = list_obj[obj_i]
|
|
|
+
|
|
|
+ # logger.debug("==obj %s"%obj.text[:20])
|
|
|
+
|
|
|
_type = "sentence"
|
|
|
_text = standard_product(obj.text)
|
|
|
if obj.name=="table":
|
|
|
@@ -355,36 +375,62 @@ class ParseDocument():
|
|
|
list_table = None
|
|
|
block = False
|
|
|
|
|
|
+ has_product = False
|
|
|
+
|
|
|
if _type=="sentence":
|
|
|
if _text in illegal_sentence:
|
|
|
continue
|
|
|
|
|
|
- _fix = False
|
|
|
- for p in products:
|
|
|
- if re.sub("^(\d[.、]?)+","",_text.strip())==p:
|
|
|
- title_before = "=产品"
|
|
|
- sentence_title = "title_0"
|
|
|
- sentence_title_text = p
|
|
|
- title_index = "0"
|
|
|
- title_after = "产品="
|
|
|
- next_index = "0"
|
|
|
- _fix = True
|
|
|
- break
|
|
|
|
|
|
- if not _fix:
|
|
|
- sentence_groups = self.find_title_by_pattern(_text[:10])
|
|
|
- if sentence_groups:
|
|
|
- title_before = standard_title_context(sentence_groups[1][1])
|
|
|
- if title_before in dict_before and dict_before[title_before]>1:
|
|
|
- sentence_title = sentence_groups[0][0]
|
|
|
- sentence_title_text = sentence_groups[0][1]
|
|
|
- title_index = sentence_groups[-2][1]
|
|
|
-
|
|
|
- title_after = sentence_groups[-1][1]
|
|
|
- next_index = self.get_next_title(title_index)
|
|
|
- else:
|
|
|
- title_before = None
|
|
|
+ sentence_groups = self.find_title_by_pattern(_text[:10])
|
|
|
+ if sentence_groups:
|
|
|
+ title_before = standard_title_context(sentence_groups[1][1])
|
|
|
+ title_after = sentence_groups[-1][1]
|
|
|
+ sentence_title_text = sentence_groups[0][1]
|
|
|
+ other_text = _text.replace(sentence_title_text,"")
|
|
|
+ if (title_before in dict_before and dict_before[title_before]>1) or title_after!="":
|
|
|
+ sentence_title = sentence_groups[0][0]
|
|
|
+
|
|
|
+ title_index = sentence_groups[-2][1]
|
|
|
+ next_index = self.get_next_title(title_index)
|
|
|
+
|
|
|
+ other_text = _text.replace(sentence_title_text,"")
|
|
|
+
|
|
|
+ for p in products:
|
|
|
+ if other_text.strip()==p.strip():
|
|
|
+ has_product = True
|
|
|
|
|
|
+ else:
|
|
|
+ _fix = False
|
|
|
+
|
|
|
+ for p in products:
|
|
|
+ if other_text.strip()==p.strip():
|
|
|
+ title_before = "=产品"
|
|
|
+ sentence_title = "title_0"
|
|
|
+ sentence_title_text = p
|
|
|
+ title_index = "0"
|
|
|
+ title_after = "产品="
|
|
|
+ next_index = "0"
|
|
|
+ _fix = True
|
|
|
+ has_product = True
|
|
|
+ break
|
|
|
+ if not _fix:
|
|
|
+ title_before = None
|
|
|
+ title_after = None
|
|
|
+ sentence_title_text = None
|
|
|
+ else:
|
|
|
+ if len(_text)<40 and re.search(_param_pattern,_text) is not None:
|
|
|
+ for p in products:
|
|
|
+ if _text.find(p)>=0:
|
|
|
+ title_before = "=产品"
|
|
|
+ sentence_title = "title_0"
|
|
|
+ sentence_title_text = p
|
|
|
+ title_index = "0"
|
|
|
+ title_after = "产品="
|
|
|
+ next_index = "0"
|
|
|
+ _fix = True
|
|
|
+ has_product = True
|
|
|
+ break
|
|
|
|
|
|
if _type=="sentence":
|
|
|
if sentence_title is None and len(list_data)>0 and list_data[-1]["sentence_title"] is not None and list_data[-1]["line_width"]>=max_length*0.6:
|
|
|
@@ -402,6 +448,8 @@ class ParseDocument():
|
|
|
_table = _soup.find("table")
|
|
|
if _table is not None:
|
|
|
list_table = getTable(_table)
|
|
|
+ if len(list_table)==0:
|
|
|
+ continue
|
|
|
table_columns = len(list_table[0])
|
|
|
|
|
|
if auto_merge_table:
|
|
|
@@ -428,7 +476,7 @@ class ParseDocument():
|
|
|
_data = {"type":_type, "text":_text,"list_table":list_table,"line_width":len(_text),"sentence_title":sentence_title,"title_index":title_index,
|
|
|
"sentence_title_text":sentence_title_text,"sentence_groups":sentence_groups,"parent_title":parent_title,
|
|
|
"child_title":childs,"title_before":title_before,"title_after":title_after,"title_next":title_next,"next_index":next_index,
|
|
|
- "block":block}
|
|
|
+ "block":block,"has_product":has_product}
|
|
|
|
|
|
if _type=="table":
|
|
|
last_table = _data
|
|
|
@@ -543,22 +591,56 @@ class ParseDocument():
|
|
|
|
|
|
list_data.append(_data)
|
|
|
|
|
|
+ for _data in list_data:
|
|
|
+
|
|
|
+ childs = _data["child_title"]
|
|
|
+
|
|
|
+ for c_i in range(len(childs)):
|
|
|
+ cdata = childs[c_i]
|
|
|
+ if cdata["has_product"]:
|
|
|
+ continue
|
|
|
+ else:
|
|
|
+ if c_i>0:
|
|
|
+ last_cdata = childs[c_i-1]
|
|
|
+ if cdata["sentence_title"] is not None and last_cdata["sentence_title"] is not None and last_cdata["title_before"]==cdata["title_before"] and last_cdata["title_after"]==cdata["title_after"] and last_cdata["has_product"]:
|
|
|
+ cdata["has_product"] = True
|
|
|
+ if c_i<len(childs)-1:
|
|
|
+ last_cdata = childs[c_i+1]
|
|
|
+ if cdata["sentence_title"] is not None and last_cdata["sentence_title"] is not None and last_cdata["title_before"]==cdata["title_before"] and last_cdata["title_after"]==cdata["title_after"] and last_cdata["has_product"]:
|
|
|
+ cdata["has_product"] = True
|
|
|
+ for c_i in range(len(childs)):
|
|
|
+ cdata = childs[len(childs)-1-c_i]
|
|
|
+ if cdata["has_product"]:
|
|
|
+ continue
|
|
|
+ else:
|
|
|
+ if c_i>0:
|
|
|
+ last_cdata = childs[c_i-1]
|
|
|
+ if cdata["sentence_title"] is not None and last_cdata["sentence_title"] is not None and last_cdata["title_before"]==cdata["title_before"] and last_cdata["title_after"]==cdata["title_after"] and last_cdata["has_product"]:
|
|
|
+ cdata["has_product"] = True
|
|
|
+ if c_i<len(childs)-1:
|
|
|
+ last_cdata = childs[c_i+1]
|
|
|
+ if cdata["sentence_title"] is not None and last_cdata["sentence_title"] is not None and last_cdata["title_before"]==cdata["title_before"] and last_cdata["title_after"]==cdata["title_after"] and last_cdata["has_product"]:
|
|
|
+ cdata["has_product"] = True
|
|
|
+
|
|
|
+
|
|
|
return list_data
|
|
|
|
|
|
+
|
|
|
def standard_title_context(_title_context):
|
|
|
return _title_context.replace("（","(").replace("）",")").replace("：",":").replace("：",";").replace("，",".").replace(",",".").replace("、",".").replace("．",".")
|
|
|
|
|
|
def standard_product(sentence):
|
|
|
return sentence.replace("（","(").replace("）",")")
|
|
|
|
|
|
-def extract_products(list_data,_product,_param_pattern = "产品名称|采购内存|标的名称|采购内容|(标的|维修|系统|报价构成|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品?|采购|物装|配件|资产|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|品目|^品名|气体|标项|分项|项目|计划|包组|标段|[分子]?包|子目|服务|招标|中标|成交|工程|招标内容)[\)）的]?([、\w]{,4}名称|内容|描述)|标的|标项|项目$|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品|物装|配件|资产|招标内容|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|菜名|^品目$|^品名$|^名称|^内容$"):
|
|
|
+def extract_products(list_data,_product,_param_pattern = "产品名称|设备材料|采购内存|标的名称|采购内容|(标的|维修|系统|报价构成|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品?|采购|物装|配件|资产|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|品目|^品名|气体|标项|分项|项目|计划|包组|标段|[分子]?包|子目|服务|招标|中标|成交|工程|招标内容)[\)）的]?([、\w]{,4}名称|内容|描述)|标的|标项|项目$|商品|产品|物料|物资|货物|设备|采购品|采购条目|物品|材料|印刷品|物装|配件|资产|招标内容|耗材|清单|器材|仪器|器械|备件|拍卖物|标的物|物件|药品|药材|药械|货品|食品|食材|菜名|^品目$|^品名$|^名称|^内容$"):
|
|
|
_product = standard_product(_product)
|
|
|
list_result = []
|
|
|
+ list_table_products = []
|
|
|
for _data_i in range(len(list_data)):
|
|
|
_data = list_data[_data_i]
|
|
|
_type = _data["type"]
|
|
|
_text = _data["text"]
|
|
|
- table_products = []
|
|
|
+
|
|
|
if _type=="table":
|
|
|
list_table = _data["list_table"]
|
|
|
if list_table is None:
|
|
|
@@ -584,6 +666,8 @@ def extract_products(list_data,_product,_param_pattern = "产品名称|采购内
|
|
|
for line_i in range(len(list_table)):
|
|
|
line = list_table[line_i]
|
|
|
for cell_i in list_head_index:
|
|
|
+ if cell_i>=len(line):
|
|
|
+ continue
|
|
|
cell = line[cell_i]
|
|
|
cell_text = cell[0]
|
|
|
head_cell_text += cell_text
|
|
|
@@ -592,27 +676,29 @@ def extract_products(list_data,_product,_param_pattern = "产品名称|采购内
|
|
|
if re.search("招标人|采购人|项目编号|项目名称|金额|^\d+$",head_cell_text) is not None:
|
|
|
list_head_index = []
|
|
|
|
|
|
-
|
|
|
for line in list_table:
|
|
|
line_text = ",".join([cell[0] for cell in line])
|
|
|
for cell_i in range(len(line)):
|
|
|
cell = line[cell_i]
|
|
|
cell_text = cell[0]
|
|
|
- if cell_text is not None and _product is not None and len(cell_text)<len(_product)*10 and re.search(_product,cell_text) is not None and re.search("单价|数量|总价|规格|品牌|型号|用途|要求|采购量",line_text) is not None:
|
|
|
+ if cell_text is not None and _product is not None and len(cell_text)<len(_product)*10 and cell_text.find(_product)>=0 and re.search("单价|数量|总价|规格|品牌|型号|用途|要求|采购量",line_text) is not None:
|
|
|
list_head_index.append(cell_i)
|
|
|
|
|
|
list_head_index = list(set(list_head_index))
|
|
|
if len(list_head_index)>0:
|
|
|
- for line_i in range(_begin_index,len(list_table)):
|
|
|
- line = list_table[line_i]
|
|
|
- has_number = False
|
|
|
- for cell_i in range(len(line)):
|
|
|
- cell = line[cell_i]
|
|
|
- cell_text = cell[0]
|
|
|
- if re.search("^\d+$",cell_text) is not None:
|
|
|
- has_number = True
|
|
|
+ has_number = False
|
|
|
+ for cell_i in list_head_index:
|
|
|
+ table_products = []
|
|
|
+
|
|
|
+ for line_i in range(_begin_index,len(list_table)):
|
|
|
+ line = list_table[line_i]
|
|
|
+
|
|
|
+ for _i in range(len(line)):
|
|
|
+ cell = line[_i]
|
|
|
+ cell_text = cell[0]
|
|
|
+ if re.search("^\d+$",cell_text) is not None:
|
|
|
+ has_number = True
|
|
|
|
|
|
- for cell_i in list_head_index:
|
|
|
if cell_i>=len(line):
|
|
|
continue
|
|
|
cell = line[cell_i]
|
|
|
@@ -621,10 +707,23 @@ def extract_products(list_data,_product,_param_pattern = "产品名称|采购内
|
|
|
if re.search("^[\da-zA-Z]+$",cell_text) is None:
|
|
|
table_products.append(cell_text)
|
|
|
|
|
|
- if len(table_products)>0:
|
|
|
- if min([len(x) for x in table_products])>0 and max([len(x) for x in table_products])<=20:
|
|
|
- list_result.extend(table_products)
|
|
|
- list_result = list(set([a for a in list_result if len(a)>1 and len(a)<20 and re.search("预算|合计|金额|万元|运费",a) is None]))
|
|
|
+ if len(table_products)>0:
|
|
|
+ logger.debug("table products %s"%(str(table_products)))
|
|
|
+ if min([len(x) for x in table_products])>0 and max([len(x) for x in table_products])<=30:
|
|
|
+ if re.search("招标人|代理人|预算|数量|交货期|品牌|产地","".join(table_products)) is None:
|
|
|
+ list_table_products.append(table_products)
|
|
|
+ _find = False
|
|
|
+ for table_products in list_table_products:
|
|
|
+ for _p in table_products:
|
|
|
+ if is_similar(_product,_p,90):
|
|
|
+ _find = True
|
|
|
+ logger.debug("similar table_products %s"%(str(table_products)))
|
|
|
+ list_result = list(set([a for a in table_products if len(a)>1 and len(a)<20 and re.search("费用|预算|合计|金额|万元|运费|^其他$",a) is None]))
|
|
|
+ break
|
|
|
+ if not _find:
|
|
|
+ for table_products in list_table_products:
|
|
|
+ list_result.extend(table_products)
|
|
|
+ list_result = list(set([a for a in list_result if len(a)>1 and len(a)<30 and re.search("费用|预算|合计|金额|万元|运费",a) is None]))
|
|
|
return list_result
|
|
|
|
|
|
|
|
|
@@ -671,12 +770,20 @@ def get_correct_product(product,products):
|
|
|
def get_childs_text(childs,_product,products,is_begin=False,is_end=False):
|
|
|
_text = ""
|
|
|
|
|
|
+ end_next = False
|
|
|
for _child in childs:
|
|
|
|
|
|
child_text = _child.get("text")
|
|
|
|
|
|
+
|
|
|
if child_text.find(_product)>=0:
|
|
|
- is_begin = True
|
|
|
+ if not is_begin:
|
|
|
+ is_begin = True
|
|
|
+ if not end_next:
|
|
|
+ if _child["sentence_title"] is not None and isinstance(_child["title_next"],dict) and _child["title_next"]["sentence_title"] is not None:
|
|
|
+ end_next = True
|
|
|
+ end_title = _child["title_next"]
|
|
|
+ logger.debug("end_title %s "%end_title["text"])
|
|
|
|
|
|
logger.debug("%s-%s-%s"%("get_childs_text",child_text[:10],str(is_begin)))
|
|
|
|
|
|
@@ -684,13 +791,15 @@ def get_childs_text(childs,_product,products,is_begin=False,is_end=False):
|
|
|
if child_text.find(p)>=0 and is_similar(_product,p,90):
|
|
|
is_begin = True
|
|
|
|
|
|
- if child_text.find(_product)<0 and child_text.find(p)>=0 and not is_similar(_product,p,80):
|
|
|
+ if child_text.find(_product)<0 and not is_similar(_product,p,80) and (child_text.find(p)>=0 or _child["has_product"]):
|
|
|
if is_begin:
|
|
|
is_end = True
|
|
|
+ logger.debug("%s-%s-%s"%("get_childs_text end",child_text[:10],p))
|
|
|
break
|
|
|
if re.search(end_pattern,child_text) is not None:
|
|
|
if is_begin:
|
|
|
is_end = True
|
|
|
+ logger.debug("%s-%s-%s"%("get_childs_text end",child_text[:10],str(is_end)))
|
|
|
|
|
|
if is_begin and is_end:
|
|
|
break
|
|
|
@@ -699,45 +808,58 @@ def get_childs_text(childs,_product,products,is_begin=False,is_end=False):
|
|
|
_text += _child.get("text")+"\r\n"
|
|
|
childs2 = _child.get("child_title",[])
|
|
|
|
|
|
+
|
|
|
if len(childs2)>0:
|
|
|
for _child2 in childs2:
|
|
|
child_text,is_begin,is_end = get_childs_text([_child2],_product,products,is_begin)
|
|
|
- if is_begin and is_end:
|
|
|
- break
|
|
|
- else:
|
|
|
- if is_begin:
|
|
|
- _text += child_text
|
|
|
+ if is_begin:
|
|
|
+ _text += child_text
|
|
|
+ if is_end:
|
|
|
+ break
|
|
|
+
|
|
|
+ if end_next:
|
|
|
+ is_end = True
|
|
|
+
|
|
|
+ # logger.debug("%s-%s-%s"%("get_childs_text1",_text,str(is_begin)))
|
|
|
+ # logger.debug("%s-%s-%s"%("get_childs_text2",_text,str(is_begin)))
|
|
|
return _text,is_begin,is_end
|
|
|
|
|
|
def extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result,):
|
|
|
_data = list_data[_data_i]
|
|
|
childs = _data.get("child_title",[])
|
|
|
if len(childs)>0:
|
|
|
- child_text,_,_ = get_childs_text([parent_title],_product,products)
|
|
|
- logger.info("extract_parameters_by_tree child_text:%s"%child_text)
|
|
|
+ child_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
if len(child_text)>0:
|
|
|
+ logger.info("extract_type by_tree child_text:%s"%child_text)
|
|
|
list_result.append(child_text)
|
|
|
- return True
|
|
|
if parent_title is not None:
|
|
|
+ child_text,_,_ = get_childs_text([parent_title],_product,products)
|
|
|
+ if len(child_text)>0:
|
|
|
+ logger.info("extract_type by_tree child_text:%s"%child_text)
|
|
|
+ list_result.append(child_text)
|
|
|
+
|
|
|
childs = parent_title.get("child_title",[])
|
|
|
if len(childs)>0:
|
|
|
|
|
|
range_data = get_range_data_by_childs(list_data[_data_i:],childs)
|
|
|
p_text = ""
|
|
|
_find = False
|
|
|
+ end_id = id(_data["title_next"]) if isinstance(_data["sentence_title"],dict) and _data["title_next"] is not None and _data["title_next"]["sentence_title"] is not None else None
|
|
|
for pdata in range_data:
|
|
|
- ptype = _data["type"]
|
|
|
ptext = pdata["text"]
|
|
|
for p in products:
|
|
|
- if ptext.find(_product)<0 and ptext.find(p)>=0:
|
|
|
+ if ptext.find(_product)<0 and (ptext.find(p)>=0 or pdata["has_product"]):
|
|
|
_find = True
|
|
|
break
|
|
|
if re.search(end_pattern,ptext) is not None:
|
|
|
_find = True
|
|
|
if _find:
|
|
|
break
|
|
|
+ if id(pdata)==end_id:
|
|
|
+ break
|
|
|
p_text += ptext+"\r\n"
|
|
|
if len(p_text)>0:
|
|
|
+ logger.debug("extract_type by parent range_text:%s"%p_text)
|
|
|
list_result.append(p_text)
|
|
|
return True
|
|
|
return False
|
|
|
@@ -766,6 +888,7 @@ def get_table_pieces(_text,_product,products,list_result,_find):
|
|
|
list_trs.append(tr)
|
|
|
if len(list_trs)>0:
|
|
|
table_html = "<table>%s</table>"%("\r\n".join([str(a) for a in list_trs]))
|
|
|
+ logger.debug("extract_type table slices %s"%(table_html))
|
|
|
list_result.append(table_html)
|
|
|
|
|
|
def extract_parameters_by_table(_product,products,_param_pattern,list_data,_data_i,list_result):
|
|
|
@@ -778,8 +901,9 @@ def extract_parameters_by_table(_product,products,_param_pattern,list_data,_data
|
|
|
max_length = max([len(a) for a in list_table])
|
|
|
min_length = min([len(a) for a in list_table])
|
|
|
text_line_first = ",".join(a[0] for a in list_table[0])
|
|
|
- if min_length<max_length/2:
|
|
|
- return
|
|
|
+ if max_length>10:
|
|
|
+ if min_length<max_length/2:
|
|
|
+ return
|
|
|
last_data = list_data[_data_i-1]
|
|
|
_flag = False
|
|
|
if last_data["type"]=="sentence" and last_data["text"].find(_product)>=0:
|
|
|
@@ -789,8 +913,8 @@ def extract_parameters_by_table(_product,products,_param_pattern,list_data,_data
|
|
|
if re.search(_param_pattern,text_line_first) is not None and text_line_first.find(_product)>=0:
|
|
|
_flag = True
|
|
|
if _flag:
|
|
|
- logger.debug("extract_type add all table %s"%_text)
|
|
|
if len(products)==0:
|
|
|
+ logger.debug("extract_type whole table by param and product %s"%(_text))
|
|
|
list_result.append(_text)
|
|
|
else:
|
|
|
for p in products:
|
|
|
@@ -813,20 +937,99 @@ def extract_parameters_by_table(_product,products,_param_pattern,list_data,_data
|
|
|
for line in list_table:
|
|
|
for cell in line:
|
|
|
cell_text = cell[0]
|
|
|
- if len(cell_text)>50 and len(re.findall("\d+",cell_text))>10 and cell_text.find(_product)>=0:
|
|
|
- list_result.append(cell_text)
|
|
|
+ if len(cell_text)>50 and len(re.findall(meter_pattern,cell_text))>5 and cell_text.find(_product)>=0:
|
|
|
+ _f = True
|
|
|
+ for cell in line:
|
|
|
+ if not _f:
|
|
|
+ break
|
|
|
+ cell_text = cell[0]
|
|
|
+ for p in products:
|
|
|
+ if cell_text.find(p)>=0 and p!=_product:
|
|
|
+ _f = False
|
|
|
+ break
|
|
|
+ if _f:
|
|
|
+ logger.debug("extract_type param column %s"%(cell_text))
|
|
|
+ list_result.append(cell_text)
|
|
|
if len(cell_text)<len(_product)*10 and str(cell_text).find(_product)>=0:
|
|
|
for _index in list_head_index:
|
|
|
if _index>=len(line):
|
|
|
continue
|
|
|
_cell = line[_index]
|
|
|
if len(cell[0])>0:
|
|
|
- logger.info("%s-%s"%("add on table",_cell[0]))
|
|
|
+ logger.info("%s-%s"%("extract_type add on table text:",_cell[0]))
|
|
|
list_result.append(_cell[0])
|
|
|
if not _flag and (re.search(_param_pattern,_text) is not None or (parent_title is not None and re.search(_param_pattern,parent_title["text"]) is not None)) and _text.find(_product)>=0:
|
|
|
get_table_pieces(_text,_product,products,list_result,False)
|
|
|
|
|
|
|
|
|
+def extract_parameters_by_sentence(list_data,_data,_data_i,_product,products,list_result,is_project):
|
|
|
+ _text = _data["text"]
|
|
|
+ if _text.find(_product)>=0:
|
|
|
+ parent_title = _data.get("parent_title")
|
|
|
+ parent_text = ""
|
|
|
+ parent_parent_title = None
|
|
|
+ parent_parent_text = ""
|
|
|
+ parent_title_index = None
|
|
|
+ parent_parent_title_index = None
|
|
|
+ childs = get_childs([_data])
|
|
|
+
|
|
|
+ child_find = False
|
|
|
+ for c in childs:
|
|
|
+ if re.search(_param_pattern,c["text"]) is not None and len(c["text"])<30:
|
|
|
+ logger.debug("child text %s"%(c["text"]))
|
|
|
+ child_find = True
|
|
|
+ break
|
|
|
+
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
+ logger.debug("childs found extract_text %s %s"%(str(child_find),extract_text))
|
|
|
+ if child_find:
|
|
|
+ if len(extract_text)>0:
|
|
|
+ list_result.append(extract_text)
|
|
|
+ else:
|
|
|
+ limit_nums = len(_product)*2+5
|
|
|
+ if len(_product)<=3:
|
|
|
+ limit_nums += 6
|
|
|
+ if _text.find("数量")>=0:
|
|
|
+ limit_nums += 6
|
|
|
+ if len(_text)<=limit_nums and _data["sentence_title"] is not None:
|
|
|
+ if re.search(meter_pattern,extract_text) is not None:
|
|
|
+ list_result.append(extract_text)
|
|
|
+ elif len(re.findall(meter_pattern,extract_text))>2:
|
|
|
+ list_result.append(extract_text)
|
|
|
+
|
|
|
+ if parent_title is not None:
|
|
|
+ parent_text = parent_title.get("text","")
|
|
|
+ parent_parent_title = parent_title.get("parent_title")
|
|
|
+ parent_title_index = parent_title["title_index"]
|
|
|
+ if parent_parent_title is not None:
|
|
|
+ parent_parent_text = parent_parent_title.get("text","")
|
|
|
+ parent_parent_title_index = parent_parent_title["title_index"]
|
|
|
+
|
|
|
+ _suit = False
|
|
|
+ if re.search(_param_pattern,_text) is not None and len(_text)<50:
|
|
|
+ _suit = True
|
|
|
+ if re.search(_param_pattern,parent_text) is not None and len(parent_text)<50:
|
|
|
+ _suit = True
|
|
|
+ if re.search(_param_pattern,parent_parent_text) is not None and len(parent_parent_text)<50:
|
|
|
+ _suit = True
|
|
|
+ if _suit:
|
|
|
+ logger.debug("extract_type sentence %s"%("extract_parameters_by_tree"))
|
|
|
+ if not extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result):
|
|
|
+ logger.debug("extract_type sentence %s"%("extract_parameters_by_tree"))
|
|
|
+ extract_parameters_by_tree(_product,products,list_data,_data_i,parent_parent_title,list_result)
|
|
|
+
|
|
|
+ if re.search(_param_pattern,_text) is not None and len(_text)<50:
|
|
|
+ childs = _data["child_title"]
|
|
|
+ if len(childs)>0:
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
+ if len(extract_text)>0:
|
|
|
+ logger.debug("extract_type param-product %s"%(extract_text))
|
|
|
+ list_result.append(extract_text)
|
|
|
+ elif is_project:
|
|
|
+ extract_text,_,_ = get_childs_text([_data],_product,products,is_begin=True)
|
|
|
+ if len(extract_text)>0 and re.search(meter_pattern,extract_text) is not None:
|
|
|
+ logger.debug("extract_type sentence is_project param-product is product %s"%(extract_text))
|
|
|
+ list_result.append(extract_text)
|
|
|
|
|
|
def getBestProductText(list_result,_product,products):
|
|
|
list_result.sort(key=lambda x:len(re.findall(meter_pattern+"|"+'[:：;；]|\d+[%A-Za-z]+',BeautifulSoup(x,"html5lib").get_text())), reverse=True)
|
|
|
@@ -840,7 +1043,7 @@ def getBestProductText(list_result,_product,products):
|
|
|
_result = list_result[i]
|
|
|
_check = True
|
|
|
_result_text = BeautifulSoup(_result,"html5lib").get_text()
|
|
|
- _search = re.search("项目编号[:：]|项目名称[:：]|联合体投标",_result)
|
|
|
+ _search = re.search("项目编号[:：]|项目名称[:：]|联合体投标|开户银行",_result)
|
|
|
if _search is not None:
|
|
|
logger.debug("result%d error illegal text %s"%(i,str(_search)))
|
|
|
_check = False
|
|
|
@@ -849,13 +1052,31 @@ def getBestProductText(list_result,_product,products):
|
|
|
if _result_text.find(p)>0 and not (is_similar(_product,p,80) or p.find(_product)>=0 or _product.find(p)>=0):
|
|
|
logger.debug("result%d error product scoss %s"%(i,p))
|
|
|
_check = False
|
|
|
- if len(_result_text)<50:
|
|
|
+ if len(_result_text)<100:
|
|
|
if re.search(meter_pattern,_result_text) is None:
|
|
|
logger.debug("result%d error text min count"%(i))
|
|
|
_check = False
|
|
|
if len(_result_text)>5000:
|
|
|
- logger.debug("result%d error text max count"%(i))
|
|
|
+ if len(_result_text)>10000:
|
|
|
+ logger.debug("result%d error text max count"%(i))
|
|
|
+ _check = False
|
|
|
+ elif len(re.findall(meter_pattern,_result_text))<10:
|
|
|
+ logger.debug("result%d error text max count less meter"%(i))
|
|
|
+ _check = False
|
|
|
+
|
|
|
+ list_find = list(set(re.findall(meter_pattern,_result_text)))
|
|
|
+
|
|
|
+ not_list_find = list(set(re.findall(not_meter_pattern,_result_text)))
|
|
|
+ _count = len(list_find)-len(not_list_find)
|
|
|
+ has_num = False
|
|
|
+ for _find in list_find:
|
|
|
+ if re.search('[0-9a-zA-Z]',_find) is not None:
|
|
|
+ has_num = True
|
|
|
+ break
|
|
|
+ if not(_count>=2 and has_num or _count>=5):
|
|
|
+ logger.debug("result%d error match not enough"%(i))
|
|
|
_check = False
|
|
|
+
|
|
|
if _check:
|
|
|
return _result
|
|
|
|
|
|
@@ -868,6 +1089,11 @@ def extract_product_parameters(list_data,_product):
|
|
|
_product = get_correct_product(_product,products)
|
|
|
logger.debug("all products %s-%s"%(_product,str(products)))
|
|
|
is_project = False
|
|
|
+ if re.search("项目名称|采购项目",_product) is not None:
|
|
|
+ is_project = True
|
|
|
+
|
|
|
+ if len(products)==1 and is_similar(products[0],_product,90):
|
|
|
+ is_project = True
|
|
|
_find_count = 0
|
|
|
for _data_i in range(len(list_data)):
|
|
|
_data = list_data[_data_i]
|
|
|
@@ -876,84 +1102,23 @@ def extract_product_parameters(list_data,_product):
|
|
|
if _type=="sentence":
|
|
|
if _text.find(_product)>=0:
|
|
|
_find_count += 1
|
|
|
- if re.search("项目名称|采购项目",_text) is not None:
|
|
|
- is_project = True
|
|
|
- if re.search("项目名称|采购项目",_product) is not None:
|
|
|
+ if re.search("项目名称|采购项目",_text) is not None and re.search("等",_text) is not None:
|
|
|
is_project = True
|
|
|
- parent_title = _data.get("parent_title")
|
|
|
- parent_text = ""
|
|
|
- parent_parent_title = None
|
|
|
- parent_parent_text = ""
|
|
|
- parent_title_index = None
|
|
|
- parent_parent_title_index = None
|
|
|
- childs = get_childs([_data])
|
|
|
-
|
|
|
-
|
|
|
- child_find = False
|
|
|
- for c in childs:
|
|
|
- if re.search(_param_pattern,c["text"]) is not None and len(c["text"])<30:
|
|
|
- child_find = True
|
|
|
- break
|
|
|
-
|
|
|
- extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
- logger.debug("childs found extract_text %s"%extract_text)
|
|
|
- if child_find:
|
|
|
- if len(extract_text)>0:
|
|
|
- list_result.append(extract_text)
|
|
|
- else:
|
|
|
- if len(_text)<len(_product)+10 and _data["sentence_title"] is not None:
|
|
|
- if re.search(meter_pattern,extract_text) is not None:
|
|
|
- list_result.append(extract_text)
|
|
|
-
|
|
|
- if parent_title is not None:
|
|
|
- parent_text = parent_title.get("text","")
|
|
|
- parent_parent_title = parent_title.get("parent_title")
|
|
|
- parent_title_index = parent_title["title_index"]
|
|
|
- if parent_parent_title is not None:
|
|
|
- parent_parent_text = parent_parent_title.get("text","")
|
|
|
- parent_parent_title_index = parent_parent_title["title_index"]
|
|
|
-
|
|
|
- _suit = False
|
|
|
- if re.search(_param_pattern,_text) is not None and len(_text)<50:
|
|
|
- _suit = True
|
|
|
- if re.search(_param_pattern,parent_text) is not None and len(parent_text)<50:
|
|
|
- _suit = True
|
|
|
- if re.search(_param_pattern,parent_parent_text) is not None and len(parent_parent_text)<50:
|
|
|
- _suit = True
|
|
|
- if _suit:
|
|
|
- logger.debug("extract_type sentence %s"%("extract_parameters_by_tree"))
|
|
|
- if not extract_parameters_by_tree(_product,products,list_data,_data_i,parent_title,list_result):
|
|
|
- logger.debug("extract_type sentence %s"%("extract_parameters_by_tree"))
|
|
|
- extract_parameters_by_tree(_product,products,list_data,_data_i,parent_parent_title,list_result)
|
|
|
-
|
|
|
-
|
|
|
- if re.search(_param_pattern,_text) is not None and len(_text)<50:
|
|
|
- childs = _data["child_title"]
|
|
|
- if len(childs)>0:
|
|
|
- logger.debug("extract_type sentence %s"%("re.search(_param_pattern,_text) is not None and len(_text)<50:"))
|
|
|
- extract_text,_,_ = get_childs_text([_data],_product,products)
|
|
|
- if len(extract_text)>0:
|
|
|
- list_result.append(extract_text)
|
|
|
- elif is_project:
|
|
|
- logger.debug("extract_type sentence is_project")
|
|
|
- extract_text,_,_ = get_childs_text([_data],_product,products,is_begin=True)
|
|
|
- if len(extract_text)>0 and re.search(meter_pattern,extract_text) is not None:
|
|
|
- list_result.append(extract_text)
|
|
|
-
|
|
|
+ extract_parameters_by_sentence(list_data,_data,_data_i,_product,products,list_result,is_project)
|
|
|
|
|
|
elif _type=="table":
|
|
|
if _text.find(_product)>=0:
|
|
|
_find_count += 1
|
|
|
extract_parameters_by_table(_product,products,_param_pattern,list_data,_data_i,list_result)
|
|
|
|
|
|
-
|
|
|
- return getBestProductText(list_result,_product,products),_find_count
|
|
|
+ _text = getBestProductText(list_result,_product,products)
|
|
|
+ return _text,_find_count
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|
- filepath = "download/8679fef3a6fff56abcbdaccb1a190c80.html"
|
|
|
- _product = "移液器"
|
|
|
+ filepath = "download/4597dcc128bfabc7584d10590ae50656.html"
|
|
|
+ _product = "彩色多普勒超声诊断仪"
|
|
|
|
|
|
_html = open(filepath, "r", encoding="utf8").read()
|
|
|
|
|
|
@@ -965,5 +1130,6 @@ if __name__ == '__main__':
|
|
|
|
|
|
_text,_count = extract_product_parameters(list_data,_product)
|
|
|
logger.info("find count:%d"%(_count))
|
|
|
- logger.info("extract_text %s"%_text)
|
|
|
+ logger.info("extract_parameter_text::%s"%(_text))
|
|
|
+
|
|
|
|
