Preprocessing.py 109 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269
  1. # -*- coding: utf-8 -*-
  2. from bs4 import BeautifulSoup, Comment
  3. import copy
  4. import sys
  5. import os
  6. import time
  7. import codecs
  8. from BiddingKG.dl.ratio.re_ratio import extract_ratio
  9. sys.setrecursionlimit(1000000)
  10. sys.path.append(os.path.abspath("../.."))
  11. sys.path.append(os.path.abspath(".."))
  12. from BiddingKG.dl.common.Utils import *
  13. from BiddingKG.dl.interface.Entitys import *
  14. from BiddingKG.dl.interface.predictor import getPredictor
  15. from BiddingKG.dl.common.nerUtils import *
  16. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  17. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  18. from BiddingKG.dl.relation_extraction.re_email import extract_email
  19. from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
  20. from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
  21. from BiddingKG.dl.entityLink.entityLink import *
  22. #
  23. def tableToText(soup):
  24. '''
  25. @param:
  26. soup:网页html的soup
  27. @return:处理完表格信息的网页text
  28. '''
  29. def getTrs(tbody):
  30. #获取所有的tr
  31. trs = []
  32. objs = tbody.find_all(recursive=False)
  33. for obj in objs:
  34. if obj.name=="tr":
  35. trs.append(obj)
  36. if obj.name=="tbody":
  37. for tr in obj.find_all("tr",recursive=False):
  38. trs.append(tr)
  39. return trs
  40. def fixSpan(tbody):
  41. # 处理colspan, rowspan信息补全问题
  42. #trs = tbody.findChildren('tr', recursive=False)
  43. trs = getTrs(tbody)
  44. ths_len = 0
  45. ths = list()
  46. trs_set = set()
  47. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  48. # 遍历每一个tr
  49. for indtr, tr in enumerate(trs):
  50. ths_tmp = tr.findChildren('th', recursive=False)
  51. #不补全含有表格的tr
  52. if len(tr.findChildren('table'))>0:
  53. continue
  54. if len(ths_tmp) > 0:
  55. ths_len = ths_len + len(ths_tmp)
  56. for th in ths_tmp:
  57. ths.append(th)
  58. trs_set.add(tr)
  59. # 遍历每行中的element
  60. tds = tr.findChildren(recursive=False)
  61. for indtd, td in enumerate(tds):
  62. # 若有colspan 则补全同一行下一个位置
  63. if 'colspan' in td.attrs:
  64. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  65. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  66. if col<100 and len(td.get_text())<1000:
  67. td['colspan'] = 1
  68. for i in range(1, col, 1):
  69. td.insert_after(copy.copy(td))
  70. for indtr, tr in enumerate(trs):
  71. ths_tmp = tr.findChildren('th', recursive=False)
  72. #不补全含有表格的tr
  73. if len(tr.findChildren('table'))>0:
  74. continue
  75. if len(ths_tmp) > 0:
  76. ths_len = ths_len + len(ths_tmp)
  77. for th in ths_tmp:
  78. ths.append(th)
  79. trs_set.add(tr)
  80. # 遍历每行中的element
  81. tds = tr.findChildren(recursive=False)
  82. for indtd, td in enumerate(tds):
  83. # 若有rowspan 则补全下一行同样位置
  84. if 'rowspan' in td.attrs:
  85. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  86. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  87. td['rowspan'] = 1
  88. for i in range(1, row, 1):
  89. # 获取下一行的所有td, 在对应的位置插入
  90. if indtr+i<len(trs):
  91. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  92. if len(tds1) >= (indtd) and len(tds1)>0:
  93. if indtd > 0:
  94. tds1[indtd - 1].insert_after(copy.copy(td))
  95. else:
  96. tds1[0].insert_before(copy.copy(td))
  97. elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全
  98. tds1[indtd-2].insert_after(copy.copy(td))
  99. def getTable(tbody):
  100. #trs = tbody.findChildren('tr', recursive=False)
  101. trs = getTrs(tbody)
  102. inner_table = []
  103. for tr in trs:
  104. tr_line = []
  105. tds = tr.findChildren(['td','th'], recursive=False)
  106. if len(tds)==0:
  107. tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
  108. for td in tds:
  109. tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
  110. #tr_line.append([td.get_text(),0])
  111. inner_table.append(tr_line)
  112. return inner_table
  113. #处理表格不对齐的问题
  114. def fixTable(inner_table,fix_value="~~"):
  115. maxWidth = 0
  116. for item in inner_table:
  117. if len(item)>maxWidth:
  118. maxWidth = len(item)
  119. for i in range(len(inner_table)):
  120. if len(inner_table[i])<maxWidth:
  121. for j in range(maxWidth-len(inner_table[i])):
  122. inner_table[i].append([fix_value,0])
  123. return inner_table
  124. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  125. height = len(inner_table)
  126. width = len(inner_table[0])
  127. for i in range(height):
  128. point = ""
  129. for j in range(width):
  130. if inner_table[i][j][0]==point and point!="":
  131. inner_table[i][j][0] = pad_row
  132. else:
  133. if inner_table[i][j][0] not in [pad_row,pad_col]:
  134. point = inner_table[i][j][0]
  135. for j in range(width):
  136. point = ""
  137. for i in range(height):
  138. if inner_table[i][j][0]==point and point!="":
  139. inner_table[i][j][0] = pad_col
  140. else:
  141. if inner_table[i][j][0] not in [pad_row,pad_col]:
  142. point = inner_table[i][j][0]
  143. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  144. height = len(inner_table)
  145. width = len(inner_table[0])
  146. for i in range(height):
  147. for j in range(width):
  148. if inner_table[i][j][0]==pad_row:
  149. inner_table[i][j][0] = inner_table[i][j-1][0]
  150. inner_table[i][j][1] = inner_table[i][j-1][1]
  151. if inner_table[i][j][0]==pad_col:
  152. inner_table[i][j][0] = inner_table[i-1][j][0]
  153. inner_table[i][j][1] = inner_table[i-1][j][1]
  154. def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
  155. '''
  156. @summary: 修复表头识别,将明显错误的进行修正
  157. '''
  158. def repairNeeded(line):
  159. first_1 = -1
  160. last_1 = -1
  161. first_0 = -1
  162. last_0 = -1
  163. count_1 = 0
  164. count_0 = 0
  165. for i in range(len(line)):
  166. if line[i][0]==fix_value:
  167. continue
  168. if line[i][1]==1:
  169. if first_1==-1:
  170. first_1 = i
  171. last_1 = i
  172. count_1 += 1
  173. if line[i][1]==0:
  174. if first_0 == -1:
  175. first_0 = i
  176. last_0 = i
  177. count_0 += 1
  178. if first_1 ==-1 or last_0 == -1:
  179. return False
  180. #异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  181. if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
  182. return True
  183. return False
  184. def getsimilarity(line,line1):
  185. same_count = 0
  186. for item,item1 in zip(line,line1):
  187. if item[1]==item1[1]:
  188. same_count += 1
  189. return same_count/len(line)
  190. def selfrepair(inner_table,index,dye_set,key_set):
  191. '''
  192. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  193. '''
  194. #print("B",inner_table[index])
  195. min_presure = 3
  196. list_dye = []
  197. first = None
  198. count = 0
  199. temp_set = set()
  200. _index = 0
  201. for item in inner_table[index]:
  202. if first is None:
  203. first = item[1]
  204. if item[0] not in temp_set:
  205. count += 1
  206. temp_set.add(item[0])
  207. else:
  208. if first == item[1]:
  209. if item[0] not in temp_set:
  210. temp_set.add(item[0])
  211. count += 1
  212. else:
  213. list_dye.append([first,count,_index])
  214. first = item[1]
  215. temp_set.add(item[0])
  216. count = 1
  217. _index += 1
  218. list_dye.append([first,count,_index])
  219. if len(list_dye)>1:
  220. begin = 0
  221. end = 0
  222. for i in range(len(list_dye)):
  223. end = list_dye[i][2]
  224. dye_flag = False
  225. #首尾要求压力减一
  226. if i==0:
  227. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  228. dye_flag = True
  229. dye_type = list_dye[i+1][0]
  230. elif i==len(list_dye)-1:
  231. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  232. dye_flag = True
  233. dye_type = list_dye[i-1][0]
  234. else:
  235. if list_dye[i][1]>1:
  236. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  237. dye_flag = True
  238. dye_type = list_dye[i+1][0]
  239. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  240. dye_flag = True
  241. dye_type = list_dye[i-1][0]
  242. else:
  243. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  244. dye_flag = True
  245. dye_type = list_dye[i+1][0]
  246. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  247. dye_flag = True
  248. dye_type = list_dye[i-1][0]
  249. if dye_flag:
  250. for h in range(begin,end):
  251. inner_table[index][h][1] = dye_type
  252. dye_set.add((inner_table[index][h][0],dye_type))
  253. key_set.add(inner_table[index][h][0])
  254. begin = end
  255. #print("E",inner_table[index])
  256. def otherrepair(inner_table,index,dye_set,key_set):
  257. list_provide_repair = []
  258. if index==0 and len(inner_table)>1:
  259. list_provide_repair.append(index+1)
  260. elif index==len(inner_table)-1:
  261. list_provide_repair.append(index-1)
  262. else:
  263. list_provide_repair.append(index+1)
  264. list_provide_repair.append(index-1)
  265. for provide_index in list_provide_repair:
  266. if not repairNeeded(inner_table[provide_index]):
  267. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  268. if same_prob>=0.8:
  269. for i in range(len(inner_table[provide_index])):
  270. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  271. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  272. key_set.add(inner_table[index][i][0])
  273. inner_table[index][i][1] = inner_table[provide_index][i][1]
  274. elif same_prob<=0.2:
  275. for i in range(len(inner_table[provide_index])):
  276. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  277. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  278. key_set.add(inner_table[index][i][0])
  279. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  280. len_dye_set = len(dye_set)
  281. height = len(inner_table)
  282. for i in range(height):
  283. if repairNeeded(inner_table[i]):
  284. selfrepair(inner_table,i,dye_set,key_set)
  285. #otherrepair(inner_table,i,dye_set,key_set)
  286. for h in range(len(inner_table)):
  287. for w in range(len(inner_table[0])):
  288. if inner_table[h][w][0] in key_set:
  289. for item in dye_set:
  290. if inner_table[h][w][0]==item[0]:
  291. inner_table[h][w][1] = item[1]
  292. #如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  293. if len(dye_set)!=len(key_set):
  294. for i in range(height):
  295. if repairNeeded(inner_table[i]):
  296. selfrepair(inner_table,i,dye_set,key_set)
  297. #otherrepair(inner_table,i,dye_set,key_set)
  298. return
  299. if len(dye_set)==len_dye_set:
  300. '''
  301. for i in range(height):
  302. if repairNeeded(inner_table[i]):
  303. otherrepair(inner_table,i,dye_set,key_set)
  304. '''
  305. return
  306. repairTable(inner_table, dye_set, key_set)
  307. def sliceTable(inner_table,fix_value="~~"):
  308. #进行分块
  309. height = len(inner_table)
  310. width = len(inner_table[0])
  311. head_list = []
  312. head_list.append(0)
  313. last_head = None
  314. last_is_same_value = False
  315. for h in range(height):
  316. is_all_key = True#是否是全表头行
  317. is_all_value = True#是否是全属性值
  318. is_same_with_lastHead = True#和上一行的结构是否相同
  319. is_same_value=True#一行的item都一样
  320. #is_same_first_item = True#与上一行的第一项是否相同
  321. same_value = inner_table[h][0][0]
  322. for w in range(width):
  323. if last_head is not None:
  324. if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
  325. is_all_key = False
  326. if inner_table[h][w][0]==1:
  327. is_all_value = False
  328. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  329. is_same_with_lastHead = False
  330. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  331. is_same_value = False
  332. else:
  333. if re.search("\d+",same_value) is not None:
  334. is_same_value = False
  335. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  336. is_same_first_item = False
  337. last_head = h
  338. if last_is_same_value:
  339. last_is_same_value = is_same_value
  340. continue
  341. if is_same_value:
  342. head_list.append(h)
  343. last_is_same_value = is_same_value
  344. continue
  345. if not is_all_key:
  346. if not is_same_with_lastHead:
  347. head_list.append(h)
  348. head_list.append(height)
  349. return head_list
  350. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  351. set_item = set()
  352. height = len(inner_table)
  353. width = len(inner_table[0])
  354. for i in range(height):
  355. for j in range(width):
  356. item = inner_table[i][j][0]
  357. set_item.add(item)
  358. list_item = list(set_item)
  359. x = []
  360. for item in list_item:
  361. x.append(getPredictor("form").encode(item))
  362. predict_y = getPredictor("form").predict(np.array(x),type="item")
  363. _dict = dict()
  364. for item,values in zip(list_item,list(predict_y)):
  365. _dict[item] = values[1]
  366. # print("##",item,values)
  367. #print(_dict)
  368. for i in range(height):
  369. for j in range(width):
  370. item = inner_table[i][j][0]
  371. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  372. # print("=====")
  373. # for item in inner_table:
  374. # print(item)
  375. # print("======")
  376. repairTable(inner_table)
  377. head_list = sliceTable(inner_table)
  378. return inner_table,head_list
  379. def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  380. data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
  381. predict_y = getPredictor("form").getModel("context").predict(data_x)
  382. for _position,_y in zip(data_position,predict_y):
  383. _w = _position[0]
  384. _h = _position[1]
  385. if _y[1]>prob_min:
  386. inner_table[_h][_w][1] = 1
  387. else:
  388. inner_table[_h][_w][1] = 0
  389. _item = inner_table[_h][_w][0]
  390. if re.search(pat_head,_item) is not None and len(_item)<8:
  391. inner_table[_h][_w][1] = 1
  392. # print("=====")
  393. # for item in inner_table:
  394. # print(item)
  395. # print("======")
  396. height = len(inner_table)
  397. width = len(inner_table[0])
  398. for i in range(height):
  399. for j in range(width):
  400. if re.search("[::]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
  401. inner_table[i][j][1] = 1
  402. repairTable(inner_table)
  403. head_list = sliceTable(inner_table)
  404. # print("inner_table:",inner_table)
  405. return inner_table,head_list
  406. #设置表头
  407. def setHead_inline(inner_table,prob_min=0.64):
  408. pad_row = "@@"
  409. pad_col = "##"
  410. removePadding(inner_table, pad_row, pad_col)
  411. pad_pattern = re.compile(pad_row+"|"+pad_col)
  412. height = len(inner_table)
  413. width = len(inner_table[0])
  414. head_list = []
  415. head_list.append(0)
  416. #行表头
  417. is_head_last = False
  418. for i in range(height):
  419. is_head = False
  420. is_long_value = False
  421. #判断是否是全padding值
  422. is_same_value = True
  423. same_value = inner_table[i][0][0]
  424. for j in range(width):
  425. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  426. is_same_value = False
  427. break
  428. #predict is head or not with model
  429. temp_item = ""
  430. for j in range(width):
  431. temp_item += inner_table[i][j][0]+"|"
  432. temp_item = re.sub(pad_pattern,"",temp_item)
  433. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  434. if form_prob is not None:
  435. if form_prob[0][1]>prob_min:
  436. is_head = True
  437. else:
  438. is_head = False
  439. #print(temp_item,form_prob)
  440. if len(inner_table[i][0][0])>40:
  441. is_long_value = True
  442. if is_head or is_long_value or is_same_value:
  443. #不把连续表头分开
  444. if not is_head_last:
  445. head_list.append(i)
  446. if is_long_value or is_same_value:
  447. head_list.append(i+1)
  448. if is_head:
  449. for j in range(width):
  450. inner_table[i][j][1] = 1
  451. is_head_last = is_head
  452. head_list.append(height)
  453. #列表头
  454. for i in range(len(head_list)-1):
  455. head_begin = head_list[i]
  456. head_end = head_list[i+1]
  457. #最后一列不设置为列表头
  458. for i in range(width-1):
  459. is_head = False
  460. #predict is head or not with model
  461. temp_item = ""
  462. for j in range(head_begin,head_end):
  463. temp_item += inner_table[j][i][0]+"|"
  464. temp_item = re.sub(pad_pattern,"",temp_item)
  465. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  466. if form_prob is not None:
  467. if form_prob[0][1]>prob_min:
  468. is_head = True
  469. else:
  470. is_head = False
  471. if is_head:
  472. for j in range(head_begin,head_end):
  473. inner_table[j][i][1] = 2
  474. addPadding(inner_table, pad_row, pad_col)
  475. return inner_table,head_list
  476. #设置表头
  477. def setHead_withRule(inner_table,pattern,pat_value,count):
  478. height = len(inner_table)
  479. width = len(inner_table[0])
  480. head_list = []
  481. head_list.append(0)
  482. #行表头
  483. is_head_last = False
  484. for i in range(height):
  485. set_match = set()
  486. is_head = False
  487. is_long_value = False
  488. is_same_value = True
  489. same_value = inner_table[i][0][0]
  490. for j in range(width):
  491. if inner_table[i][j][0]!=same_value:
  492. is_same_value = False
  493. break
  494. for j in range(width):
  495. if re.search(pat_value,inner_table[i][j][0]) is not None:
  496. is_head = False
  497. break
  498. str_find = re.findall(pattern,inner_table[i][j][0])
  499. if len(str_find)>0:
  500. set_match.add(inner_table[i][j][0])
  501. if len(set_match)>=count:
  502. is_head = True
  503. if len(inner_table[i][0][0])>40:
  504. is_long_value = True
  505. if is_head or is_long_value or is_same_value:
  506. if not is_head_last:
  507. head_list.append(i)
  508. if is_head:
  509. for j in range(width):
  510. inner_table[i][j][1] = 1
  511. is_head_last = is_head
  512. head_list.append(height)
  513. #列表头
  514. for i in range(len(head_list)-1):
  515. head_begin = head_list[i]
  516. head_end = head_list[i+1]
  517. #最后一列不设置为列表头
  518. for i in range(width-1):
  519. set_match = set()
  520. is_head = False
  521. for j in range(head_begin,head_end):
  522. if re.search(pat_value,inner_table[j][i][0]) is not None:
  523. is_head = False
  524. break
  525. str_find = re.findall(pattern,inner_table[j][i][0])
  526. if len(str_find)>0:
  527. set_match.add(inner_table[j][i][0])
  528. if len(set_match)>=count:
  529. is_head = True
  530. if is_head:
  531. for j in range(head_begin,head_end):
  532. inner_table[j][i][1] = 2
  533. return inner_table,head_list
  534. #取得表格的处理方向
  535. def getDirect(inner_table,begin,end):
  536. '''
  537. column_head = set()
  538. row_head = set()
  539. widths = len(inner_table[0])
  540. for height in range(begin,end):
  541. for width in range(widths):
  542. if inner_table[height][width][1] ==1:
  543. row_head.add(height)
  544. if inner_table[height][width][1] ==2:
  545. column_head.add(width)
  546. company_pattern = re.compile("公司")
  547. if 0 in column_head and begin not in row_head:
  548. return "column"
  549. if 0 in column_head and begin in row_head:
  550. for height in range(begin,end):
  551. count = 0
  552. count_flag = True
  553. for width_index in range(width):
  554. if inner_table[height][width_index][1]==0:
  555. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  556. count += 1
  557. else:
  558. count_flag = False
  559. if count_flag and count>=2:
  560. return "column"
  561. return "row"
  562. '''
  563. count_row_keys = 0
  564. count_column_keys = 0
  565. width = len(inner_table[0])
  566. if begin<end:
  567. for w in range(len(inner_table[begin])):
  568. if inner_table[begin][w][1]!=0:
  569. count_row_keys += 1
  570. for h in range(begin,end):
  571. if inner_table[h][0][1]!=0:
  572. count_column_keys += 1
  573. company_pattern = re.compile("有限(责任)?公司")
  574. for height in range(begin,end):
  575. count_set = set()
  576. count_flag = True
  577. for width_index in range(width):
  578. if inner_table[height][width_index][1]==0:
  579. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  580. count_set.add(inner_table[height][width_index][0])
  581. else:
  582. count_flag = False
  583. if count_flag and len(count_set)>=2:
  584. return "column"
  585. if count_column_keys>count_row_keys:
  586. return "column"
  587. return "row"
  588. #根据表格处理方向生成句子,
  589. def getTableText(inner_table,head_list,key_direct=False):
  590. # packPattern = "(标包|[标包][号段名])"
  591. packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  592. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)" # 2020/11/23 大网站规则,添加序号为排序
  593. entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))"
  594. moneyPattern = "([中投]标|报价)(金额|价)"
  595. height = len(inner_table)
  596. width = len(inner_table[0])
  597. text = ""
  598. for head_i in range(len(head_list)-1):
  599. head_begin = head_list[head_i]
  600. head_end = head_list[head_i+1]
  601. direct = getDirect(inner_table, head_begin, head_end)
  602. #若只有一行,则直接按行读取
  603. if head_end-head_begin==1:
  604. text_line = ""
  605. for i in range(head_begin,head_end):
  606. for w in range(len(inner_table[i])):
  607. if inner_table[i][w][1]==1:
  608. _punctuation = ":"
  609. else:
  610. _punctuation = "," #2021/12/15 统一为中文标点,避免 206893924 国际F座1108,1,009,197.49元
  611. if w>0:
  612. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  613. text_line += inner_table[i][w][0]+_punctuation
  614. else:
  615. text_line += inner_table[i][w][0]+_punctuation
  616. text_line = text_line+"。" if text_line!="" else text_line
  617. text += text_line
  618. else:
  619. #构建一个共现矩阵
  620. table_occurence = []
  621. for i in range(head_begin,head_end):
  622. line_oc = []
  623. for j in range(width):
  624. cell = inner_table[i][j]
  625. line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
  626. table_occurence.append(line_oc)
  627. occu_height = len(table_occurence)
  628. occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
  629. #为每个属性值寻找表头
  630. for i in range(occu_height):
  631. for j in range(occu_width):
  632. cell = table_occurence[i][j]
  633. #是属性值
  634. if cell["type"]==0 and cell["text"]!="":
  635. left_head = ""
  636. top_head = ""
  637. find_flag = False
  638. temp_head = ""
  639. for loop_i in range(1,i+1):
  640. if not key_direct:
  641. key_values = [1,2]
  642. else:
  643. key_values = [1]
  644. if table_occurence[i-loop_i][j]["type"] in key_values:
  645. if find_flag:
  646. if table_occurence[i-loop_i][j]["text"]!=temp_head:
  647. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  648. else:
  649. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  650. find_flag = True
  651. temp_head = table_occurence[i-loop_i][j]["text"]
  652. table_occurence[i-loop_i][j]["occu_count"] += 1
  653. else:
  654. #找到表头后遇到属性值就返回
  655. if find_flag:
  656. break
  657. cell["top_head"] += top_head
  658. find_flag = False
  659. temp_head = ""
  660. for loop_j in range(1,j+1):
  661. if not key_direct:
  662. key_values = [1,2]
  663. else:
  664. key_values = [2]
  665. if table_occurence[i][j-loop_j]["type"] in key_values:
  666. if find_flag:
  667. if table_occurence[i][j-loop_j]["text"]!=temp_head:
  668. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  669. else:
  670. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  671. find_flag = True
  672. temp_head = table_occurence[i][j-loop_j]["text"]
  673. table_occurence[i][j-loop_j]["occu_count"] += 1
  674. else:
  675. if find_flag:
  676. break
  677. cell["left_head"] += left_head
  678. if direct=="row":
  679. for i in range(occu_height):
  680. pack_text = ""
  681. rank_text = ""
  682. entity_text = ""
  683. text_line = ""
  684. money_text = ""
  685. #在同一句话中重复的可以去掉
  686. text_set = set()
  687. for j in range(width):
  688. cell = table_occurence[i][j]
  689. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  690. cell = table_occurence[i][j]
  691. head = (cell["top_head"]+":") if len(cell["top_head"])>0 else ""
  692. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  693. head = cell["left_head"] + head
  694. else:
  695. head += cell["left_head"]
  696. if str(head+cell["text"]) in text_set:
  697. continue
  698. if re.search(packPattern,head) is not None:
  699. pack_text += head+cell["text"]+","
  700. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  701. #排名替换为同一种表达
  702. rank_text += head+cell["text"]+","
  703. #print(rank_text)
  704. elif re.search(entityPattern,head) is not None:
  705. entity_text += head+cell["text"]+","
  706. #print(entity_text)
  707. else:
  708. if re.search(moneyPattern,head) is not None and entity_text!="":
  709. money_text += head+cell["text"]+","
  710. else:
  711. text_line += head+cell["text"]+","
  712. text_set.add(str(head+cell["text"]))
  713. text += pack_text+rank_text+entity_text+money_text+text_line
  714. text = text[:-1]+"。" if len(text)>0 else text
  715. else:
  716. for j in range(occu_width):
  717. pack_text = ""
  718. rank_text = ""
  719. entity_text = ""
  720. text_line = ""
  721. text_set = set()
  722. for i in range(occu_height):
  723. cell = table_occurence[i][j]
  724. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  725. cell = table_occurence[i][j]
  726. head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
  727. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  728. head = cell["top_head"] + head
  729. else:
  730. head += cell["top_head"]
  731. if str(head+cell["text"]) in text_set:
  732. continue
  733. if re.search(packPattern,head) is not None:
  734. pack_text += head+cell["text"]+","
  735. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  736. #排名替换为同一种表达
  737. rank_text += head+cell["text"]+","
  738. #print(rank_text)
  739. elif re.search(entityPattern,head) is not None and \
  740. re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
  741. entity_text += head+cell["text"]+","
  742. #print(entity_text)
  743. else:
  744. text_line += head+cell["text"]+","
  745. text_set.add(str(head+cell["text"]))
  746. text += pack_text+rank_text+entity_text+text_line
  747. text = text[:-1]+"。" if len(text)>0 else text
  748. # if direct=="row":
  749. # for i in range(head_begin,head_end):
  750. # pack_text = ""
  751. # rank_text = ""
  752. # entity_text = ""
  753. # text_line = ""
  754. # #在同一句话中重复的可以去掉
  755. # text_set = set()
  756. # for j in range(width):
  757. # cell = inner_table[i][j]
  758. # #是属性值
  759. # if cell[1]==0 and cell[0]!="":
  760. # head = ""
  761. #
  762. # find_flag = False
  763. # temp_head = ""
  764. # for loop_i in range(0,i+1-head_begin):
  765. # if not key_direct:
  766. # key_values = [1,2]
  767. # else:
  768. # key_values = [1]
  769. # if inner_table[i-loop_i][j][1] in key_values:
  770. # if find_flag:
  771. # if inner_table[i-loop_i][j][0]!=temp_head:
  772. # head = inner_table[i-loop_i][j][0]+":"+head
  773. # else:
  774. # head = inner_table[i-loop_i][j][0]+":"+head
  775. # find_flag = True
  776. # temp_head = inner_table[i-loop_i][j][0]
  777. # else:
  778. # #找到表头后遇到属性值就返回
  779. # if find_flag:
  780. # break
  781. #
  782. # find_flag = False
  783. # temp_head = ""
  784. #
  785. #
  786. #
  787. # for loop_j in range(1,j+1):
  788. # if not key_direct:
  789. # key_values = [1,2]
  790. # else:
  791. # key_values = [2]
  792. # if inner_table[i][j-loop_j][1] in key_values:
  793. # if find_flag:
  794. # if inner_table[i][j-loop_j][0]!=temp_head:
  795. # head = inner_table[i][j-loop_j][0]+":"+head
  796. # else:
  797. # head = inner_table[i][j-loop_j][0]+":"+head
  798. # find_flag = True
  799. # temp_head = inner_table[i][j-loop_j][0]
  800. # else:
  801. # if find_flag:
  802. # break
  803. #
  804. # if str(head+inner_table[i][j][0]) in text_set:
  805. # continue
  806. # if re.search(packPattern,head) is not None:
  807. # pack_text += head+inner_table[i][j][0]+","
  808. # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  809. # #排名替换为同一种表达
  810. # rank_text += head+inner_table[i][j][0]+","
  811. # #print(rank_text)
  812. # elif re.search(entityPattern,head) is not None:
  813. # entity_text += head+inner_table[i][j][0]+","
  814. # #print(entity_text)
  815. # else:
  816. # text_line += head+inner_table[i][j][0]+","
  817. # text_set.add(str(head+inner_table[i][j][0]))
  818. # text += pack_text+rank_text+entity_text+text_line
  819. # text = text[:-1]+"。" if len(text)>0 else text
  820. # else:
  821. # for j in range(width):
  822. #
  823. # rank_text = ""
  824. # entity_text = ""
  825. # text_line = ""
  826. # text_set = set()
  827. # for i in range(head_begin,head_end):
  828. # cell = inner_table[i][j]
  829. # #是属性值
  830. # if cell[1]==0 and cell[0]!="":
  831. # find_flag = False
  832. # head = ""
  833. # temp_head = ""
  834. #
  835. # for loop_j in range(1,j+1):
  836. # if not key_direct:
  837. # key_values = [1,2]
  838. # else:
  839. # key_values = [2]
  840. # if inner_table[i][j-loop_j][1] in key_values:
  841. # if find_flag:
  842. # if inner_table[i][j-loop_j][0]!=temp_head:
  843. # head = inner_table[i][j-loop_j][0]+":"+head
  844. # else:
  845. # head = inner_table[i][j-loop_j][0]+":"+head
  846. # find_flag = True
  847. # temp_head = inner_table[i][j-loop_j][0]
  848. # else:
  849. # if find_flag:
  850. # break
  851. # find_flag = False
  852. # temp_head = ""
  853. # for loop_i in range(0,i+1-head_begin):
  854. # if not key_direct:
  855. # key_values = [1,2]
  856. # else:
  857. # key_values = [1]
  858. # if inner_table[i-loop_i][j][1] in key_values:
  859. # if find_flag:
  860. # if inner_table[i-loop_i][j][0]!=temp_head:
  861. # head = inner_table[i-loop_i][j][0]+":"+head
  862. # else:
  863. # head = inner_table[i-loop_i][j][0]+":"+head
  864. # find_flag = True
  865. # temp_head = inner_table[i-loop_i][j][0]
  866. # else:
  867. # if find_flag:
  868. # break
  869. # if str(head+inner_table[i][j][0]) in text_set:
  870. # continue
  871. # if re.search(rankPattern,head) is not None:
  872. # rank_text += head+inner_table[i][j][0]+","
  873. # #print(rank_text)
  874. # elif re.search(entityPattern,head) is not None:
  875. # entity_text += head+inner_table[i][j][0]+","
  876. # #print(entity_text)
  877. # else:
  878. # text_line += head+inner_table[i][j][0]+","
  879. # text_set.add(str(head+inner_table[i][j][0]))
  880. # text += rank_text+entity_text+text_line
  881. # text = text[:-1]+"。" if len(text)>0 else text
  882. return text
  883. def removeFix(inner_table,fix_value="~~"):
  884. height = len(inner_table)
  885. width = len(inner_table[0])
  886. for h in range(height):
  887. for w in range(width):
  888. if inner_table[h][w][0]==fix_value:
  889. inner_table[h][w][0] = ""
  890. def trunTable(tbody):
  891. fixSpan(tbody)
  892. inner_table = getTable(tbody)
  893. inner_table = fixTable(inner_table)
  894. if len(inner_table)>0 and len(inner_table[0])>0:
  895. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  896. #inner_table,head_list = setHead_inline(inner_table)
  897. # inner_table,head_list = setHead_initem(inner_table,pat_head)
  898. inner_table,head_list = setHead_incontext(inner_table,pat_head)
  899. # print(inner_table)
  900. # for begin in range(len(head_list[:-1])):
  901. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  902. # print(item)
  903. # print("====")
  904. removeFix(inner_table)
  905. # print("----")
  906. # print(head_list)
  907. # for item in inner_table:
  908. # print(item)
  909. tbody.string = getTableText(inner_table,head_list)
  910. #print(tbody.string)
  911. tbody.name = "turntable"
  912. return inner_table
  913. return None
  914. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$')
  915. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  916. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  917. list_innerTable = []
  918. tbodies = soup.find_all('table')
  919. # 遍历表格中的每个tbody
  920. #逆序处理嵌套表格
  921. for tbody_index in range(1,len(tbodies)+1):
  922. tbody = tbodies[len(tbodies)-tbody_index]
  923. inner_table = trunTable(tbody)
  924. list_innerTable.append(inner_table)
  925. tbodies = soup.find_all('tbody')
  926. # 遍历表格中的每个tbody
  927. #逆序处理嵌套表格
  928. for tbody_index in range(1,len(tbodies)+1):
  929. tbody = tbodies[len(tbodies)-tbody_index]
  930. inner_table = trunTable(tbody)
  931. list_innerTable.append(inner_table)
  932. return soup
  933. # return list_innerTable
  934. #数据清洗
  935. def segment(soup,final=True):
  936. # print("==")
  937. # print(soup)
  938. # print("====")
  939. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  940. subspaceList = ["td",'a',"span","p"]
  941. if soup.name in subspaceList:
  942. #判断有值叶子节点数
  943. _count = 0
  944. for child in soup.find_all(recursive=True):
  945. if child.get_text().strip()!="" and len(child.find_all())==0:
  946. _count += 1
  947. if _count<=1:
  948. text = soup.get_text()
  949. # 2020/11/24 大网站规则添加
  950. if 'title' in soup.attrs:
  951. if '...' in soup.get_text() and (soup.get_text()[:-3]).strip() in soup.attrs['title']:
  952. text = soup.attrs['title']
  953. _list = []
  954. for x in re.split("\s+",text):
  955. if x.strip()!="":
  956. _list.append(len(x))
  957. if len(_list)>0:
  958. _minLength = min(_list)
  959. if _minLength>2:
  960. _substr = ","
  961. else:
  962. _substr = ""
  963. else:
  964. _substr = ""
  965. text = text.replace("\r\n",",").replace("\n",",")
  966. text = re.sub("\s+",_substr,text)
  967. # text = re.sub("\s+","##space##",text)
  968. return text
  969. segList = ["title"]
  970. commaList = ["div","br","td","p"]
  971. #commaList = []
  972. spaceList = ["span"]
  973. tbodies = soup.find_all('tbody')
  974. if len(tbodies) == 0:
  975. tbodies = soup.find_all('table')
  976. # 递归遍历所有节点,插入符号
  977. for child in soup.find_all(recursive=True):
  978. if child.name in segList:
  979. child.insert_after("。")
  980. if child.name in commaList:
  981. child.insert_after(",")
  982. # if child.name in subspaceList:
  983. # child.insert_before("#subs"+str(child.name)+"#")
  984. # child.insert_after("#sube"+str(child.name)+"#")
  985. # if child.name in spaceList:
  986. # child.insert_after(" ")
  987. text = str(soup.get_text())
  988. #替换英文冒号为中文冒号
  989. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  990. #替换为中文逗号
  991. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  992. #替换为中文分号
  993. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  994. # 感叹号替换为中文句号
  995. text = re.sub("(?<=[\u4e00-\u9fa5])[!!]|[!!](?=[\u4e00-\u9fa5])","。",text)
  996. #替换格式未识别的问号为" " ,update:2021/7/20
  997. text = re.sub("[?\?]{2,}"," ",text)
  998. #替换"""为"“",否则导入deepdive出错
  999. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1000. # print('==1',text)
  1001. # text = re.sub("\s{4,}",",",text)
  1002. # 解决公告中的" "空格替换问题
  1003. if re.search("\s{4,}",text):
  1004. _text = ""
  1005. for _sent in re.split("。+",text):
  1006. for _sent2 in re.split(',+',_sent):
  1007. for _sent3 in re.split(":+",_sent2):
  1008. for _t in re.split("\s{4,}",_sent3):
  1009. if len(_t)<3:
  1010. _text += _t
  1011. else:
  1012. _text += ","+_t
  1013. _text += ":"
  1014. _text = _text[:-1]
  1015. _text += ","
  1016. _text = _text[:-1]
  1017. _text += "。"
  1018. _text = _text[:-1]
  1019. text = _text
  1020. # print('==2',text)
  1021. #替换标点
  1022. #替换连续的标点
  1023. if final:
  1024. text = re.sub("##space##"," ",text)
  1025. punc_pattern = "(?P<del>[。,;::,\s]+)"
  1026. list_punc = re.findall(punc_pattern,text)
  1027. list_punc.sort(key=lambda x:len(x),reverse=True)
  1028. for punc_del in list_punc:
  1029. if len(punc_del)>1:
  1030. if len(punc_del.strip())>0:
  1031. if ":" in punc_del.strip():
  1032. text = re.sub(punc_del,":",text)
  1033. else:
  1034. text = re.sub(punc_del,punc_del.strip()[0],text) #2021/12/09 修正由于某些标签后插入符号把原来符号替换
  1035. else:
  1036. text = re.sub(punc_del,"",text)
  1037. #将连续的中文句号替换为一个
  1038. text_split = text.split("。")
  1039. text_split = [x for x in text_split if len(x)>0]
  1040. text = "。".join(text_split)
  1041. # #删除标签中的所有空格
  1042. # for subs in subspaceList:
  1043. # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  1044. # while(True):
  1045. # oneMatch = re.search(re.compile(patten),text)
  1046. # if oneMatch is not None:
  1047. # _match = oneMatch.group(1)
  1048. # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
  1049. # else:
  1050. # break
  1051. # text过大报错
  1052. LOOP_LEN = 10000
  1053. LOOP_BEGIN = 0
  1054. _text = ""
  1055. if len(text)<10000000:
  1056. while(LOOP_BEGIN<len(text)):
  1057. _text += re.sub(")",")",re.sub("(","(",re.sub("\s+","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  1058. LOOP_BEGIN += LOOP_LEN
  1059. text = _text
  1060. return text
  1061. '''
  1062. #数据清洗
  1063. def segment(soup):
  1064. segList = ["title"]
  1065. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  1066. spaceList = ["span"]
  1067. tbodies = soup.find_all('tbody')
  1068. if len(tbodies) == 0:
  1069. tbodies = soup.find_all('table')
  1070. # 递归遍历所有节点,插入符号
  1071. for child in soup.find_all(recursive=True):
  1072. if child.name == 'br':
  1073. child.insert_before(',')
  1074. child_text = re.sub('\s', '', child.get_text())
  1075. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  1076. continue
  1077. if child.name in segList:
  1078. child.insert_after("。")
  1079. if child.name in commaList:
  1080. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  1081. child.insert_after(",")
  1082. elif len(child_text) >=50:
  1083. child.insert_after("。")
  1084. #if child.name in spaceList:
  1085. #child.insert_after(" ")
  1086. text = str(soup.get_text())
  1087. text = re.sub("\s{5,}",",",text)
  1088. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1089. #替换"""为"“",否则导入deepdive出错
  1090. text = text.replace('"',"“")
  1091. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  1092. #删除所有空格
  1093. text = re.sub("\s+","#nbsp#",text)
  1094. text_list = text.split('#nbsp#')
  1095. new_text = ''
  1096. for i in range(len(text_list)-1):
  1097. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  1098. new_text += text_list[i]
  1099. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  1100. new_text += text_list[i] + '。'
  1101. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  1102. new_text += text_list[i] + ';'
  1103. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  1104. new_text += text_list[i] + ' '
  1105. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  1106. new_text += text_list[i]
  1107. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  1108. new_text += text_list[i] + ','
  1109. else:
  1110. new_text += text_list[i]
  1111. new_text += text_list[-1]
  1112. text = new_text
  1113. #替换英文冒号为中文冒号
  1114. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1115. #替换为中文逗号
  1116. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1117. #替换为中文分号
  1118. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1119. #替换标点
  1120. while(True):
  1121. #替换连续的标点
  1122. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  1123. if punc is not None:
  1124. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  1125. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  1126. if punc is not None:
  1127. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  1128. else:
  1129. #替换标点之后的空格
  1130. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  1131. if punc is not None:
  1132. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  1133. else:
  1134. break
  1135. #将连续的中文句号替换为一个
  1136. text_split = text.split("。")
  1137. text_split = [x for x in text_split if len(x)>0]
  1138. text = "。".join(text_split)
  1139. #替换中文括号为英文括号
  1140. text = re.sub("(","(",text)
  1141. text = re.sub(")",")",text)
  1142. return text
  1143. '''
  1144. #连续实体合并(弃用)
  1145. def union_ner(list_ner):
  1146. result_list = []
  1147. union_index = []
  1148. union_index_set = set()
  1149. for i in range(len(list_ner)-1):
  1150. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  1151. if list_ner[i][1]-list_ner[i+1][0]==1:
  1152. union_index_set.add(i)
  1153. union_index_set.add(i+1)
  1154. union_index.append((i,i+1))
  1155. for i in range(len(list_ner)):
  1156. if i not in union_index_set:
  1157. result_list.append(list_ner[i])
  1158. for item in union_index:
  1159. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  1160. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  1161. return result_list
  1162. # def get_preprocessed(articles,useselffool=False):
  1163. # '''
  1164. # @summary:预处理步骤,NLP处理、实体识别
  1165. # @param:
  1166. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1167. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1168. # '''
  1169. # list_articles = []
  1170. # list_sentences = []
  1171. # list_entitys = []
  1172. # cost_time = dict()
  1173. # for article in articles:
  1174. # list_sentences_temp = []
  1175. # list_entitys_temp = []
  1176. # doc_id = article[0]
  1177. # sourceContent = article[1]
  1178. # _send_doc_id = article[3]
  1179. # _title = article[4]
  1180. # #表格处理
  1181. # key_preprocess = "tableToText"
  1182. # start_time = time.time()
  1183. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1184. #
  1185. # # log(article_processed)
  1186. #
  1187. # if key_preprocess not in cost_time:
  1188. # cost_time[key_preprocess] = 0
  1189. # cost_time[key_preprocess] += time.time()-start_time
  1190. #
  1191. # #article_processed = article[1]
  1192. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1193. # #nlp处理
  1194. # if article_processed is not None and len(article_processed)!=0:
  1195. # split_patten = "。"
  1196. # sentences = []
  1197. # _begin = 0
  1198. # for _iter in re.finditer(split_patten,article_processed):
  1199. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1200. # _begin = _iter.span()[1]
  1201. # sentences.append(article_processed[_begin:])
  1202. #
  1203. # lemmas = []
  1204. # doc_offsets = []
  1205. # dep_types = []
  1206. # dep_tokens = []
  1207. #
  1208. # time1 = time.time()
  1209. #
  1210. # '''
  1211. # tokens_all = fool.cut(sentences)
  1212. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1213. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1214. # ner_entitys_all = fool.ner(sentences)
  1215. # '''
  1216. # #限流执行
  1217. # key_nerToken = "nerToken"
  1218. # start_time = time.time()
  1219. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1220. # if key_nerToken not in cost_time:
  1221. # cost_time[key_nerToken] = 0
  1222. # cost_time[key_nerToken] += time.time()-start_time
  1223. #
  1224. #
  1225. # for sentence_index in range(len(sentences)):
  1226. #
  1227. #
  1228. #
  1229. # list_sentence_entitys = []
  1230. # sentence_text = sentences[sentence_index]
  1231. # tokens = tokens_all[sentence_index]
  1232. #
  1233. # list_tokenbegin = []
  1234. # begin = 0
  1235. # for i in range(0,len(tokens)):
  1236. # list_tokenbegin.append(begin)
  1237. # begin += len(str(tokens[i]))
  1238. # list_tokenbegin.append(begin+1)
  1239. # #pos_tag = pos_all[sentence_index]
  1240. # pos_tag = ""
  1241. #
  1242. # ner_entitys = ner_entitys_all[sentence_index]
  1243. #
  1244. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1245. #
  1246. # #识别package
  1247. #
  1248. #
  1249. # #识别实体
  1250. # for ner_entity in ner_entitys:
  1251. # begin_index_temp = ner_entity[0]
  1252. # end_index_temp = ner_entity[1]
  1253. # entity_type = ner_entity[2]
  1254. # entity_text = ner_entity[3]
  1255. #
  1256. # for j in range(len(list_tokenbegin)):
  1257. # if list_tokenbegin[j]==begin_index_temp:
  1258. # begin_index = j
  1259. # break
  1260. # elif list_tokenbegin[j]>begin_index_temp:
  1261. # begin_index = j-1
  1262. # break
  1263. # begin_index_temp += len(str(entity_text))
  1264. # for j in range(begin_index,len(list_tokenbegin)):
  1265. # if list_tokenbegin[j]>=begin_index_temp:
  1266. # end_index = j-1
  1267. # break
  1268. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1269. #
  1270. # #去掉标点符号
  1271. # entity_text = re.sub("[,,。:]","",entity_text)
  1272. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1273. #
  1274. #
  1275. # #使用正则识别金额
  1276. # entity_type = "money"
  1277. #
  1278. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1279. #
  1280. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1281. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1282. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1283. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1284. #
  1285. # set_begin = set()
  1286. # for pattern_key in list_money_pattern.keys():
  1287. # pattern = re.compile(list_money_pattern[pattern_key])
  1288. # all_match = re.findall(pattern, sentence_text)
  1289. # index = 0
  1290. # for i in range(len(all_match)):
  1291. # if len(all_match[i][0])>0:
  1292. # # print("===",all_match[i])
  1293. # #print(all_match[i][0])
  1294. # unit = ""
  1295. # entity_text = all_match[i][3]
  1296. # if pattern_key in ["key_word","front_m"]:
  1297. # unit = all_match[i][1]
  1298. # else:
  1299. # unit = all_match[i][4]
  1300. # if entity_text.find("元")>=0:
  1301. # unit = ""
  1302. #
  1303. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1304. #
  1305. # begin_index_temp = index
  1306. # for j in range(len(list_tokenbegin)):
  1307. # if list_tokenbegin[j]==index:
  1308. # begin_index = j
  1309. # break
  1310. # elif list_tokenbegin[j]>index:
  1311. # begin_index = j-1
  1312. # break
  1313. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1314. # end_index_temp = index
  1315. # #index += len(str(all_match[i][0]))
  1316. # for j in range(begin_index,len(list_tokenbegin)):
  1317. # if list_tokenbegin[j]>=index:
  1318. # end_index = j-1
  1319. # break
  1320. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1321. #
  1322. #
  1323. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1324. # if len(unit)>0:
  1325. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1326. # else:
  1327. # entity_text = str(getUnifyMoney(entity_text))
  1328. #
  1329. # _exists = False
  1330. # for item in list_sentence_entitys:
  1331. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1332. # _exists = True
  1333. # if not _exists:
  1334. # if float(entity_text)>10:
  1335. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1336. #
  1337. # else:
  1338. # index += 1
  1339. #
  1340. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1341. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1342. # list_sentences.append(list_sentences_temp)
  1343. # list_entitys.append(list_entitys_temp)
  1344. # return list_articles,list_sentences,list_entitys,cost_time
  1345. def get_preprocessed(articles,useselffool=False):
  1346. '''
  1347. @summary:预处理步骤,NLP处理、实体识别
  1348. @param:
  1349. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1350. @return:list of articles,list of each article of sentences,list of each article of entitys
  1351. '''
  1352. cost_time = dict()
  1353. list_articles = get_preprocessed_article(articles,cost_time)
  1354. list_sentences = get_preprocessed_sentences(list_articles,True,cost_time)
  1355. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1356. calibrateEnterprise(list_articles,list_sentences,list_entitys)
  1357. return list_articles,list_sentences,list_entitys,cost_time
  1358. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  1359. '''
  1360. :param articles: 待处理的article source html
  1361. :param useselffool: 是否使用selffool
  1362. :return: list_articles
  1363. '''
  1364. list_articles = []
  1365. for article in articles:
  1366. doc_id = article[0]
  1367. sourceContent = article[1]
  1368. sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
  1369. _send_doc_id = article[3]
  1370. _title = article[4]
  1371. page_time = article[5]
  1372. #表格处理
  1373. key_preprocess = "tableToText"
  1374. start_time = time.time()
  1375. article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1376. article_processed = article_processed.replace('.','.') # 2021/12/01 修正OCR识别PDF小数点错误问题
  1377. article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
  1378. article_processed = article_processed.replace('成交工程价款', '成交工程价') # 2021/12/21 修正为中标价
  1379. # 提取bidway
  1380. list_bidway = extract_bidway(article_processed, _title)
  1381. if list_bidway:
  1382. bidway = list_bidway[0].get("body")
  1383. # bidway名称统一规范
  1384. bidway = bidway_integrate(bidway)
  1385. else:
  1386. bidway = ""
  1387. # 修正被","逗号分隔的时间
  1388. repair_time = re.compile("[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d|"
  1389. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[:时点],?[0-6]\d分?|"
  1390. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[时点]|"
  1391. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]|"
  1392. "[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d"
  1393. )
  1394. for _time in set(re.findall(repair_time,article_processed)):
  1395. if re.search(",",_time):
  1396. _time2 = re.sub(",", "", _time)
  1397. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time2)
  1398. if item:
  1399. _time2 = _time2.replace(item.group(),item.group() + " ")
  1400. article_processed = article_processed.replace(_time, _time2)
  1401. else:
  1402. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time)
  1403. if item:
  1404. _time2 = _time.replace(item.group(),item.group() + " ")
  1405. article_processed = article_processed.replace(_time, _time2)
  1406. # print('re_rtime',re.findall(repair_time,article_processed))
  1407. # log(article_processed)
  1408. if key_preprocess not in cost_time:
  1409. cost_time[key_preprocess] = 0
  1410. cost_time[key_preprocess] += round(time.time()-start_time,2)
  1411. #article_processed = article[1]
  1412. _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
  1413. bidway=bidway)
  1414. _article.fingerprint = getFingerprint(_title+sourceContent)
  1415. _article.page_time = page_time
  1416. list_articles.append(_article)
  1417. return list_articles
  1418. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  1419. '''
  1420. :param list_articles: 经过预处理的article text
  1421. :return: list_sentences
  1422. '''
  1423. list_sentences = []
  1424. for article in list_articles:
  1425. list_sentences_temp = []
  1426. list_entitys_temp = []
  1427. doc_id = article.id
  1428. _send_doc_id = article.doc_id
  1429. _title = article.title
  1430. #表格处理
  1431. key_preprocess = "tableToText"
  1432. start_time = time.time()
  1433. article_processed = article.content
  1434. if key_preprocess not in cost_time:
  1435. cost_time[key_preprocess] = 0
  1436. cost_time[key_preprocess] += time.time()-start_time
  1437. #nlp处理
  1438. if article_processed is not None and len(article_processed)!=0:
  1439. split_patten = "。"
  1440. sentences = []
  1441. _begin = 0
  1442. sentences_set = set()
  1443. for _iter in re.finditer(split_patten,article_processed):
  1444. _sen = article_processed[_begin:_iter.span()[1]]
  1445. if len(_sen)>0 and _sen not in sentences_set:
  1446. sentences.append(_sen)
  1447. sentences_set.add(_sen)
  1448. _begin = _iter.span()[1]
  1449. _sen = article_processed[_begin:]
  1450. if len(_sen)>0 and _sen not in sentences_set:
  1451. sentences.append(_sen)
  1452. sentences_set.add(_sen)
  1453. article.content = "".join(sentences)
  1454. # sentences.append(article_processed[_begin:])
  1455. lemmas = []
  1456. doc_offsets = []
  1457. dep_types = []
  1458. dep_tokens = []
  1459. time1 = time.time()
  1460. '''
  1461. tokens_all = fool.cut(sentences)
  1462. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1463. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1464. ner_entitys_all = fool.ner(sentences)
  1465. '''
  1466. #限流执行
  1467. key_nerToken = "nerToken"
  1468. start_time = time.time()
  1469. tokens_all = getTokens(sentences,useselffool=useselffool)
  1470. if key_nerToken not in cost_time:
  1471. cost_time[key_nerToken] = 0
  1472. cost_time[key_nerToken] += round(time.time()-start_time,2)
  1473. for sentence_index in range(len(sentences)):
  1474. sentence_text = sentences[sentence_index]
  1475. tokens = tokens_all[sentence_index]
  1476. #pos_tag = pos_all[sentence_index]
  1477. pos_tag = ""
  1478. ner_entitys = ""
  1479. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1480. if len(list_sentences_temp)==0:
  1481. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  1482. list_sentences.append(list_sentences_temp)
  1483. return list_sentences
  1484. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  1485. '''
  1486. :param list_sentences:分局情况
  1487. :param cost_time:
  1488. :return: list_entitys
  1489. '''
  1490. list_entitys = []
  1491. for list_sentence in list_sentences:
  1492. sentences = []
  1493. list_entitys_temp = []
  1494. for _sentence in list_sentence:
  1495. sentences.append(_sentence.sentence_text)
  1496. lemmas = []
  1497. doc_offsets = []
  1498. dep_types = []
  1499. dep_tokens = []
  1500. time1 = time.time()
  1501. '''
  1502. tokens_all = fool.cut(sentences)
  1503. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1504. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1505. ner_entitys_all = fool.ner(sentences)
  1506. '''
  1507. #限流执行
  1508. key_nerToken = "nerToken"
  1509. start_time = time.time()
  1510. found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
  1511. # found_pingbiao = False
  1512. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  1513. if key_nerToken not in cost_time:
  1514. cost_time[key_nerToken] = 0
  1515. cost_time[key_nerToken] += round(time.time()-start_time,2)
  1516. for sentence_index in range(len(list_sentence)):
  1517. list_sentence_entitys = []
  1518. sentence_text = list_sentence[sentence_index].sentence_text
  1519. tokens = list_sentence[sentence_index].tokens
  1520. doc_id = list_sentence[sentence_index].doc_id
  1521. list_tokenbegin = []
  1522. begin = 0
  1523. for i in range(0,len(tokens)):
  1524. list_tokenbegin.append(begin)
  1525. begin += len(str(tokens[i]))
  1526. list_tokenbegin.append(begin+1)
  1527. #pos_tag = pos_all[sentence_index]
  1528. pos_tag = ""
  1529. ner_entitys = ner_entitys_all[sentence_index]
  1530. '''正则识别角色实体 经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
  1531. for it in re.finditer(
  1532. '(?P<text_key_word>[^,。、;《]{,5}(单一来源|中标|中选|中价|成交)?(供应商|供货商|服务商|候选人|单位|人)(名称)?为?[::]+)(?P<text>([^,。、;《]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[,。]',
  1533. sentence_text):
  1534. for k, v in it.groupdict().items():
  1535. if k == 'text_key_word':
  1536. keyword = v
  1537. if k == 'text':
  1538. entity = v
  1539. b = it.start() + len(keyword)
  1540. e = it.end() - 1
  1541. if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  1542. ner_entitys.append((b, e, 'company', entity))
  1543. # print('正则新增 :',(b, e, 'company', entity))
  1544. #识别package
  1545. #识别实体
  1546. for ner_entity in ner_entitys:
  1547. begin_index_temp = ner_entity[0]
  1548. end_index_temp = ner_entity[1]
  1549. entity_type = ner_entity[2]
  1550. entity_text = ner_entity[3]
  1551. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  1552. continue
  1553. for j in range(len(list_tokenbegin)):
  1554. if list_tokenbegin[j]==begin_index_temp:
  1555. begin_index = j
  1556. break
  1557. elif list_tokenbegin[j]>begin_index_temp:
  1558. begin_index = j-1
  1559. break
  1560. begin_index_temp += len(str(entity_text))
  1561. for j in range(begin_index,len(list_tokenbegin)):
  1562. if list_tokenbegin[j]>=begin_index_temp:
  1563. end_index = j-1
  1564. break
  1565. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1566. #去掉标点符号
  1567. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  1568. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  1569. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1570. # 标记文章末尾的"发布人”、“发布时间”实体
  1571. if sentence_index==len(list_sentence)-1:
  1572. if len(list_sentence_entitys[-2:])>2:
  1573. second2last = list_sentence_entitys[-2]
  1574. last = list_sentence_entitys[-1]
  1575. if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
  1576. second2last.entity_type=="time" and last.entity_type in ["company",'org']):
  1577. if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
  1578. last.is_tail = True
  1579. second2last.is_tail = True
  1580. #使用正则识别金额
  1581. entity_type = "money"
  1582. #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1583. # list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  1584. # "key_word":"((?P<text_key_word>(?:[¥¥]+,?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果:)(?:[,(\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[)\)]?))",
  1585. # "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())",
  1586. # "behind_m":"(()()(?P<money_behind_m>[0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\))]?)"}
  1587. list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  1588. "key_word": "((?P<text_key_word>(?:[¥¥]+,?|[单报标限总]价|金额|成交报?价|价格|预算|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果|成交额|中标额)(?:[,,(\(]*\s*(人民币)?(?P<unit_key_word_before>[万亿]?元?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[::])?(\d+(\*\d+%)+=)?(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]{,1})(?:[(\(]?(?P<filter_>[%])*\s*(单位[::])?(?P<unit_key_word_behind>[万亿]?元?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[)\)]?))",
  1589. "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万亿]?元)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)())",
  1590. "behind_m":"(()()(?P<money_behind_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)(人民币)?[\((]?(?P<unit_behind_m>[万亿]?元(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\))]?)"}
  1591. # 2021/7/19 调整金额,单位提取正则,修复部分金额因为单位提取失败被过滤问题。
  1592. pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
  1593. set_begin = set()
  1594. # for pattern_key in list_money_pattern.keys():
  1595. # for pattern_key in ["cn","key_word","behind_m","front_m"]:
  1596. # # pattern = re.compile(list_money_pattern[pattern_key])
  1597. # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?\s*([万元]*)\s*[)\)]?))*|(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*|((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*")
  1598. # all_match = re.findall(pattern, sentence_text)
  1599. # index = 0
  1600. # for i in range(len(all_match)):
  1601. # if len(all_match[i][0])>0:
  1602. # print("===",all_match[i])
  1603. # #print(all_match[i][0])
  1604. # unit = ""
  1605. # entity_text = all_match[i][3]
  1606. # if pattern_key in ["key_word","front_m"]:
  1607. # unit = all_match[i][1]
  1608. # if pattern_key=="key_word":
  1609. # if all_match[i][1]=="" and all_match[i][4]!="":
  1610. # unit = all_match[i][4]
  1611. # else:
  1612. # unit = all_match[i][4]
  1613. # if entity_text.find("元")>=0:
  1614. # unit = ""
  1615. #
  1616. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1617. # begin_index_temp = index
  1618. # for j in range(len(list_tokenbegin)):
  1619. # if list_tokenbegin[j]==index:
  1620. # begin_index = j
  1621. # break
  1622. # elif list_tokenbegin[j]>index:
  1623. # begin_index = j-1
  1624. # break
  1625. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1626. # end_index_temp = index
  1627. # #index += len(str(all_match[i][0]))
  1628. # for j in range(begin_index,len(list_tokenbegin)):
  1629. # if list_tokenbegin[j]>=index:
  1630. # end_index = j-1
  1631. # break
  1632. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1633. #
  1634. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1635. # if len(unit)>0:
  1636. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1637. # else:
  1638. # entity_text = str(getUnifyMoney(entity_text))
  1639. #
  1640. # _exists = False
  1641. # for item in list_sentence_entitys:
  1642. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1643. # _exists = True
  1644. # if not _exists:
  1645. # if float(entity_text)>1:
  1646. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1647. #
  1648. # else:
  1649. # index += 1
  1650. # if re.search('评标结果|候选人公示', sentence_text):
  1651. # found_pingbiao = True
  1652. if re.search('业绩', sentence_text):
  1653. found_yeji += 1
  1654. if found_yeji >= 2: # 过滤掉业绩后面的所有金额
  1655. all_match = []
  1656. else:
  1657. all_match = re.finditer(pattern_money, sentence_text)
  1658. index = 0
  1659. for _match in all_match:
  1660. if len(_match.group())>0:
  1661. # print("===",_match.group())
  1662. # # print(_match.groupdict())
  1663. notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
  1664. unit = ""
  1665. entity_text = ""
  1666. text_beforeMoney = ""
  1667. filter = ""
  1668. filter_unit = False
  1669. notSure = False
  1670. if re.search('业绩', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额
  1671. # print('金额在业绩后面: ', _match.group(0))
  1672. found_yeji += 1
  1673. break
  1674. for k,v in _match.groupdict().items():
  1675. if v!="" and v is not None:
  1676. if k=='text_key_word':
  1677. notSure = True
  1678. if k.split("_")[0]=="money":
  1679. entity_text = v
  1680. if k.split("_")[0]=="unit":
  1681. unit = v
  1682. if k.split("_")[0]=="text":
  1683. text_beforeMoney = v
  1684. if k.split("_")[0]=="filter":
  1685. filter = v
  1686. if re.search("filter_unit",k) is not None:
  1687. filter_unit = True
  1688. if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()): # 2021/7/19 修正OCR识别小数点为逗号
  1689. if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]):
  1690. entity_text = re.sub('\d+,', '', entity_text)
  1691. else:
  1692. entity_text = entity_text.replace(',', '.')
  1693. # print(' 修正OCR识别小数点为逗号')
  1694. if entity_text.find("元")>=0:
  1695. unit = ""
  1696. if unit == "": #2021/7/21 有明显金额特征的补充单位,避免被过滤
  1697. if ('¥' in text_beforeMoney or '¥' in text_beforeMoney):
  1698. unit = '元'
  1699. # print('明显金额特征补充单位 元')
  1700. elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[::为]+$', text_beforeMoney.strip()) and \
  1701. re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None:
  1702. unit = '元'
  1703. # print('明显金额特征补充单位 元')
  1704. elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7},?)$)|(^\d{,3}(,\d{3})+,?$)',entity_text):
  1705. unit = '元'
  1706. # print('明显金额特征补充单位 元')
  1707. if unit.find("万") >= 0 and entity_text.find("万") >= 0: #2021/7/19修改为金额文本有万,不计算单位
  1708. # print('修正金额及单位都有万, 金额:',entity_text, '单位:',unit)
  1709. unit = "元"
  1710. if re.search('.*万元万元', entity_text): #2021/7/19 修正两个万元
  1711. # print(' 修正两个万元',entity_text)
  1712. entity_text = entity_text.replace('万元万元','万元')
  1713. else:
  1714. if filter_unit:
  1715. continue
  1716. if filter!="":
  1717. continue
  1718. index = _match.span()[0]+len(text_beforeMoney)
  1719. begin_index_temp = index
  1720. for j in range(len(list_tokenbegin)):
  1721. if list_tokenbegin[j]==index:
  1722. begin_index = j
  1723. break
  1724. elif list_tokenbegin[j]>index:
  1725. begin_index = j-1
  1726. break
  1727. index = _match.span()[1]
  1728. end_index_temp = index
  1729. #index += len(str(all_match[i][0]))
  1730. for j in range(begin_index,len(list_tokenbegin)):
  1731. if list_tokenbegin[j]>=index:
  1732. end_index = j-1
  1733. break
  1734. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1735. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
  1736. # print('转换前金额:', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
  1737. if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/8/5过滤掉总投资金额
  1738. # print('总投资金额: ', _match.group(0))
  1739. notes = '总投资'
  1740. elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额
  1741. notes = '投资'
  1742. elif re.search('工程造价', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额
  1743. notes = '工程造价'
  1744. elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
  1745. or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\((]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\))]*[::为]',
  1746. sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
  1747. or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
  1748. sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
  1749. notes = '保证金'
  1750. # print('保证金信息:', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
  1751. elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
  1752. sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
  1753. notes = '成本警戒线'
  1754. elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为:]', sentence_text[_match.span()[0]:_match.span()[1]]):
  1755. cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
  1756. notes = cost_re.group(1)
  1757. elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
  1758. notes = '单价'
  1759. elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
  1760. notes = '大写'
  1761. if entity_text[0] == "拾": # 2021/12/16 修正大写金额省略了数字转换错误问题
  1762. entity_text = "壹"+entity_text
  1763. # print("补充备注:notes = 大写")
  1764. if len(unit)>0:
  1765. if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况
  1766. # print('修正单位万元金额过大的情况 金额:', entity_text, '单位:', unit)
  1767. entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000)
  1768. unit = '元' # 修正金额后单位 重置为元
  1769. else:
  1770. # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
  1771. entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1772. else:
  1773. if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8:
  1774. entity_text = str(getUnifyMoney(entity_text)/10000)
  1775. # print('修正金额字段含万 过大的情况')
  1776. else:
  1777. entity_text = str(getUnifyMoney(entity_text))
  1778. if float(entity_text)<100 or float(entity_text)>100000000000:
  1779. # print('过滤掉金额:float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
  1780. continue
  1781. if notSure and unit=="" and float(entity_text)>100*10000:
  1782. # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000:', entity_text, unit)
  1783. continue
  1784. _exists = False
  1785. for item in list_sentence_entitys:
  1786. if item.entity_id==entity_id and item.entity_type==entity_type:
  1787. _exists = True
  1788. if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index):
  1789. _exists = True
  1790. if not _exists:
  1791. if float(entity_text)>1:
  1792. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1793. list_sentence_entitys[-1].notes = notes # 2021/7/20 新增金额备注
  1794. list_sentence_entitys[-1].money_unit = unit # 2021/7/20 新增金额备注
  1795. # print('预处理中的 金额:%s, 单位:%s'%(entity_text,unit))
  1796. else:
  1797. index += 1
  1798. # "联系人"正则补充提取 2021/11/15 新增
  1799. list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
  1800. error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
  1801. '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理','代理人','采购','附件']
  1802. list_person_text = set(list_person_text + error_text)
  1803. re_person = re.compile("联系人[::]([\u4e00-\u9fa5]工)|"
  1804. "联系人[::]([\u4e00-\u9fa5]{2,3})(?=联系)|"
  1805. "联系人[::]([\u4e00-\u9fa5]{2,3})")
  1806. list_person = []
  1807. for match_result in re_person.finditer(sentence_text):
  1808. match_text = match_result.group()
  1809. entity_text = match_text[4:]
  1810. wordOffset_begin = match_result.start() + 4
  1811. wordOffset_end = match_result.end()
  1812. # print(text[wordOffset_begin:wordOffset_end])
  1813. # 排除一些不为人名的实体
  1814. if re.search("^[\u4e00-\u9fa5]{7,}([,。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
  1815. continue
  1816. if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
  1817. _person = dict()
  1818. _person['body'] = entity_text
  1819. _person['begin_index'] = wordOffset_begin
  1820. _person['end_index'] = wordOffset_end
  1821. list_person.append(_person)
  1822. entity_type = "person"
  1823. for person in list_person:
  1824. begin_index_temp = person['begin_index']
  1825. for j in range(len(list_tokenbegin)):
  1826. if list_tokenbegin[j] == begin_index_temp:
  1827. begin_index = j
  1828. break
  1829. elif list_tokenbegin[j] > begin_index_temp:
  1830. begin_index = j - 1
  1831. break
  1832. index = person['end_index']
  1833. end_index_temp = index
  1834. for j in range(begin_index, len(list_tokenbegin)):
  1835. if list_tokenbegin[j] >= index:
  1836. end_index = j - 1
  1837. break
  1838. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1839. entity_text = person['body']
  1840. list_sentence_entitys.append(
  1841. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1842. begin_index_temp, end_index_temp))
  1843. # 资金来源提取 2020/12/30 新增
  1844. list_moneySource = extract_moneySource(sentence_text)
  1845. entity_type = "moneysource"
  1846. for moneySource in list_moneySource:
  1847. begin_index_temp = moneySource['begin_index']
  1848. for j in range(len(list_tokenbegin)):
  1849. if list_tokenbegin[j] == begin_index_temp:
  1850. begin_index = j
  1851. break
  1852. elif list_tokenbegin[j] > begin_index_temp:
  1853. begin_index = j - 1
  1854. break
  1855. index = moneySource['end_index']
  1856. end_index_temp = index
  1857. for j in range(begin_index, len(list_tokenbegin)):
  1858. if list_tokenbegin[j] >= index:
  1859. end_index = j - 1
  1860. break
  1861. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1862. entity_text = moneySource['body']
  1863. list_sentence_entitys.append(
  1864. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1865. begin_index_temp, end_index_temp))
  1866. # 电子邮箱提取 2021/11/04 新增
  1867. list_email = extract_email(sentence_text)
  1868. entity_type = "email" # 电子邮箱
  1869. for email in list_email:
  1870. begin_index_temp = email['begin_index']
  1871. for j in range(len(list_tokenbegin)):
  1872. if list_tokenbegin[j] == begin_index_temp:
  1873. begin_index = j
  1874. break
  1875. elif list_tokenbegin[j] > begin_index_temp:
  1876. begin_index = j - 1
  1877. break
  1878. index = email['end_index']
  1879. end_index_temp = index
  1880. for j in range(begin_index, len(list_tokenbegin)):
  1881. if list_tokenbegin[j] >= index:
  1882. end_index = j - 1
  1883. break
  1884. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1885. entity_text = email['body']
  1886. list_sentence_entitys.append(
  1887. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1888. begin_index_temp, end_index_temp))
  1889. # 服务期限提取 2020/12/30 新增
  1890. list_servicetime = extract_servicetime(sentence_text)
  1891. entity_type = "serviceTime"
  1892. for servicetime in list_servicetime:
  1893. begin_index_temp = servicetime['begin_index']
  1894. for j in range(len(list_tokenbegin)):
  1895. if list_tokenbegin[j] == begin_index_temp:
  1896. begin_index = j
  1897. break
  1898. elif list_tokenbegin[j] > begin_index_temp:
  1899. begin_index = j - 1
  1900. break
  1901. index = servicetime['end_index']
  1902. end_index_temp = index
  1903. for j in range(begin_index, len(list_tokenbegin)):
  1904. if list_tokenbegin[j] >= index:
  1905. end_index = j - 1
  1906. break
  1907. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1908. entity_text = servicetime['body']
  1909. list_sentence_entitys.append(
  1910. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1911. begin_index_temp, end_index_temp))
  1912. # 招标方式提取 2020/12/30 新增
  1913. # list_bidway = extract_bidway(sentence_text, )
  1914. # entity_type = "bidway"
  1915. # for bidway in list_bidway:
  1916. # begin_index_temp = bidway['begin_index']
  1917. # end_index_temp = bidway['end_index']
  1918. # begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
  1919. # end_index = changeIndexFromWordToWords(tokens, end_index_temp)
  1920. # if begin_index is None or end_index is None:
  1921. # continue
  1922. # print(begin_index_temp,end_index_temp,begin_index,end_index)
  1923. # entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1924. # entity_text = bidway['body']
  1925. # list_sentence_entitys.append(
  1926. # Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1927. # begin_index_temp, end_index_temp))
  1928. # 2021/12/29 新增比率提取
  1929. list_ratio = extract_ratio(sentence_text)
  1930. entity_type = "ratio"
  1931. for ratio in list_ratio:
  1932. # print("ratio", ratio)
  1933. begin_index_temp = ratio['begin_index']
  1934. for j in range(len(list_tokenbegin)):
  1935. if list_tokenbegin[j] == begin_index_temp:
  1936. begin_index = j
  1937. break
  1938. elif list_tokenbegin[j] > begin_index_temp:
  1939. begin_index = j - 1
  1940. break
  1941. index = ratio['end_index']
  1942. end_index_temp = index
  1943. for j in range(begin_index, len(list_tokenbegin)):
  1944. if list_tokenbegin[j] >= index:
  1945. end_index = j - 1
  1946. break
  1947. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1948. entity_text = ratio['body']
  1949. list_sentence_entitys.append(
  1950. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1951. begin_index_temp, end_index_temp))
  1952. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1953. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1954. list_entitys.append(list_entitys_temp)
  1955. return list_entitys
  1956. def union_result(codeName,prem):
  1957. '''
  1958. @summary:模型的结果拼成字典
  1959. @param:
  1960. codeName:编号名称模型的结果字典
  1961. prem:拿到属性的角色的字典
  1962. @return:拼接起来的字典
  1963. '''
  1964. result = []
  1965. assert len(codeName)==len(prem)
  1966. for item_code,item_prem in zip(codeName,prem):
  1967. result.append(dict(item_code,**item_prem))
  1968. return result
  1969. def persistenceData(data):
  1970. '''
  1971. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  1972. '''
  1973. import psycopg2
  1974. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  1975. cursor = conn.cursor()
  1976. for item_index in range(len(data)):
  1977. item = data[item_index]
  1978. doc_id = item[0]
  1979. dic = item[1]
  1980. code = dic['code']
  1981. name = dic['name']
  1982. prem = dic['prem']
  1983. if len(code)==0:
  1984. code_insert = ""
  1985. else:
  1986. code_insert = ";".join(code)
  1987. prem_insert = ""
  1988. for item in prem:
  1989. for x in item:
  1990. if isinstance(x, list):
  1991. if len(x)>0:
  1992. for x1 in x:
  1993. prem_insert+="/".join(x1)+","
  1994. prem_insert+="$"
  1995. else:
  1996. prem_insert+=str(x)+"$"
  1997. prem_insert+=";"
  1998. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  1999. cursor.execute(sql)
  2000. conn.commit()
  2001. conn.close()
  2002. def persistenceData1(list_entitys,list_sentences):
  2003. '''
  2004. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  2005. '''
  2006. import psycopg2
  2007. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  2008. cursor = conn.cursor()
  2009. for list_entity in list_entitys:
  2010. for entity in list_entity:
  2011. if entity.values is not None:
  2012. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  2013. else:
  2014. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  2015. cursor.execute(sql)
  2016. for list_sentence in list_sentences:
  2017. for sentence in list_sentence:
  2018. str_tokens = "["
  2019. for item in sentence.tokens:
  2020. str_tokens += "'"
  2021. if item=="'":
  2022. str_tokens += "''"
  2023. else:
  2024. str_tokens += item
  2025. str_tokens += "',"
  2026. str_tokens = str_tokens[:-1]+"]"
  2027. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  2028. cursor.execute(sql)
  2029. conn.commit()
  2030. conn.close()
  2031. def _handle(item,result_queue):
  2032. dochtml = item["dochtml"]
  2033. docid = item["docid"]
  2034. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  2035. flag = False
  2036. if list_innerTable:
  2037. flag = True
  2038. for table in list_innerTable:
  2039. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  2040. def getPredictTable():
  2041. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  2042. import pandas as pd
  2043. import json
  2044. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  2045. df = pd.read_csv(filename)
  2046. df_data = {"json_table":[],"docid":[]}
  2047. _count = 0
  2048. _sum = len(df["docid"])
  2049. task_queue = Queue()
  2050. result_queue = Queue()
  2051. _index = 0
  2052. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  2053. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  2054. _index += 1
  2055. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  2056. mh.run()
  2057. while True:
  2058. try:
  2059. item = result_queue.get(block=True,timeout=1)
  2060. df_data["docid"].append(item["docid"])
  2061. df_data["json_table"].append(item["json_table"])
  2062. except Exception as e:
  2063. print(e)
  2064. break
  2065. df_1 = pd.DataFrame(df_data)
  2066. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  2067. if __name__=="__main__":
  2068. '''
  2069. import glob
  2070. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  2071. file_txt = str(file).replace("html","txt")
  2072. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  2073. f.write("\n================\n")
  2074. content = codecs.open(file,"r",encoding="utf8").read()
  2075. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2076. '''
  2077. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  2078. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2079. # getPredictTable()
  2080. with open('D:/138786703.html', 'r', encoding='utf-8') as f:
  2081. sourceContent = f.read()
  2082. # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
  2083. # print(article_processed)
  2084. list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
  2085. for entity in list_entitys[0]:
  2086. print(entity.entity_type, entity.entity_text)