Preprocessing.py 110 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281
  1. # -*- coding: utf-8 -*-
  2. from bs4 import BeautifulSoup, Comment
  3. import copy
  4. import sys
  5. import os
  6. import time
  7. import codecs
  8. from BiddingKG.dl.ratio.re_ratio import extract_ratio
  9. sys.setrecursionlimit(1000000)
  10. sys.path.append(os.path.abspath("../.."))
  11. sys.path.append(os.path.abspath(".."))
  12. from BiddingKG.dl.common.Utils import *
  13. from BiddingKG.dl.interface.Entitys import *
  14. from BiddingKG.dl.interface.predictor import getPredictor
  15. from BiddingKG.dl.common.nerUtils import *
  16. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  17. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  18. from BiddingKG.dl.relation_extraction.re_email import extract_email
  19. from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
  20. from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
  21. from BiddingKG.dl.entityLink.entityLink import *
  22. #
  23. def tableToText(soup):
  24. '''
  25. @param:
  26. soup:网页html的soup
  27. @return:处理完表格信息的网页text
  28. '''
  29. def getTrs(tbody):
  30. #获取所有的tr
  31. trs = []
  32. objs = tbody.find_all(recursive=False)
  33. for obj in objs:
  34. if obj.name=="tr":
  35. trs.append(obj)
  36. if obj.name=="tbody":
  37. for tr in obj.find_all("tr",recursive=False):
  38. trs.append(tr)
  39. return trs
  40. def fixSpan(tbody):
  41. # 处理colspan, rowspan信息补全问题
  42. #trs = tbody.findChildren('tr', recursive=False)
  43. trs = getTrs(tbody)
  44. ths_len = 0
  45. ths = list()
  46. trs_set = set()
  47. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  48. # 遍历每一个tr
  49. for indtr, tr in enumerate(trs):
  50. ths_tmp = tr.findChildren('th', recursive=False)
  51. #不补全含有表格的tr
  52. if len(tr.findChildren('table'))>0:
  53. continue
  54. if len(ths_tmp) > 0:
  55. ths_len = ths_len + len(ths_tmp)
  56. for th in ths_tmp:
  57. ths.append(th)
  58. trs_set.add(tr)
  59. # 遍历每行中的element
  60. tds = tr.findChildren(recursive=False)
  61. for indtd, td in enumerate(tds):
  62. # 若有colspan 则补全同一行下一个位置
  63. if 'colspan' in td.attrs:
  64. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  65. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  66. if col<100 and len(td.get_text())<1000:
  67. td['colspan'] = 1
  68. for i in range(1, col, 1):
  69. td.insert_after(copy.copy(td))
  70. for indtr, tr in enumerate(trs):
  71. ths_tmp = tr.findChildren('th', recursive=False)
  72. #不补全含有表格的tr
  73. if len(tr.findChildren('table'))>0:
  74. continue
  75. if len(ths_tmp) > 0:
  76. ths_len = ths_len + len(ths_tmp)
  77. for th in ths_tmp:
  78. ths.append(th)
  79. trs_set.add(tr)
  80. # 遍历每行中的element
  81. tds = tr.findChildren(recursive=False)
  82. for indtd, td in enumerate(tds):
  83. # 若有rowspan 则补全下一行同样位置
  84. if 'rowspan' in td.attrs:
  85. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  86. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  87. td['rowspan'] = 1
  88. for i in range(1, row, 1):
  89. # 获取下一行的所有td, 在对应的位置插入
  90. if indtr+i<len(trs):
  91. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  92. if len(tds1) >= (indtd) and len(tds1)>0:
  93. if indtd > 0:
  94. tds1[indtd - 1].insert_after(copy.copy(td))
  95. else:
  96. tds1[0].insert_before(copy.copy(td))
  97. elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全
  98. tds1[indtd-2].insert_after(copy.copy(td))
  99. def getTable(tbody):
  100. #trs = tbody.findChildren('tr', recursive=False)
  101. trs = getTrs(tbody)
  102. inner_table = []
  103. for tr in trs:
  104. tr_line = []
  105. tds = tr.findChildren(['td','th'], recursive=False)
  106. if len(tds)==0:
  107. tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
  108. for td in tds:
  109. tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
  110. #tr_line.append([td.get_text(),0])
  111. inner_table.append(tr_line)
  112. return inner_table
  113. #处理表格不对齐的问题
  114. def fixTable(inner_table,fix_value="~~"):
  115. maxWidth = 0
  116. for item in inner_table:
  117. if len(item)>maxWidth:
  118. maxWidth = len(item)
  119. for i in range(len(inner_table)):
  120. if len(inner_table[i])<maxWidth:
  121. for j in range(maxWidth-len(inner_table[i])):
  122. inner_table[i].append([fix_value,0])
  123. return inner_table
  124. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  125. height = len(inner_table)
  126. width = len(inner_table[0])
  127. for i in range(height):
  128. point = ""
  129. for j in range(width):
  130. if inner_table[i][j][0]==point and point!="":
  131. inner_table[i][j][0] = pad_row
  132. else:
  133. if inner_table[i][j][0] not in [pad_row,pad_col]:
  134. point = inner_table[i][j][0]
  135. for j in range(width):
  136. point = ""
  137. for i in range(height):
  138. if inner_table[i][j][0]==point and point!="":
  139. inner_table[i][j][0] = pad_col
  140. else:
  141. if inner_table[i][j][0] not in [pad_row,pad_col]:
  142. point = inner_table[i][j][0]
  143. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  144. height = len(inner_table)
  145. width = len(inner_table[0])
  146. for i in range(height):
  147. for j in range(width):
  148. if inner_table[i][j][0]==pad_row:
  149. inner_table[i][j][0] = inner_table[i][j-1][0]
  150. inner_table[i][j][1] = inner_table[i][j-1][1]
  151. if inner_table[i][j][0]==pad_col:
  152. inner_table[i][j][0] = inner_table[i-1][j][0]
  153. inner_table[i][j][1] = inner_table[i-1][j][1]
  154. def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
  155. '''
  156. @summary: 修复表头识别,将明显错误的进行修正
  157. '''
  158. def repairNeeded(line):
  159. first_1 = -1
  160. last_1 = -1
  161. first_0 = -1
  162. last_0 = -1
  163. count_1 = 0
  164. count_0 = 0
  165. for i in range(len(line)):
  166. if line[i][0]==fix_value:
  167. continue
  168. if line[i][1]==1:
  169. if first_1==-1:
  170. first_1 = i
  171. last_1 = i
  172. count_1 += 1
  173. if line[i][1]==0:
  174. if first_0 == -1:
  175. first_0 = i
  176. last_0 = i
  177. count_0 += 1
  178. if first_1 ==-1 or last_0 == -1:
  179. return False
  180. #异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  181. if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
  182. return True
  183. return False
  184. def getsimilarity(line,line1):
  185. same_count = 0
  186. for item,item1 in zip(line,line1):
  187. if item[1]==item1[1]:
  188. same_count += 1
  189. return same_count/len(line)
  190. def selfrepair(inner_table,index,dye_set,key_set):
  191. '''
  192. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  193. '''
  194. #print("B",inner_table[index])
  195. min_presure = 3
  196. list_dye = []
  197. first = None
  198. count = 0
  199. temp_set = set()
  200. _index = 0
  201. for item in inner_table[index]:
  202. if first is None:
  203. first = item[1]
  204. if item[0] not in temp_set:
  205. count += 1
  206. temp_set.add(item[0])
  207. else:
  208. if first == item[1]:
  209. if item[0] not in temp_set:
  210. temp_set.add(item[0])
  211. count += 1
  212. else:
  213. list_dye.append([first,count,_index])
  214. first = item[1]
  215. temp_set.add(item[0])
  216. count = 1
  217. _index += 1
  218. list_dye.append([first,count,_index])
  219. if len(list_dye)>1:
  220. begin = 0
  221. end = 0
  222. for i in range(len(list_dye)):
  223. end = list_dye[i][2]
  224. dye_flag = False
  225. #首尾要求压力减一
  226. if i==0:
  227. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  228. dye_flag = True
  229. dye_type = list_dye[i+1][0]
  230. elif i==len(list_dye)-1:
  231. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  232. dye_flag = True
  233. dye_type = list_dye[i-1][0]
  234. else:
  235. if list_dye[i][1]>1:
  236. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  237. dye_flag = True
  238. dye_type = list_dye[i+1][0]
  239. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  240. dye_flag = True
  241. dye_type = list_dye[i-1][0]
  242. else:
  243. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  244. dye_flag = True
  245. dye_type = list_dye[i+1][0]
  246. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  247. dye_flag = True
  248. dye_type = list_dye[i-1][0]
  249. if dye_flag:
  250. for h in range(begin,end):
  251. inner_table[index][h][1] = dye_type
  252. dye_set.add((inner_table[index][h][0],dye_type))
  253. key_set.add(inner_table[index][h][0])
  254. begin = end
  255. #print("E",inner_table[index])
  256. def otherrepair(inner_table,index,dye_set,key_set):
  257. list_provide_repair = []
  258. if index==0 and len(inner_table)>1:
  259. list_provide_repair.append(index+1)
  260. elif index==len(inner_table)-1:
  261. list_provide_repair.append(index-1)
  262. else:
  263. list_provide_repair.append(index+1)
  264. list_provide_repair.append(index-1)
  265. for provide_index in list_provide_repair:
  266. if not repairNeeded(inner_table[provide_index]):
  267. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  268. if same_prob>=0.8:
  269. for i in range(len(inner_table[provide_index])):
  270. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  271. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  272. key_set.add(inner_table[index][i][0])
  273. inner_table[index][i][1] = inner_table[provide_index][i][1]
  274. elif same_prob<=0.2:
  275. for i in range(len(inner_table[provide_index])):
  276. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  277. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  278. key_set.add(inner_table[index][i][0])
  279. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  280. len_dye_set = len(dye_set)
  281. height = len(inner_table)
  282. for i in range(height):
  283. if repairNeeded(inner_table[i]):
  284. selfrepair(inner_table,i,dye_set,key_set)
  285. #otherrepair(inner_table,i,dye_set,key_set)
  286. for h in range(len(inner_table)):
  287. for w in range(len(inner_table[0])):
  288. if inner_table[h][w][0] in key_set:
  289. for item in dye_set:
  290. if inner_table[h][w][0]==item[0]:
  291. inner_table[h][w][1] = item[1]
  292. #如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  293. if len(dye_set)!=len(key_set):
  294. for i in range(height):
  295. if repairNeeded(inner_table[i]):
  296. selfrepair(inner_table,i,dye_set,key_set)
  297. #otherrepair(inner_table,i,dye_set,key_set)
  298. return
  299. if len(dye_set)==len_dye_set:
  300. '''
  301. for i in range(height):
  302. if repairNeeded(inner_table[i]):
  303. otherrepair(inner_table,i,dye_set,key_set)
  304. '''
  305. return
  306. repairTable(inner_table, dye_set, key_set)
  307. def sliceTable(inner_table,fix_value="~~"):
  308. #进行分块
  309. height = len(inner_table)
  310. width = len(inner_table[0])
  311. head_list = []
  312. head_list.append(0)
  313. last_head = None
  314. last_is_same_value = False
  315. for h in range(height):
  316. is_all_key = True#是否是全表头行
  317. is_all_value = True#是否是全属性值
  318. is_same_with_lastHead = True#和上一行的结构是否相同
  319. is_same_value=True#一行的item都一样
  320. #is_same_first_item = True#与上一行的第一项是否相同
  321. same_value = inner_table[h][0][0]
  322. for w in range(width):
  323. if last_head is not None:
  324. if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
  325. is_all_key = False
  326. if inner_table[h][w][0]==1:
  327. is_all_value = False
  328. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  329. is_same_with_lastHead = False
  330. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  331. is_same_value = False
  332. else:
  333. if re.search("\d+",same_value) is not None:
  334. is_same_value = False
  335. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  336. is_same_first_item = False
  337. last_head = h
  338. if last_is_same_value:
  339. last_is_same_value = is_same_value
  340. continue
  341. if is_same_value:
  342. head_list.append(h)
  343. last_is_same_value = is_same_value
  344. continue
  345. if not is_all_key:
  346. if not is_same_with_lastHead:
  347. head_list.append(h)
  348. head_list.append(height)
  349. return head_list
  350. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  351. set_item = set()
  352. height = len(inner_table)
  353. width = len(inner_table[0])
  354. for i in range(height):
  355. for j in range(width):
  356. item = inner_table[i][j][0]
  357. set_item.add(item)
  358. list_item = list(set_item)
  359. x = []
  360. for item in list_item:
  361. x.append(getPredictor("form").encode(item))
  362. predict_y = getPredictor("form").predict(np.array(x),type="item")
  363. _dict = dict()
  364. for item,values in zip(list_item,list(predict_y)):
  365. _dict[item] = values[1]
  366. # print("##",item,values)
  367. #print(_dict)
  368. for i in range(height):
  369. for j in range(width):
  370. item = inner_table[i][j][0]
  371. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  372. # print("=====")
  373. # for item in inner_table:
  374. # print(item)
  375. # print("======")
  376. repairTable(inner_table)
  377. head_list = sliceTable(inner_table)
  378. return inner_table,head_list
  379. def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  380. data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
  381. predict_y = getPredictor("form").getModel("context").predict(data_x)
  382. for _position,_y in zip(data_position,predict_y):
  383. _w = _position[0]
  384. _h = _position[1]
  385. if _y[1]>prob_min:
  386. inner_table[_h][_w][1] = 1
  387. else:
  388. inner_table[_h][_w][1] = 0
  389. _item = inner_table[_h][_w][0]
  390. if re.search(pat_head,_item) is not None and len(_item)<8:
  391. inner_table[_h][_w][1] = 1
  392. # print("=====")
  393. # for item in inner_table:
  394. # print(item)
  395. # print("======")
  396. height = len(inner_table)
  397. width = len(inner_table[0])
  398. for i in range(height):
  399. for j in range(width):
  400. if re.search("[::]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
  401. inner_table[i][j][1] = 1
  402. repairTable(inner_table)
  403. head_list = sliceTable(inner_table)
  404. # print("inner_table:",inner_table)
  405. return inner_table,head_list
  406. #设置表头
  407. def setHead_inline(inner_table,prob_min=0.64):
  408. pad_row = "@@"
  409. pad_col = "##"
  410. removePadding(inner_table, pad_row, pad_col)
  411. pad_pattern = re.compile(pad_row+"|"+pad_col)
  412. height = len(inner_table)
  413. width = len(inner_table[0])
  414. head_list = []
  415. head_list.append(0)
  416. #行表头
  417. is_head_last = False
  418. for i in range(height):
  419. is_head = False
  420. is_long_value = False
  421. #判断是否是全padding值
  422. is_same_value = True
  423. same_value = inner_table[i][0][0]
  424. for j in range(width):
  425. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  426. is_same_value = False
  427. break
  428. #predict is head or not with model
  429. temp_item = ""
  430. for j in range(width):
  431. temp_item += inner_table[i][j][0]+"|"
  432. temp_item = re.sub(pad_pattern,"",temp_item)
  433. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  434. if form_prob is not None:
  435. if form_prob[0][1]>prob_min:
  436. is_head = True
  437. else:
  438. is_head = False
  439. #print(temp_item,form_prob)
  440. if len(inner_table[i][0][0])>40:
  441. is_long_value = True
  442. if is_head or is_long_value or is_same_value:
  443. #不把连续表头分开
  444. if not is_head_last:
  445. head_list.append(i)
  446. if is_long_value or is_same_value:
  447. head_list.append(i+1)
  448. if is_head:
  449. for j in range(width):
  450. inner_table[i][j][1] = 1
  451. is_head_last = is_head
  452. head_list.append(height)
  453. #列表头
  454. for i in range(len(head_list)-1):
  455. head_begin = head_list[i]
  456. head_end = head_list[i+1]
  457. #最后一列不设置为列表头
  458. for i in range(width-1):
  459. is_head = False
  460. #predict is head or not with model
  461. temp_item = ""
  462. for j in range(head_begin,head_end):
  463. temp_item += inner_table[j][i][0]+"|"
  464. temp_item = re.sub(pad_pattern,"",temp_item)
  465. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  466. if form_prob is not None:
  467. if form_prob[0][1]>prob_min:
  468. is_head = True
  469. else:
  470. is_head = False
  471. if is_head:
  472. for j in range(head_begin,head_end):
  473. inner_table[j][i][1] = 2
  474. addPadding(inner_table, pad_row, pad_col)
  475. return inner_table,head_list
  476. #设置表头
  477. def setHead_withRule(inner_table,pattern,pat_value,count):
  478. height = len(inner_table)
  479. width = len(inner_table[0])
  480. head_list = []
  481. head_list.append(0)
  482. #行表头
  483. is_head_last = False
  484. for i in range(height):
  485. set_match = set()
  486. is_head = False
  487. is_long_value = False
  488. is_same_value = True
  489. same_value = inner_table[i][0][0]
  490. for j in range(width):
  491. if inner_table[i][j][0]!=same_value:
  492. is_same_value = False
  493. break
  494. for j in range(width):
  495. if re.search(pat_value,inner_table[i][j][0]) is not None:
  496. is_head = False
  497. break
  498. str_find = re.findall(pattern,inner_table[i][j][0])
  499. if len(str_find)>0:
  500. set_match.add(inner_table[i][j][0])
  501. if len(set_match)>=count:
  502. is_head = True
  503. if len(inner_table[i][0][0])>40:
  504. is_long_value = True
  505. if is_head or is_long_value or is_same_value:
  506. if not is_head_last:
  507. head_list.append(i)
  508. if is_head:
  509. for j in range(width):
  510. inner_table[i][j][1] = 1
  511. is_head_last = is_head
  512. head_list.append(height)
  513. #列表头
  514. for i in range(len(head_list)-1):
  515. head_begin = head_list[i]
  516. head_end = head_list[i+1]
  517. #最后一列不设置为列表头
  518. for i in range(width-1):
  519. set_match = set()
  520. is_head = False
  521. for j in range(head_begin,head_end):
  522. if re.search(pat_value,inner_table[j][i][0]) is not None:
  523. is_head = False
  524. break
  525. str_find = re.findall(pattern,inner_table[j][i][0])
  526. if len(str_find)>0:
  527. set_match.add(inner_table[j][i][0])
  528. if len(set_match)>=count:
  529. is_head = True
  530. if is_head:
  531. for j in range(head_begin,head_end):
  532. inner_table[j][i][1] = 2
  533. return inner_table,head_list
  534. #取得表格的处理方向
  535. def getDirect(inner_table,begin,end):
  536. '''
  537. column_head = set()
  538. row_head = set()
  539. widths = len(inner_table[0])
  540. for height in range(begin,end):
  541. for width in range(widths):
  542. if inner_table[height][width][1] ==1:
  543. row_head.add(height)
  544. if inner_table[height][width][1] ==2:
  545. column_head.add(width)
  546. company_pattern = re.compile("公司")
  547. if 0 in column_head and begin not in row_head:
  548. return "column"
  549. if 0 in column_head and begin in row_head:
  550. for height in range(begin,end):
  551. count = 0
  552. count_flag = True
  553. for width_index in range(width):
  554. if inner_table[height][width_index][1]==0:
  555. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  556. count += 1
  557. else:
  558. count_flag = False
  559. if count_flag and count>=2:
  560. return "column"
  561. return "row"
  562. '''
  563. count_row_keys = 0
  564. count_column_keys = 0
  565. width = len(inner_table[0])
  566. if begin<end:
  567. for w in range(len(inner_table[begin])):
  568. if inner_table[begin][w][1]!=0:
  569. count_row_keys += 1
  570. for h in range(begin,end):
  571. if inner_table[h][0][1]!=0:
  572. count_column_keys += 1
  573. company_pattern = re.compile("有限(责任)?公司")
  574. for height in range(begin,end):
  575. count_set = set()
  576. count_flag = True
  577. for width_index in range(width):
  578. if inner_table[height][width_index][1]==0:
  579. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  580. count_set.add(inner_table[height][width_index][0])
  581. else:
  582. count_flag = False
  583. if count_flag and len(count_set)>=2:
  584. return "column"
  585. if count_column_keys>count_row_keys:
  586. return "column"
  587. return "row"
  588. #根据表格处理方向生成句子,
  589. def getTableText(inner_table,head_list,key_direct=False):
  590. # packPattern = "(标包|[标包][号段名])"
  591. packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  592. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)" # 2020/11/23 大网站规则,添加序号为排序
  593. entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))"
  594. moneyPattern = "([中投]标|报价)(金额|价)"
  595. height = len(inner_table)
  596. width = len(inner_table[0])
  597. text = ""
  598. for head_i in range(len(head_list)-1):
  599. head_begin = head_list[head_i]
  600. head_end = head_list[head_i+1]
  601. direct = getDirect(inner_table, head_begin, head_end)
  602. #若只有一行,则直接按行读取
  603. if head_end-head_begin==1:
  604. text_line = ""
  605. for i in range(head_begin,head_end):
  606. for w in range(len(inner_table[i])):
  607. if inner_table[i][w][1]==1:
  608. _punctuation = ":"
  609. else:
  610. _punctuation = "," #2021/12/15 统一为中文标点,避免 206893924 国际F座1108,1,009,197.49元
  611. if w>0:
  612. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  613. text_line += inner_table[i][w][0]+_punctuation
  614. else:
  615. text_line += inner_table[i][w][0]+_punctuation
  616. text_line = text_line+"。" if text_line!="" else text_line
  617. text += text_line
  618. else:
  619. #构建一个共现矩阵
  620. table_occurence = []
  621. for i in range(head_begin,head_end):
  622. line_oc = []
  623. for j in range(width):
  624. cell = inner_table[i][j]
  625. line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
  626. table_occurence.append(line_oc)
  627. occu_height = len(table_occurence)
  628. occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
  629. #为每个属性值寻找表头
  630. for i in range(occu_height):
  631. for j in range(occu_width):
  632. cell = table_occurence[i][j]
  633. #是属性值
  634. if cell["type"]==0 and cell["text"]!="":
  635. left_head = ""
  636. top_head = ""
  637. find_flag = False
  638. temp_head = ""
  639. for loop_i in range(1,i+1):
  640. if not key_direct:
  641. key_values = [1,2]
  642. else:
  643. key_values = [1]
  644. if table_occurence[i-loop_i][j]["type"] in key_values:
  645. if find_flag:
  646. if table_occurence[i-loop_i][j]["text"]!=temp_head:
  647. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  648. else:
  649. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  650. find_flag = True
  651. temp_head = table_occurence[i-loop_i][j]["text"]
  652. table_occurence[i-loop_i][j]["occu_count"] += 1
  653. else:
  654. #找到表头后遇到属性值就返回
  655. if find_flag:
  656. break
  657. cell["top_head"] += top_head
  658. find_flag = False
  659. temp_head = ""
  660. for loop_j in range(1,j+1):
  661. if not key_direct:
  662. key_values = [1,2]
  663. else:
  664. key_values = [2]
  665. if table_occurence[i][j-loop_j]["type"] in key_values:
  666. if find_flag:
  667. if table_occurence[i][j-loop_j]["text"]!=temp_head:
  668. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  669. else:
  670. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  671. find_flag = True
  672. temp_head = table_occurence[i][j-loop_j]["text"]
  673. table_occurence[i][j-loop_j]["occu_count"] += 1
  674. else:
  675. if find_flag:
  676. break
  677. cell["left_head"] += left_head
  678. if direct=="row":
  679. for i in range(occu_height):
  680. pack_text = ""
  681. rank_text = ""
  682. entity_text = ""
  683. text_line = ""
  684. money_text = ""
  685. #在同一句话中重复的可以去掉
  686. text_set = set()
  687. for j in range(width):
  688. cell = table_occurence[i][j]
  689. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  690. cell = table_occurence[i][j]
  691. head = (cell["top_head"]+":") if len(cell["top_head"])>0 else ""
  692. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  693. head = cell["left_head"] + head
  694. else:
  695. head += cell["left_head"]
  696. if str(head+cell["text"]) in text_set:
  697. continue
  698. if re.search(packPattern,head) is not None:
  699. pack_text += head+cell["text"]+","
  700. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  701. #排名替换为同一种表达
  702. rank_text += head+cell["text"]+","
  703. #print(rank_text)
  704. elif re.search(entityPattern,head) is not None:
  705. entity_text += head+cell["text"]+","
  706. #print(entity_text)
  707. else:
  708. if re.search(moneyPattern,head) is not None and entity_text!="":
  709. money_text += head+cell["text"]+","
  710. else:
  711. text_line += head+cell["text"]+","
  712. text_set.add(str(head+cell["text"]))
  713. text += pack_text+rank_text+entity_text+money_text+text_line
  714. text = text[:-1]+"。" if len(text)>0 else text
  715. else:
  716. for j in range(occu_width):
  717. pack_text = ""
  718. rank_text = ""
  719. entity_text = ""
  720. text_line = ""
  721. text_set = set()
  722. for i in range(occu_height):
  723. cell = table_occurence[i][j]
  724. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  725. cell = table_occurence[i][j]
  726. head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
  727. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  728. head = cell["top_head"] + head
  729. else:
  730. head += cell["top_head"]
  731. if str(head+cell["text"]) in text_set:
  732. continue
  733. if re.search(packPattern,head) is not None:
  734. pack_text += head+cell["text"]+","
  735. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  736. #排名替换为同一种表达
  737. rank_text += head+cell["text"]+","
  738. #print(rank_text)
  739. elif re.search(entityPattern,head) is not None and \
  740. re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
  741. entity_text += head+cell["text"]+","
  742. #print(entity_text)
  743. else:
  744. text_line += head+cell["text"]+","
  745. text_set.add(str(head+cell["text"]))
  746. text += pack_text+rank_text+entity_text+text_line
  747. text = text[:-1]+"。" if len(text)>0 else text
  748. # if direct=="row":
  749. # for i in range(head_begin,head_end):
  750. # pack_text = ""
  751. # rank_text = ""
  752. # entity_text = ""
  753. # text_line = ""
  754. # #在同一句话中重复的可以去掉
  755. # text_set = set()
  756. # for j in range(width):
  757. # cell = inner_table[i][j]
  758. # #是属性值
  759. # if cell[1]==0 and cell[0]!="":
  760. # head = ""
  761. #
  762. # find_flag = False
  763. # temp_head = ""
  764. # for loop_i in range(0,i+1-head_begin):
  765. # if not key_direct:
  766. # key_values = [1,2]
  767. # else:
  768. # key_values = [1]
  769. # if inner_table[i-loop_i][j][1] in key_values:
  770. # if find_flag:
  771. # if inner_table[i-loop_i][j][0]!=temp_head:
  772. # head = inner_table[i-loop_i][j][0]+":"+head
  773. # else:
  774. # head = inner_table[i-loop_i][j][0]+":"+head
  775. # find_flag = True
  776. # temp_head = inner_table[i-loop_i][j][0]
  777. # else:
  778. # #找到表头后遇到属性值就返回
  779. # if find_flag:
  780. # break
  781. #
  782. # find_flag = False
  783. # temp_head = ""
  784. #
  785. #
  786. #
  787. # for loop_j in range(1,j+1):
  788. # if not key_direct:
  789. # key_values = [1,2]
  790. # else:
  791. # key_values = [2]
  792. # if inner_table[i][j-loop_j][1] in key_values:
  793. # if find_flag:
  794. # if inner_table[i][j-loop_j][0]!=temp_head:
  795. # head = inner_table[i][j-loop_j][0]+":"+head
  796. # else:
  797. # head = inner_table[i][j-loop_j][0]+":"+head
  798. # find_flag = True
  799. # temp_head = inner_table[i][j-loop_j][0]
  800. # else:
  801. # if find_flag:
  802. # break
  803. #
  804. # if str(head+inner_table[i][j][0]) in text_set:
  805. # continue
  806. # if re.search(packPattern,head) is not None:
  807. # pack_text += head+inner_table[i][j][0]+","
  808. # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  809. # #排名替换为同一种表达
  810. # rank_text += head+inner_table[i][j][0]+","
  811. # #print(rank_text)
  812. # elif re.search(entityPattern,head) is not None:
  813. # entity_text += head+inner_table[i][j][0]+","
  814. # #print(entity_text)
  815. # else:
  816. # text_line += head+inner_table[i][j][0]+","
  817. # text_set.add(str(head+inner_table[i][j][0]))
  818. # text += pack_text+rank_text+entity_text+text_line
  819. # text = text[:-1]+"。" if len(text)>0 else text
  820. # else:
  821. # for j in range(width):
  822. #
  823. # rank_text = ""
  824. # entity_text = ""
  825. # text_line = ""
  826. # text_set = set()
  827. # for i in range(head_begin,head_end):
  828. # cell = inner_table[i][j]
  829. # #是属性值
  830. # if cell[1]==0 and cell[0]!="":
  831. # find_flag = False
  832. # head = ""
  833. # temp_head = ""
  834. #
  835. # for loop_j in range(1,j+1):
  836. # if not key_direct:
  837. # key_values = [1,2]
  838. # else:
  839. # key_values = [2]
  840. # if inner_table[i][j-loop_j][1] in key_values:
  841. # if find_flag:
  842. # if inner_table[i][j-loop_j][0]!=temp_head:
  843. # head = inner_table[i][j-loop_j][0]+":"+head
  844. # else:
  845. # head = inner_table[i][j-loop_j][0]+":"+head
  846. # find_flag = True
  847. # temp_head = inner_table[i][j-loop_j][0]
  848. # else:
  849. # if find_flag:
  850. # break
  851. # find_flag = False
  852. # temp_head = ""
  853. # for loop_i in range(0,i+1-head_begin):
  854. # if not key_direct:
  855. # key_values = [1,2]
  856. # else:
  857. # key_values = [1]
  858. # if inner_table[i-loop_i][j][1] in key_values:
  859. # if find_flag:
  860. # if inner_table[i-loop_i][j][0]!=temp_head:
  861. # head = inner_table[i-loop_i][j][0]+":"+head
  862. # else:
  863. # head = inner_table[i-loop_i][j][0]+":"+head
  864. # find_flag = True
  865. # temp_head = inner_table[i-loop_i][j][0]
  866. # else:
  867. # if find_flag:
  868. # break
  869. # if str(head+inner_table[i][j][0]) in text_set:
  870. # continue
  871. # if re.search(rankPattern,head) is not None:
  872. # rank_text += head+inner_table[i][j][0]+","
  873. # #print(rank_text)
  874. # elif re.search(entityPattern,head) is not None:
  875. # entity_text += head+inner_table[i][j][0]+","
  876. # #print(entity_text)
  877. # else:
  878. # text_line += head+inner_table[i][j][0]+","
  879. # text_set.add(str(head+inner_table[i][j][0]))
  880. # text += rank_text+entity_text+text_line
  881. # text = text[:-1]+"。" if len(text)>0 else text
  882. return text
  883. def removeFix(inner_table,fix_value="~~"):
  884. height = len(inner_table)
  885. width = len(inner_table[0])
  886. for h in range(height):
  887. for w in range(width):
  888. if inner_table[h][w][0]==fix_value:
  889. inner_table[h][w][0] = ""
  890. def trunTable(tbody):
  891. fixSpan(tbody)
  892. inner_table = getTable(tbody)
  893. inner_table = fixTable(inner_table)
  894. if len(inner_table)>0 and len(inner_table[0])>0:
  895. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  896. #inner_table,head_list = setHead_inline(inner_table)
  897. # inner_table,head_list = setHead_initem(inner_table,pat_head)
  898. inner_table,head_list = setHead_incontext(inner_table,pat_head)
  899. # print(inner_table)
  900. # for begin in range(len(head_list[:-1])):
  901. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  902. # print(item)
  903. # print("====")
  904. removeFix(inner_table)
  905. # print("----")
  906. # print(head_list)
  907. # for item in inner_table:
  908. # print(item)
  909. tbody.string = getTableText(inner_table,head_list)
  910. #print(tbody.string)
  911. tbody.name = "turntable"
  912. return inner_table
  913. return None
  914. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$')
  915. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  916. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  917. list_innerTable = []
  918. tbodies = soup.find_all('table')
  919. # 遍历表格中的每个tbody
  920. #逆序处理嵌套表格
  921. for tbody_index in range(1,len(tbodies)+1):
  922. tbody = tbodies[len(tbodies)-tbody_index]
  923. inner_table = trunTable(tbody)
  924. list_innerTable.append(inner_table)
  925. tbodies = soup.find_all('tbody')
  926. # 遍历表格中的每个tbody
  927. #逆序处理嵌套表格
  928. for tbody_index in range(1,len(tbodies)+1):
  929. tbody = tbodies[len(tbodies)-tbody_index]
  930. inner_table = trunTable(tbody)
  931. list_innerTable.append(inner_table)
  932. return soup
  933. # return list_innerTable
  934. #数据清洗
  935. def segment(soup,final=True):
  936. # print("==")
  937. # print(soup)
  938. # print("====")
  939. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  940. subspaceList = ["td",'a',"span","p"]
  941. if soup.name in subspaceList:
  942. #判断有值叶子节点数
  943. _count = 0
  944. for child in soup.find_all(recursive=True):
  945. if child.get_text().strip()!="" and len(child.find_all())==0:
  946. _count += 1
  947. if _count<=1:
  948. text = soup.get_text()
  949. # 2020/11/24 大网站规则添加
  950. if 'title' in soup.attrs:
  951. if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']:
  952. text = soup.attrs['title']
  953. _list = []
  954. for x in re.split("\s+",text):
  955. if x.strip()!="":
  956. _list.append(len(x))
  957. if len(_list)>0:
  958. _minLength = min(_list)
  959. if _minLength>2:
  960. _substr = ","
  961. else:
  962. _substr = ""
  963. else:
  964. _substr = ""
  965. text = text.replace("\r\n",",").replace("\n",",")
  966. text = re.sub("\s+",_substr,text)
  967. # text = re.sub("\s+","##space##",text)
  968. return text
  969. segList = ["title"]
  970. commaList = ["div","br","td","p"]
  971. #commaList = []
  972. spaceList = ["span"]
  973. tbodies = soup.find_all('tbody')
  974. if len(tbodies) == 0:
  975. tbodies = soup.find_all('table')
  976. # 递归遍历所有节点,插入符号
  977. for child in soup.find_all(recursive=True):
  978. if child.name in segList:
  979. child.insert_after("。")
  980. if child.name in commaList:
  981. child.insert_after(",")
  982. # if child.name in subspaceList:
  983. # child.insert_before("#subs"+str(child.name)+"#")
  984. # child.insert_after("#sube"+str(child.name)+"#")
  985. # if child.name in spaceList:
  986. # child.insert_after(" ")
  987. text = str(soup.get_text())
  988. #替换英文冒号为中文冒号
  989. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  990. #替换为中文逗号
  991. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  992. #替换为中文分号
  993. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  994. # 感叹号替换为中文句号
  995. text = re.sub("(?<=[\u4e00-\u9fa5])[!!]|[!!](?=[\u4e00-\u9fa5])","。",text)
  996. #替换格式未识别的问号为" " ,update:2021/7/20
  997. text = re.sub("[?\?]{2,}"," ",text)
  998. #替换"""为"“",否则导入deepdive出错
  999. # text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1000. text = text.replace('"',"“").replace("\r","").replace("\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
  1001. # print('==1',text)
  1002. # text = re.sub("\s{4,}",",",text)
  1003. # 解决公告中的" "空格替换问题
  1004. if re.search("\s{4,}",text):
  1005. _text = ""
  1006. for _sent in re.split("。+",text):
  1007. for _sent2 in re.split(',+',_sent):
  1008. for _sent3 in re.split(":+",_sent2):
  1009. for _t in re.split("\s{4,}",_sent3):
  1010. if len(_t)<3:
  1011. _text += _t
  1012. else:
  1013. _text += ","+_t
  1014. _text += ":"
  1015. _text = _text[:-1]
  1016. _text += ","
  1017. _text = _text[:-1]
  1018. _text += "。"
  1019. _text = _text[:-1]
  1020. text = _text
  1021. # print('==2',text)
  1022. #替换标点
  1023. #替换连续的标点
  1024. if final:
  1025. text = re.sub("##space##"," ",text)
  1026. punc_pattern = "(?P<del>[。,;::,\s]+)"
  1027. list_punc = re.findall(punc_pattern,text)
  1028. list_punc.sort(key=lambda x:len(x),reverse=True)
  1029. for punc_del in list_punc:
  1030. if len(punc_del)>1:
  1031. if len(punc_del.strip())>0:
  1032. if ":" in punc_del.strip():
  1033. text = re.sub(punc_del,":",text)
  1034. else:
  1035. text = re.sub(punc_del,punc_del.strip()[0],text) #2021/12/09 修正由于某些标签后插入符号把原来符号替换
  1036. else:
  1037. text = re.sub(punc_del,"",text)
  1038. #将连续的中文句号替换为一个
  1039. text_split = text.split("。")
  1040. text_split = [x for x in text_split if len(x)>0]
  1041. text = "。".join(text_split)
  1042. # #删除标签中的所有空格
  1043. # for subs in subspaceList:
  1044. # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  1045. # while(True):
  1046. # oneMatch = re.search(re.compile(patten),text)
  1047. # if oneMatch is not None:
  1048. # _match = oneMatch.group(1)
  1049. # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
  1050. # else:
  1051. # break
  1052. # text过大报错
  1053. LOOP_LEN = 10000
  1054. LOOP_BEGIN = 0
  1055. _text = ""
  1056. if len(text)<10000000:
  1057. while(LOOP_BEGIN<len(text)):
  1058. _text += re.sub(")",")",re.sub("(","(",re.sub("\s+","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  1059. LOOP_BEGIN += LOOP_LEN
  1060. text = _text
  1061. return text
  1062. '''
  1063. #数据清洗
  1064. def segment(soup):
  1065. segList = ["title"]
  1066. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  1067. spaceList = ["span"]
  1068. tbodies = soup.find_all('tbody')
  1069. if len(tbodies) == 0:
  1070. tbodies = soup.find_all('table')
  1071. # 递归遍历所有节点,插入符号
  1072. for child in soup.find_all(recursive=True):
  1073. if child.name == 'br':
  1074. child.insert_before(',')
  1075. child_text = re.sub('\s', '', child.get_text())
  1076. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  1077. continue
  1078. if child.name in segList:
  1079. child.insert_after("。")
  1080. if child.name in commaList:
  1081. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  1082. child.insert_after(",")
  1083. elif len(child_text) >=50:
  1084. child.insert_after("。")
  1085. #if child.name in spaceList:
  1086. #child.insert_after(" ")
  1087. text = str(soup.get_text())
  1088. text = re.sub("\s{5,}",",",text)
  1089. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1090. #替换"""为"“",否则导入deepdive出错
  1091. text = text.replace('"',"“")
  1092. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  1093. #删除所有空格
  1094. text = re.sub("\s+","#nbsp#",text)
  1095. text_list = text.split('#nbsp#')
  1096. new_text = ''
  1097. for i in range(len(text_list)-1):
  1098. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  1099. new_text += text_list[i]
  1100. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  1101. new_text += text_list[i] + '。'
  1102. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  1103. new_text += text_list[i] + ';'
  1104. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  1105. new_text += text_list[i] + ' '
  1106. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  1107. new_text += text_list[i]
  1108. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  1109. new_text += text_list[i] + ','
  1110. else:
  1111. new_text += text_list[i]
  1112. new_text += text_list[-1]
  1113. text = new_text
  1114. #替换英文冒号为中文冒号
  1115. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1116. #替换为中文逗号
  1117. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1118. #替换为中文分号
  1119. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1120. #替换标点
  1121. while(True):
  1122. #替换连续的标点
  1123. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  1124. if punc is not None:
  1125. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  1126. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  1127. if punc is not None:
  1128. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  1129. else:
  1130. #替换标点之后的空格
  1131. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  1132. if punc is not None:
  1133. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  1134. else:
  1135. break
  1136. #将连续的中文句号替换为一个
  1137. text_split = text.split("。")
  1138. text_split = [x for x in text_split if len(x)>0]
  1139. text = "。".join(text_split)
  1140. #替换中文括号为英文括号
  1141. text = re.sub("(","(",text)
  1142. text = re.sub(")",")",text)
  1143. return text
  1144. '''
  1145. #连续实体合并(弃用)
  1146. def union_ner(list_ner):
  1147. result_list = []
  1148. union_index = []
  1149. union_index_set = set()
  1150. for i in range(len(list_ner)-1):
  1151. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  1152. if list_ner[i][1]-list_ner[i+1][0]==1:
  1153. union_index_set.add(i)
  1154. union_index_set.add(i+1)
  1155. union_index.append((i,i+1))
  1156. for i in range(len(list_ner)):
  1157. if i not in union_index_set:
  1158. result_list.append(list_ner[i])
  1159. for item in union_index:
  1160. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  1161. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  1162. return result_list
  1163. # def get_preprocessed(articles,useselffool=False):
  1164. # '''
  1165. # @summary:预处理步骤,NLP处理、实体识别
  1166. # @param:
  1167. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1168. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1169. # '''
  1170. # list_articles = []
  1171. # list_sentences = []
  1172. # list_entitys = []
  1173. # cost_time = dict()
  1174. # for article in articles:
  1175. # list_sentences_temp = []
  1176. # list_entitys_temp = []
  1177. # doc_id = article[0]
  1178. # sourceContent = article[1]
  1179. # _send_doc_id = article[3]
  1180. # _title = article[4]
  1181. # #表格处理
  1182. # key_preprocess = "tableToText"
  1183. # start_time = time.time()
  1184. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1185. #
  1186. # # log(article_processed)
  1187. #
  1188. # if key_preprocess not in cost_time:
  1189. # cost_time[key_preprocess] = 0
  1190. # cost_time[key_preprocess] += time.time()-start_time
  1191. #
  1192. # #article_processed = article[1]
  1193. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1194. # #nlp处理
  1195. # if article_processed is not None and len(article_processed)!=0:
  1196. # split_patten = "。"
  1197. # sentences = []
  1198. # _begin = 0
  1199. # for _iter in re.finditer(split_patten,article_processed):
  1200. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1201. # _begin = _iter.span()[1]
  1202. # sentences.append(article_processed[_begin:])
  1203. #
  1204. # lemmas = []
  1205. # doc_offsets = []
  1206. # dep_types = []
  1207. # dep_tokens = []
  1208. #
  1209. # time1 = time.time()
  1210. #
  1211. # '''
  1212. # tokens_all = fool.cut(sentences)
  1213. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1214. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1215. # ner_entitys_all = fool.ner(sentences)
  1216. # '''
  1217. # #限流执行
  1218. # key_nerToken = "nerToken"
  1219. # start_time = time.time()
  1220. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1221. # if key_nerToken not in cost_time:
  1222. # cost_time[key_nerToken] = 0
  1223. # cost_time[key_nerToken] += time.time()-start_time
  1224. #
  1225. #
  1226. # for sentence_index in range(len(sentences)):
  1227. #
  1228. #
  1229. #
  1230. # list_sentence_entitys = []
  1231. # sentence_text = sentences[sentence_index]
  1232. # tokens = tokens_all[sentence_index]
  1233. #
  1234. # list_tokenbegin = []
  1235. # begin = 0
  1236. # for i in range(0,len(tokens)):
  1237. # list_tokenbegin.append(begin)
  1238. # begin += len(str(tokens[i]))
  1239. # list_tokenbegin.append(begin+1)
  1240. # #pos_tag = pos_all[sentence_index]
  1241. # pos_tag = ""
  1242. #
  1243. # ner_entitys = ner_entitys_all[sentence_index]
  1244. #
  1245. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1246. #
  1247. # #识别package
  1248. #
  1249. #
  1250. # #识别实体
  1251. # for ner_entity in ner_entitys:
  1252. # begin_index_temp = ner_entity[0]
  1253. # end_index_temp = ner_entity[1]
  1254. # entity_type = ner_entity[2]
  1255. # entity_text = ner_entity[3]
  1256. #
  1257. # for j in range(len(list_tokenbegin)):
  1258. # if list_tokenbegin[j]==begin_index_temp:
  1259. # begin_index = j
  1260. # break
  1261. # elif list_tokenbegin[j]>begin_index_temp:
  1262. # begin_index = j-1
  1263. # break
  1264. # begin_index_temp += len(str(entity_text))
  1265. # for j in range(begin_index,len(list_tokenbegin)):
  1266. # if list_tokenbegin[j]>=begin_index_temp:
  1267. # end_index = j-1
  1268. # break
  1269. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1270. #
  1271. # #去掉标点符号
  1272. # entity_text = re.sub("[,,。:]","",entity_text)
  1273. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1274. #
  1275. #
  1276. # #使用正则识别金额
  1277. # entity_type = "money"
  1278. #
  1279. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1280. #
  1281. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1282. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1283. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1284. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1285. #
  1286. # set_begin = set()
  1287. # for pattern_key in list_money_pattern.keys():
  1288. # pattern = re.compile(list_money_pattern[pattern_key])
  1289. # all_match = re.findall(pattern, sentence_text)
  1290. # index = 0
  1291. # for i in range(len(all_match)):
  1292. # if len(all_match[i][0])>0:
  1293. # # print("===",all_match[i])
  1294. # #print(all_match[i][0])
  1295. # unit = ""
  1296. # entity_text = all_match[i][3]
  1297. # if pattern_key in ["key_word","front_m"]:
  1298. # unit = all_match[i][1]
  1299. # else:
  1300. # unit = all_match[i][4]
  1301. # if entity_text.find("元")>=0:
  1302. # unit = ""
  1303. #
  1304. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1305. #
  1306. # begin_index_temp = index
  1307. # for j in range(len(list_tokenbegin)):
  1308. # if list_tokenbegin[j]==index:
  1309. # begin_index = j
  1310. # break
  1311. # elif list_tokenbegin[j]>index:
  1312. # begin_index = j-1
  1313. # break
  1314. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1315. # end_index_temp = index
  1316. # #index += len(str(all_match[i][0]))
  1317. # for j in range(begin_index,len(list_tokenbegin)):
  1318. # if list_tokenbegin[j]>=index:
  1319. # end_index = j-1
  1320. # break
  1321. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1322. #
  1323. #
  1324. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1325. # if len(unit)>0:
  1326. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1327. # else:
  1328. # entity_text = str(getUnifyMoney(entity_text))
  1329. #
  1330. # _exists = False
  1331. # for item in list_sentence_entitys:
  1332. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1333. # _exists = True
  1334. # if not _exists:
  1335. # if float(entity_text)>10:
  1336. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1337. #
  1338. # else:
  1339. # index += 1
  1340. #
  1341. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1342. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1343. # list_sentences.append(list_sentences_temp)
  1344. # list_entitys.append(list_entitys_temp)
  1345. # return list_articles,list_sentences,list_entitys,cost_time
  1346. def get_preprocessed(articles,useselffool=False):
  1347. '''
  1348. @summary:预处理步骤,NLP处理、实体识别
  1349. @param:
  1350. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1351. @return:list of articles,list of each article of sentences,list of each article of entitys
  1352. '''
  1353. cost_time = dict()
  1354. list_articles = get_preprocessed_article(articles,cost_time)
  1355. list_sentences = get_preprocessed_sentences(list_articles,True,cost_time)
  1356. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1357. calibrateEnterprise(list_articles,list_sentences,list_entitys)
  1358. return list_articles,list_sentences,list_entitys,cost_time
  1359. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  1360. '''
  1361. :param articles: 待处理的article source html
  1362. :param useselffool: 是否使用selffool
  1363. :return: list_articles
  1364. '''
  1365. list_articles = []
  1366. for article in articles:
  1367. doc_id = article[0]
  1368. sourceContent = article[1]
  1369. sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
  1370. _send_doc_id = article[3]
  1371. _title = article[4]
  1372. page_time = article[5]
  1373. #表格处理
  1374. key_preprocess = "tableToText"
  1375. start_time = time.time()
  1376. article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1377. article_processed = article_processed.replace('.','.') # 2021/12/01 修正OCR识别PDF小数点错误问题
  1378. article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
  1379. article_processed = article_processed.replace('成交工程价款', '成交工程价') # 2021/12/21 修正为中标价
  1380. # 提取bidway
  1381. list_bidway = extract_bidway(article_processed, _title)
  1382. if list_bidway:
  1383. bidway = list_bidway[0].get("body")
  1384. # bidway名称统一规范
  1385. bidway = bidway_integrate(bidway)
  1386. else:
  1387. bidway = ""
  1388. # 修正被","逗号分隔的时间
  1389. repair_time = re.compile("[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d|"
  1390. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[:时点],?[0-6]\d分?|"
  1391. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[时点]|"
  1392. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]|"
  1393. "[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d"
  1394. )
  1395. for _time in set(re.findall(repair_time,article_processed)):
  1396. if re.search(",",_time):
  1397. _time2 = re.sub(",", "", _time)
  1398. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time2)
  1399. if item:
  1400. _time2 = _time2.replace(item.group(),item.group() + " ")
  1401. article_processed = article_processed.replace(_time, _time2)
  1402. else:
  1403. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time)
  1404. if item:
  1405. _time2 = _time.replace(item.group(),item.group() + " ")
  1406. article_processed = article_processed.replace(_time, _time2)
  1407. # print('re_rtime',re.findall(repair_time,article_processed))
  1408. # log(article_processed)
  1409. if key_preprocess not in cost_time:
  1410. cost_time[key_preprocess] = 0
  1411. cost_time[key_preprocess] += round(time.time()-start_time,2)
  1412. #article_processed = article[1]
  1413. _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
  1414. bidway=bidway)
  1415. _article.fingerprint = getFingerprint(_title+sourceContent)
  1416. _article.page_time = page_time
  1417. list_articles.append(_article)
  1418. return list_articles
  1419. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  1420. '''
  1421. :param list_articles: 经过预处理的article text
  1422. :return: list_sentences
  1423. '''
  1424. list_sentences = []
  1425. for article in list_articles:
  1426. list_sentences_temp = []
  1427. list_entitys_temp = []
  1428. doc_id = article.id
  1429. _send_doc_id = article.doc_id
  1430. _title = article.title
  1431. #表格处理
  1432. key_preprocess = "tableToText"
  1433. start_time = time.time()
  1434. article_processed = article.content
  1435. if key_preprocess not in cost_time:
  1436. cost_time[key_preprocess] = 0
  1437. cost_time[key_preprocess] += time.time()-start_time
  1438. #nlp处理
  1439. if article_processed is not None and len(article_processed)!=0:
  1440. split_patten = "。"
  1441. sentences = []
  1442. _begin = 0
  1443. sentences_set = set()
  1444. for _iter in re.finditer(split_patten,article_processed):
  1445. _sen = article_processed[_begin:_iter.span()[1]]
  1446. if len(_sen)>0 and _sen not in sentences_set:
  1447. sentences.append(_sen)
  1448. sentences_set.add(_sen)
  1449. _begin = _iter.span()[1]
  1450. _sen = article_processed[_begin:]
  1451. if len(_sen)>0 and _sen not in sentences_set:
  1452. sentences.append(_sen)
  1453. sentences_set.add(_sen)
  1454. article.content = "".join(sentences)
  1455. # sentences.append(article_processed[_begin:])
  1456. lemmas = []
  1457. doc_offsets = []
  1458. dep_types = []
  1459. dep_tokens = []
  1460. time1 = time.time()
  1461. '''
  1462. tokens_all = fool.cut(sentences)
  1463. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1464. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1465. ner_entitys_all = fool.ner(sentences)
  1466. '''
  1467. #限流执行
  1468. key_nerToken = "nerToken"
  1469. start_time = time.time()
  1470. tokens_all = getTokens(sentences,useselffool=useselffool)
  1471. if key_nerToken not in cost_time:
  1472. cost_time[key_nerToken] = 0
  1473. cost_time[key_nerToken] += round(time.time()-start_time,2)
  1474. for sentence_index in range(len(sentences)):
  1475. sentence_text = sentences[sentence_index]
  1476. tokens = tokens_all[sentence_index]
  1477. #pos_tag = pos_all[sentence_index]
  1478. pos_tag = ""
  1479. ner_entitys = ""
  1480. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1481. if len(list_sentences_temp)==0:
  1482. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  1483. list_sentences.append(list_sentences_temp)
  1484. return list_sentences
  1485. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  1486. '''
  1487. :param list_sentences:分局情况
  1488. :param cost_time:
  1489. :return: list_entitys
  1490. '''
  1491. list_entitys = []
  1492. for list_sentence in list_sentences:
  1493. sentences = []
  1494. list_entitys_temp = []
  1495. for _sentence in list_sentence:
  1496. sentences.append(_sentence.sentence_text)
  1497. lemmas = []
  1498. doc_offsets = []
  1499. dep_types = []
  1500. dep_tokens = []
  1501. time1 = time.time()
  1502. '''
  1503. tokens_all = fool.cut(sentences)
  1504. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1505. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1506. ner_entitys_all = fool.ner(sentences)
  1507. '''
  1508. #限流执行
  1509. key_nerToken = "nerToken"
  1510. start_time = time.time()
  1511. found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
  1512. # found_pingbiao = False
  1513. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  1514. if key_nerToken not in cost_time:
  1515. cost_time[key_nerToken] = 0
  1516. cost_time[key_nerToken] += round(time.time()-start_time,2)
  1517. for sentence_index in range(len(list_sentence)):
  1518. list_sentence_entitys = []
  1519. sentence_text = list_sentence[sentence_index].sentence_text
  1520. tokens = list_sentence[sentence_index].tokens
  1521. doc_id = list_sentence[sentence_index].doc_id
  1522. list_tokenbegin = []
  1523. begin = 0
  1524. for i in range(0,len(tokens)):
  1525. list_tokenbegin.append(begin)
  1526. begin += len(str(tokens[i]))
  1527. list_tokenbegin.append(begin+1)
  1528. #pos_tag = pos_all[sentence_index]
  1529. pos_tag = ""
  1530. ner_entitys = ner_entitys_all[sentence_index]
  1531. '''正则识别角色实体 经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
  1532. for it in re.finditer(
  1533. '(?P<text_key_word>[^,。、;《]{,5}(单一来源|中标|中选|中价|成交)?(供应商|供货商|服务商|候选人|单位|人)(名称)?[为::]+)(?P<text>([^,。、;《]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[,。]',
  1534. sentence_text):
  1535. for k, v in it.groupdict().items():
  1536. if k == 'text_key_word':
  1537. keyword = v
  1538. if k == 'text':
  1539. entity = v
  1540. b = it.start() + len(keyword)
  1541. e = it.end() - 1
  1542. if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  1543. ner_entitys.append((b, e, 'company', entity))
  1544. for it in re.finditer(
  1545. '(?P<text_key_word>((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为::]+)(?P<text>\w{2,4}[省市县区镇]([^,。、;《]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[,。]',
  1546. sentence_text):
  1547. for k, v in it.groupdict().items():
  1548. if k == 'text_key_word':
  1549. keyword = v
  1550. if k == 'text':
  1551. entity = v
  1552. b = it.start() + len(keyword)
  1553. e = it.end() - 1
  1554. if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  1555. ner_entitys.append((b, e, 'org', entity))
  1556. #识别package
  1557. #识别实体
  1558. for ner_entity in ner_entitys:
  1559. begin_index_temp = ner_entity[0]
  1560. end_index_temp = ner_entity[1]
  1561. entity_type = ner_entity[2]
  1562. entity_text = ner_entity[3]
  1563. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  1564. continue
  1565. for j in range(len(list_tokenbegin)):
  1566. if list_tokenbegin[j]==begin_index_temp:
  1567. begin_index = j
  1568. break
  1569. elif list_tokenbegin[j]>begin_index_temp:
  1570. begin_index = j-1
  1571. break
  1572. begin_index_temp += len(str(entity_text))
  1573. for j in range(begin_index,len(list_tokenbegin)):
  1574. if list_tokenbegin[j]>=begin_index_temp:
  1575. end_index = j-1
  1576. break
  1577. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1578. #去掉标点符号
  1579. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  1580. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  1581. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1582. # 标记文章末尾的"发布人”、“发布时间”实体
  1583. if sentence_index==len(list_sentence)-1:
  1584. if len(list_sentence_entitys[-2:])>2:
  1585. second2last = list_sentence_entitys[-2]
  1586. last = list_sentence_entitys[-1]
  1587. if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
  1588. second2last.entity_type=="time" and last.entity_type in ["company",'org']):
  1589. if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
  1590. last.is_tail = True
  1591. second2last.is_tail = True
  1592. #使用正则识别金额
  1593. entity_type = "money"
  1594. #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1595. # list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  1596. # "key_word":"((?P<text_key_word>(?:[¥¥]+,?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果:)(?:[,(\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[)\)]?))",
  1597. # "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())",
  1598. # "behind_m":"(()()(?P<money_behind_m>[0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\))]?)"}
  1599. list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  1600. "key_word": "((?P<text_key_word>(?:[¥¥]+,?|[单报标限总]价|金额|成交报?价|价格|预算|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果|成交额|中标额)(?:[,,(\(]*\s*(人民币)?(?P<unit_key_word_before>[万亿]?元?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[::])?(\d+(\*\d+%)+=)?(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]{,1})(?:[(\(]?(?P<filter_>[%])*\s*(单位[::])?(?P<unit_key_word_behind>[万亿]?元?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[)\)]?))",
  1601. "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万亿]?元)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)())",
  1602. "behind_m":"(()()(?P<money_behind_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)(人民币)?[\((]?(?P<unit_behind_m>[万亿]?元(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\))]?)"}
  1603. # 2021/7/19 调整金额,单位提取正则,修复部分金额因为单位提取失败被过滤问题。
  1604. pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
  1605. set_begin = set()
  1606. # for pattern_key in list_money_pattern.keys():
  1607. # for pattern_key in ["cn","key_word","behind_m","front_m"]:
  1608. # # pattern = re.compile(list_money_pattern[pattern_key])
  1609. # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?\s*([万元]*)\s*[)\)]?))*|(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*|((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*")
  1610. # all_match = re.findall(pattern, sentence_text)
  1611. # index = 0
  1612. # for i in range(len(all_match)):
  1613. # if len(all_match[i][0])>0:
  1614. # print("===",all_match[i])
  1615. # #print(all_match[i][0])
  1616. # unit = ""
  1617. # entity_text = all_match[i][3]
  1618. # if pattern_key in ["key_word","front_m"]:
  1619. # unit = all_match[i][1]
  1620. # if pattern_key=="key_word":
  1621. # if all_match[i][1]=="" and all_match[i][4]!="":
  1622. # unit = all_match[i][4]
  1623. # else:
  1624. # unit = all_match[i][4]
  1625. # if entity_text.find("元")>=0:
  1626. # unit = ""
  1627. #
  1628. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1629. # begin_index_temp = index
  1630. # for j in range(len(list_tokenbegin)):
  1631. # if list_tokenbegin[j]==index:
  1632. # begin_index = j
  1633. # break
  1634. # elif list_tokenbegin[j]>index:
  1635. # begin_index = j-1
  1636. # break
  1637. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1638. # end_index_temp = index
  1639. # #index += len(str(all_match[i][0]))
  1640. # for j in range(begin_index,len(list_tokenbegin)):
  1641. # if list_tokenbegin[j]>=index:
  1642. # end_index = j-1
  1643. # break
  1644. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1645. #
  1646. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1647. # if len(unit)>0:
  1648. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1649. # else:
  1650. # entity_text = str(getUnifyMoney(entity_text))
  1651. #
  1652. # _exists = False
  1653. # for item in list_sentence_entitys:
  1654. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1655. # _exists = True
  1656. # if not _exists:
  1657. # if float(entity_text)>1:
  1658. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1659. #
  1660. # else:
  1661. # index += 1
  1662. # if re.search('评标结果|候选人公示', sentence_text):
  1663. # found_pingbiao = True
  1664. if re.search('业绩', sentence_text):
  1665. found_yeji += 1
  1666. if found_yeji >= 2: # 过滤掉业绩后面的所有金额
  1667. all_match = []
  1668. else:
  1669. all_match = re.finditer(pattern_money, sentence_text)
  1670. index = 0
  1671. for _match in all_match:
  1672. if len(_match.group())>0:
  1673. # print("===",_match.group())
  1674. # # print(_match.groupdict())
  1675. notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
  1676. unit = ""
  1677. entity_text = ""
  1678. text_beforeMoney = ""
  1679. filter = ""
  1680. filter_unit = False
  1681. notSure = False
  1682. if re.search('业绩', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额
  1683. # print('金额在业绩后面: ', _match.group(0))
  1684. found_yeji += 1
  1685. break
  1686. for k,v in _match.groupdict().items():
  1687. if v!="" and v is not None:
  1688. if k=='text_key_word':
  1689. notSure = True
  1690. if k.split("_")[0]=="money":
  1691. entity_text = v
  1692. if k.split("_")[0]=="unit":
  1693. unit = v
  1694. if k.split("_")[0]=="text":
  1695. text_beforeMoney = v
  1696. if k.split("_")[0]=="filter":
  1697. filter = v
  1698. if re.search("filter_unit",k) is not None:
  1699. filter_unit = True
  1700. if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()): # 2021/7/19 修正OCR识别小数点为逗号
  1701. if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]):
  1702. entity_text = re.sub('\d+,', '', entity_text)
  1703. else:
  1704. entity_text = entity_text.replace(',', '.')
  1705. # print(' 修正OCR识别小数点为逗号')
  1706. if entity_text.find("元")>=0:
  1707. unit = ""
  1708. if unit == "": #2021/7/21 有明显金额特征的补充单位,避免被过滤
  1709. if ('¥' in text_beforeMoney or '¥' in text_beforeMoney):
  1710. unit = '元'
  1711. # print('明显金额特征补充单位 元')
  1712. elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[::为]+$', text_beforeMoney.strip()) and \
  1713. re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None:
  1714. unit = '元'
  1715. # print('明显金额特征补充单位 元')
  1716. elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7},?)$)|(^\d{,3}(,\d{3})+,?$)',entity_text):
  1717. unit = '元'
  1718. # print('明显金额特征补充单位 元')
  1719. if unit.find("万") >= 0 and entity_text.find("万") >= 0: #2021/7/19修改为金额文本有万,不计算单位
  1720. # print('修正金额及单位都有万, 金额:',entity_text, '单位:',unit)
  1721. unit = "元"
  1722. if re.search('.*万元万元', entity_text): #2021/7/19 修正两个万元
  1723. # print(' 修正两个万元',entity_text)
  1724. entity_text = entity_text.replace('万元万元','万元')
  1725. else:
  1726. if filter_unit:
  1727. continue
  1728. if filter!="":
  1729. continue
  1730. index = _match.span()[0]+len(text_beforeMoney)
  1731. begin_index_temp = index
  1732. for j in range(len(list_tokenbegin)):
  1733. if list_tokenbegin[j]==index:
  1734. begin_index = j
  1735. break
  1736. elif list_tokenbegin[j]>index:
  1737. begin_index = j-1
  1738. break
  1739. index = _match.span()[1]
  1740. end_index_temp = index
  1741. #index += len(str(all_match[i][0]))
  1742. for j in range(begin_index,len(list_tokenbegin)):
  1743. if list_tokenbegin[j]>=index:
  1744. end_index = j-1
  1745. break
  1746. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1747. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
  1748. # print('转换前金额:', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
  1749. if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/8/5过滤掉总投资金额
  1750. # print('总投资金额: ', _match.group(0))
  1751. notes = '总投资'
  1752. elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额
  1753. notes = '投资'
  1754. elif re.search('工程造价', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额
  1755. notes = '工程造价'
  1756. elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
  1757. or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\((]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\))]*[::为]',
  1758. sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
  1759. or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
  1760. sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
  1761. notes = '保证金'
  1762. # print('保证金信息:', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
  1763. elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
  1764. sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
  1765. notes = '成本警戒线'
  1766. elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为:]', sentence_text[_match.span()[0]:_match.span()[1]]):
  1767. cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
  1768. notes = cost_re.group(1)
  1769. elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
  1770. notes = '单价'
  1771. elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
  1772. notes = '大写'
  1773. if entity_text[0] == "拾": # 2021/12/16 修正大写金额省略了数字转换错误问题
  1774. entity_text = "壹"+entity_text
  1775. # print("补充备注:notes = 大写")
  1776. if len(unit)>0:
  1777. if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况
  1778. # print('修正单位万元金额过大的情况 金额:', entity_text, '单位:', unit)
  1779. entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000)
  1780. unit = '元' # 修正金额后单位 重置为元
  1781. else:
  1782. # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
  1783. entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1784. else:
  1785. if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8:
  1786. entity_text = str(getUnifyMoney(entity_text)/10000)
  1787. # print('修正金额字段含万 过大的情况')
  1788. else:
  1789. entity_text = str(getUnifyMoney(entity_text))
  1790. if float(entity_text)<100 or float(entity_text)>100000000000:
  1791. # print('过滤掉金额:float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
  1792. continue
  1793. if notSure and unit=="" and float(entity_text)>100*10000:
  1794. # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000:', entity_text, unit)
  1795. continue
  1796. _exists = False
  1797. for item in list_sentence_entitys:
  1798. if item.entity_id==entity_id and item.entity_type==entity_type:
  1799. _exists = True
  1800. if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index):
  1801. _exists = True
  1802. if not _exists:
  1803. if float(entity_text)>1:
  1804. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1805. list_sentence_entitys[-1].notes = notes # 2021/7/20 新增金额备注
  1806. list_sentence_entitys[-1].money_unit = unit # 2021/7/20 新增金额备注
  1807. # print('预处理中的 金额:%s, 单位:%s'%(entity_text,unit))
  1808. else:
  1809. index += 1
  1810. # "联系人"正则补充提取 2021/11/15 新增
  1811. list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
  1812. error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
  1813. '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理','代理人','采购','附件']
  1814. list_person_text = set(list_person_text + error_text)
  1815. re_person = re.compile("联系人[::]([\u4e00-\u9fa5]工)|"
  1816. "联系人[::]([\u4e00-\u9fa5]{2,3})(?=联系)|"
  1817. "联系人[::]([\u4e00-\u9fa5]{2,3})")
  1818. list_person = []
  1819. for match_result in re_person.finditer(sentence_text):
  1820. match_text = match_result.group()
  1821. entity_text = match_text[4:]
  1822. wordOffset_begin = match_result.start() + 4
  1823. wordOffset_end = match_result.end()
  1824. # print(text[wordOffset_begin:wordOffset_end])
  1825. # 排除一些不为人名的实体
  1826. if re.search("^[\u4e00-\u9fa5]{7,}([,。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
  1827. continue
  1828. if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
  1829. _person = dict()
  1830. _person['body'] = entity_text
  1831. _person['begin_index'] = wordOffset_begin
  1832. _person['end_index'] = wordOffset_end
  1833. list_person.append(_person)
  1834. entity_type = "person"
  1835. for person in list_person:
  1836. begin_index_temp = person['begin_index']
  1837. for j in range(len(list_tokenbegin)):
  1838. if list_tokenbegin[j] == begin_index_temp:
  1839. begin_index = j
  1840. break
  1841. elif list_tokenbegin[j] > begin_index_temp:
  1842. begin_index = j - 1
  1843. break
  1844. index = person['end_index']
  1845. end_index_temp = index
  1846. for j in range(begin_index, len(list_tokenbegin)):
  1847. if list_tokenbegin[j] >= index:
  1848. end_index = j - 1
  1849. break
  1850. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1851. entity_text = person['body']
  1852. list_sentence_entitys.append(
  1853. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1854. begin_index_temp, end_index_temp))
  1855. # 资金来源提取 2020/12/30 新增
  1856. list_moneySource = extract_moneySource(sentence_text)
  1857. entity_type = "moneysource"
  1858. for moneySource in list_moneySource:
  1859. begin_index_temp = moneySource['begin_index']
  1860. for j in range(len(list_tokenbegin)):
  1861. if list_tokenbegin[j] == begin_index_temp:
  1862. begin_index = j
  1863. break
  1864. elif list_tokenbegin[j] > begin_index_temp:
  1865. begin_index = j - 1
  1866. break
  1867. index = moneySource['end_index']
  1868. end_index_temp = index
  1869. for j in range(begin_index, len(list_tokenbegin)):
  1870. if list_tokenbegin[j] >= index:
  1871. end_index = j - 1
  1872. break
  1873. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1874. entity_text = moneySource['body']
  1875. list_sentence_entitys.append(
  1876. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1877. begin_index_temp, end_index_temp))
  1878. # 电子邮箱提取 2021/11/04 新增
  1879. list_email = extract_email(sentence_text)
  1880. entity_type = "email" # 电子邮箱
  1881. for email in list_email:
  1882. begin_index_temp = email['begin_index']
  1883. for j in range(len(list_tokenbegin)):
  1884. if list_tokenbegin[j] == begin_index_temp:
  1885. begin_index = j
  1886. break
  1887. elif list_tokenbegin[j] > begin_index_temp:
  1888. begin_index = j - 1
  1889. break
  1890. index = email['end_index']
  1891. end_index_temp = index
  1892. for j in range(begin_index, len(list_tokenbegin)):
  1893. if list_tokenbegin[j] >= index:
  1894. end_index = j - 1
  1895. break
  1896. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1897. entity_text = email['body']
  1898. list_sentence_entitys.append(
  1899. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1900. begin_index_temp, end_index_temp))
  1901. # 服务期限提取 2020/12/30 新增
  1902. list_servicetime = extract_servicetime(sentence_text)
  1903. entity_type = "serviceTime"
  1904. for servicetime in list_servicetime:
  1905. begin_index_temp = servicetime['begin_index']
  1906. for j in range(len(list_tokenbegin)):
  1907. if list_tokenbegin[j] == begin_index_temp:
  1908. begin_index = j
  1909. break
  1910. elif list_tokenbegin[j] > begin_index_temp:
  1911. begin_index = j - 1
  1912. break
  1913. index = servicetime['end_index']
  1914. end_index_temp = index
  1915. for j in range(begin_index, len(list_tokenbegin)):
  1916. if list_tokenbegin[j] >= index:
  1917. end_index = j - 1
  1918. break
  1919. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1920. entity_text = servicetime['body']
  1921. list_sentence_entitys.append(
  1922. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1923. begin_index_temp, end_index_temp))
  1924. # 招标方式提取 2020/12/30 新增
  1925. # list_bidway = extract_bidway(sentence_text, )
  1926. # entity_type = "bidway"
  1927. # for bidway in list_bidway:
  1928. # begin_index_temp = bidway['begin_index']
  1929. # end_index_temp = bidway['end_index']
  1930. # begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
  1931. # end_index = changeIndexFromWordToWords(tokens, end_index_temp)
  1932. # if begin_index is None or end_index is None:
  1933. # continue
  1934. # print(begin_index_temp,end_index_temp,begin_index,end_index)
  1935. # entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1936. # entity_text = bidway['body']
  1937. # list_sentence_entitys.append(
  1938. # Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1939. # begin_index_temp, end_index_temp))
  1940. # 2021/12/29 新增比率提取
  1941. list_ratio = extract_ratio(sentence_text)
  1942. entity_type = "ratio"
  1943. for ratio in list_ratio:
  1944. # print("ratio", ratio)
  1945. begin_index_temp = ratio['begin_index']
  1946. for j in range(len(list_tokenbegin)):
  1947. if list_tokenbegin[j] == begin_index_temp:
  1948. begin_index = j
  1949. break
  1950. elif list_tokenbegin[j] > begin_index_temp:
  1951. begin_index = j - 1
  1952. break
  1953. index = ratio['end_index']
  1954. end_index_temp = index
  1955. for j in range(begin_index, len(list_tokenbegin)):
  1956. if list_tokenbegin[j] >= index:
  1957. end_index = j - 1
  1958. break
  1959. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1960. entity_text = ratio['body']
  1961. list_sentence_entitys.append(
  1962. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1963. begin_index_temp, end_index_temp))
  1964. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1965. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1966. list_entitys.append(list_entitys_temp)
  1967. return list_entitys
  1968. def union_result(codeName,prem):
  1969. '''
  1970. @summary:模型的结果拼成字典
  1971. @param:
  1972. codeName:编号名称模型的结果字典
  1973. prem:拿到属性的角色的字典
  1974. @return:拼接起来的字典
  1975. '''
  1976. result = []
  1977. assert len(codeName)==len(prem)
  1978. for item_code,item_prem in zip(codeName,prem):
  1979. result.append(dict(item_code,**item_prem))
  1980. return result
  1981. def persistenceData(data):
  1982. '''
  1983. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  1984. '''
  1985. import psycopg2
  1986. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  1987. cursor = conn.cursor()
  1988. for item_index in range(len(data)):
  1989. item = data[item_index]
  1990. doc_id = item[0]
  1991. dic = item[1]
  1992. code = dic['code']
  1993. name = dic['name']
  1994. prem = dic['prem']
  1995. if len(code)==0:
  1996. code_insert = ""
  1997. else:
  1998. code_insert = ";".join(code)
  1999. prem_insert = ""
  2000. for item in prem:
  2001. for x in item:
  2002. if isinstance(x, list):
  2003. if len(x)>0:
  2004. for x1 in x:
  2005. prem_insert+="/".join(x1)+","
  2006. prem_insert+="$"
  2007. else:
  2008. prem_insert+=str(x)+"$"
  2009. prem_insert+=";"
  2010. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  2011. cursor.execute(sql)
  2012. conn.commit()
  2013. conn.close()
  2014. def persistenceData1(list_entitys,list_sentences):
  2015. '''
  2016. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  2017. '''
  2018. import psycopg2
  2019. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  2020. cursor = conn.cursor()
  2021. for list_entity in list_entitys:
  2022. for entity in list_entity:
  2023. if entity.values is not None:
  2024. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  2025. else:
  2026. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  2027. cursor.execute(sql)
  2028. for list_sentence in list_sentences:
  2029. for sentence in list_sentence:
  2030. str_tokens = "["
  2031. for item in sentence.tokens:
  2032. str_tokens += "'"
  2033. if item=="'":
  2034. str_tokens += "''"
  2035. else:
  2036. str_tokens += item
  2037. str_tokens += "',"
  2038. str_tokens = str_tokens[:-1]+"]"
  2039. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  2040. cursor.execute(sql)
  2041. conn.commit()
  2042. conn.close()
  2043. def _handle(item,result_queue):
  2044. dochtml = item["dochtml"]
  2045. docid = item["docid"]
  2046. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  2047. flag = False
  2048. if list_innerTable:
  2049. flag = True
  2050. for table in list_innerTable:
  2051. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  2052. def getPredictTable():
  2053. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  2054. import pandas as pd
  2055. import json
  2056. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  2057. df = pd.read_csv(filename)
  2058. df_data = {"json_table":[],"docid":[]}
  2059. _count = 0
  2060. _sum = len(df["docid"])
  2061. task_queue = Queue()
  2062. result_queue = Queue()
  2063. _index = 0
  2064. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  2065. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  2066. _index += 1
  2067. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  2068. mh.run()
  2069. while True:
  2070. try:
  2071. item = result_queue.get(block=True,timeout=1)
  2072. df_data["docid"].append(item["docid"])
  2073. df_data["json_table"].append(item["json_table"])
  2074. except Exception as e:
  2075. print(e)
  2076. break
  2077. df_1 = pd.DataFrame(df_data)
  2078. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  2079. if __name__=="__main__":
  2080. '''
  2081. import glob
  2082. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  2083. file_txt = str(file).replace("html","txt")
  2084. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  2085. f.write("\n================\n")
  2086. content = codecs.open(file,"r",encoding="utf8").read()
  2087. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2088. '''
  2089. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  2090. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2091. # getPredictTable()
  2092. with open('D:/138786703.html', 'r', encoding='utf-8') as f:
  2093. sourceContent = f.read()
  2094. # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
  2095. # print(article_processed)
  2096. list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
  2097. for entity in list_entitys[0]:
  2098. print(entity.entity_type, entity.entity_text)