Preprocessing.py 144 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909
  1. # -*- coding: utf-8 -*-
  2. from bs4 import BeautifulSoup, Comment
  3. import copy
  4. import sys
  5. import os
  6. import time
  7. import codecs
  8. from BiddingKG.dl.ratio.re_ratio import extract_ratio
  9. from BiddingKG.dl.table_head.predict import predict
  10. sys.setrecursionlimit(1000000)
  11. sys.path.append(os.path.abspath("../.."))
  12. sys.path.append(os.path.abspath(".."))
  13. from BiddingKG.dl.common.Utils import *
  14. from BiddingKG.dl.interface.Entitys import *
  15. from BiddingKG.dl.interface.predictor import getPredictor
  16. from BiddingKG.dl.common.nerUtils import *
  17. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  18. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  19. from BiddingKG.dl.relation_extraction.re_email import extract_email
  20. from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
  21. from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
  22. from BiddingKG.dl.entityLink.entityLink import *
  23. #
  24. def tableToText(soup):
  25. '''
  26. @param:
  27. soup:网页html的soup
  28. @return:处理完表格信息的网页text
  29. '''
  30. def getTrs(tbody):
  31. #获取所有的tr
  32. trs = []
  33. objs = tbody.find_all(recursive=False)
  34. for obj in objs:
  35. if obj.name=="tr":
  36. trs.append(obj)
  37. if obj.name=="tbody":
  38. for tr in obj.find_all("tr",recursive=False):
  39. trs.append(tr)
  40. return trs
  41. def fixSpan(tbody):
  42. # 处理colspan, rowspan信息补全问题
  43. #trs = tbody.findChildren('tr', recursive=False)
  44. trs = getTrs(tbody)
  45. ths_len = 0
  46. ths = list()
  47. trs_set = set()
  48. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  49. # 遍历每一个tr
  50. for indtr, tr in enumerate(trs):
  51. ths_tmp = tr.findChildren('th', recursive=False)
  52. #不补全含有表格的tr
  53. if len(tr.findChildren('table'))>0:
  54. continue
  55. if len(ths_tmp) > 0:
  56. ths_len = ths_len + len(ths_tmp)
  57. for th in ths_tmp:
  58. ths.append(th)
  59. trs_set.add(tr)
  60. # 遍历每行中的element
  61. tds = tr.findChildren(recursive=False)
  62. for indtd, td in enumerate(tds):
  63. # 若有colspan 则补全同一行下一个位置
  64. if 'colspan' in td.attrs:
  65. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  66. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  67. if col<100 and len(td.get_text())<1000:
  68. td['colspan'] = 1
  69. for i in range(1, col, 1):
  70. td.insert_after(copy.copy(td))
  71. for indtr, tr in enumerate(trs):
  72. ths_tmp = tr.findChildren('th', recursive=False)
  73. #不补全含有表格的tr
  74. if len(tr.findChildren('table'))>0:
  75. continue
  76. if len(ths_tmp) > 0:
  77. ths_len = ths_len + len(ths_tmp)
  78. for th in ths_tmp:
  79. ths.append(th)
  80. trs_set.add(tr)
  81. # 遍历每行中的element
  82. tds = tr.findChildren(recursive=False)
  83. for indtd, td in enumerate(tds):
  84. # 若有rowspan 则补全下一行同样位置
  85. if 'rowspan' in td.attrs:
  86. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  87. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  88. td['rowspan'] = 1
  89. for i in range(1, row, 1):
  90. # 获取下一行的所有td, 在对应的位置插入
  91. if indtr+i<len(trs):
  92. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  93. if len(tds1) >= (indtd) and len(tds1)>0:
  94. if indtd > 0:
  95. tds1[indtd - 1].insert_after(copy.copy(td))
  96. else:
  97. tds1[0].insert_before(copy.copy(td))
  98. elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全
  99. tds1[indtd-2].insert_after(copy.copy(td))
  100. def getTable(tbody):
  101. #trs = tbody.findChildren('tr', recursive=False)
  102. trs = getTrs(tbody)
  103. inner_table = []
  104. for tr in trs:
  105. tr_line = []
  106. tds = tr.findChildren(['td','th'], recursive=False)
  107. if len(tds)==0:
  108. tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
  109. for td in tds:
  110. tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
  111. #tr_line.append([td.get_text(),0])
  112. inner_table.append(tr_line)
  113. return inner_table
  114. #处理表格不对齐的问题
  115. def fixTable(inner_table,fix_value="~~"):
  116. maxWidth = 0
  117. for item in inner_table:
  118. if len(item)>maxWidth:
  119. maxWidth = len(item)
  120. for i in range(len(inner_table)):
  121. if len(inner_table[i])<maxWidth:
  122. for j in range(maxWidth-len(inner_table[i])):
  123. inner_table[i].append([fix_value,0])
  124. return inner_table
  125. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  126. height = len(inner_table)
  127. width = len(inner_table[0])
  128. for i in range(height):
  129. point = ""
  130. for j in range(width):
  131. if inner_table[i][j][0]==point and point!="":
  132. inner_table[i][j][0] = pad_row
  133. else:
  134. if inner_table[i][j][0] not in [pad_row,pad_col]:
  135. point = inner_table[i][j][0]
  136. for j in range(width):
  137. point = ""
  138. for i in range(height):
  139. if inner_table[i][j][0]==point and point!="":
  140. inner_table[i][j][0] = pad_col
  141. else:
  142. if inner_table[i][j][0] not in [pad_row,pad_col]:
  143. point = inner_table[i][j][0]
  144. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  145. height = len(inner_table)
  146. width = len(inner_table[0])
  147. for i in range(height):
  148. for j in range(width):
  149. if inner_table[i][j][0]==pad_row:
  150. inner_table[i][j][0] = inner_table[i][j-1][0]
  151. inner_table[i][j][1] = inner_table[i][j-1][1]
  152. if inner_table[i][j][0]==pad_col:
  153. inner_table[i][j][0] = inner_table[i-1][j][0]
  154. inner_table[i][j][1] = inner_table[i-1][j][1]
  155. def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
  156. '''
  157. @summary: 修复表头识别,将明显错误的进行修正
  158. '''
  159. def repairNeeded(line):
  160. first_1 = -1
  161. last_1 = -1
  162. first_0 = -1
  163. last_0 = -1
  164. count_1 = 0
  165. count_0 = 0
  166. for i in range(len(line)):
  167. if line[i][0]==fix_value:
  168. continue
  169. if line[i][1]==1:
  170. if first_1==-1:
  171. first_1 = i
  172. last_1 = i
  173. count_1 += 1
  174. if line[i][1]==0:
  175. if first_0 == -1:
  176. first_0 = i
  177. last_0 = i
  178. count_0 += 1
  179. if first_1 ==-1 or last_0 == -1:
  180. return False
  181. #异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  182. if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
  183. return True
  184. return False
  185. def getsimilarity(line,line1):
  186. same_count = 0
  187. for item,item1 in zip(line,line1):
  188. if item[1]==item1[1]:
  189. same_count += 1
  190. return same_count/len(line)
  191. def selfrepair(inner_table,index,dye_set,key_set):
  192. '''
  193. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  194. '''
  195. #print("B",inner_table[index])
  196. min_presure = 3
  197. list_dye = []
  198. first = None
  199. count = 0
  200. temp_set = set()
  201. _index = 0
  202. for item in inner_table[index]:
  203. if first is None:
  204. first = item[1]
  205. if item[0] not in temp_set:
  206. count += 1
  207. temp_set.add(item[0])
  208. else:
  209. if first == item[1]:
  210. if item[0] not in temp_set:
  211. temp_set.add(item[0])
  212. count += 1
  213. else:
  214. list_dye.append([first,count,_index])
  215. first = item[1]
  216. temp_set.add(item[0])
  217. count = 1
  218. _index += 1
  219. list_dye.append([first,count,_index])
  220. if len(list_dye)>1:
  221. begin = 0
  222. end = 0
  223. for i in range(len(list_dye)):
  224. end = list_dye[i][2]
  225. dye_flag = False
  226. #首尾要求压力减一
  227. if i==0:
  228. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  229. dye_flag = True
  230. dye_type = list_dye[i+1][0]
  231. elif i==len(list_dye)-1:
  232. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  233. dye_flag = True
  234. dye_type = list_dye[i-1][0]
  235. else:
  236. if list_dye[i][1]>1:
  237. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  238. dye_flag = True
  239. dye_type = list_dye[i+1][0]
  240. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  241. dye_flag = True
  242. dye_type = list_dye[i-1][0]
  243. else:
  244. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  245. dye_flag = True
  246. dye_type = list_dye[i+1][0]
  247. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  248. dye_flag = True
  249. dye_type = list_dye[i-1][0]
  250. if dye_flag:
  251. for h in range(begin,end):
  252. inner_table[index][h][1] = dye_type
  253. dye_set.add((inner_table[index][h][0],dye_type))
  254. key_set.add(inner_table[index][h][0])
  255. begin = end
  256. #print("E",inner_table[index])
  257. def otherrepair(inner_table,index,dye_set,key_set):
  258. list_provide_repair = []
  259. if index==0 and len(inner_table)>1:
  260. list_provide_repair.append(index+1)
  261. elif index==len(inner_table)-1:
  262. list_provide_repair.append(index-1)
  263. else:
  264. list_provide_repair.append(index+1)
  265. list_provide_repair.append(index-1)
  266. for provide_index in list_provide_repair:
  267. if not repairNeeded(inner_table[provide_index]):
  268. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  269. if same_prob>=0.8:
  270. for i in range(len(inner_table[provide_index])):
  271. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  272. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  273. key_set.add(inner_table[index][i][0])
  274. inner_table[index][i][1] = inner_table[provide_index][i][1]
  275. elif same_prob<=0.2:
  276. for i in range(len(inner_table[provide_index])):
  277. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  278. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  279. key_set.add(inner_table[index][i][0])
  280. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  281. len_dye_set = len(dye_set)
  282. height = len(inner_table)
  283. for i in range(height):
  284. if repairNeeded(inner_table[i]):
  285. selfrepair(inner_table,i,dye_set,key_set)
  286. #otherrepair(inner_table,i,dye_set,key_set)
  287. for h in range(len(inner_table)):
  288. for w in range(len(inner_table[0])):
  289. if inner_table[h][w][0] in key_set:
  290. for item in dye_set:
  291. if inner_table[h][w][0]==item[0]:
  292. inner_table[h][w][1] = item[1]
  293. #如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  294. if len(dye_set)!=len(key_set):
  295. for i in range(height):
  296. if repairNeeded(inner_table[i]):
  297. selfrepair(inner_table,i,dye_set,key_set)
  298. #otherrepair(inner_table,i,dye_set,key_set)
  299. return
  300. if len(dye_set)==len_dye_set:
  301. '''
  302. for i in range(height):
  303. if repairNeeded(inner_table[i]):
  304. otherrepair(inner_table,i,dye_set,key_set)
  305. '''
  306. return
  307. repairTable(inner_table, dye_set, key_set)
  308. def sliceTable(inner_table,fix_value="~~"):
  309. #进行分块
  310. height = len(inner_table)
  311. width = len(inner_table[0])
  312. head_list = []
  313. head_list.append(0)
  314. last_head = None
  315. last_is_same_value = False
  316. for h in range(height):
  317. is_all_key = True#是否是全表头行
  318. is_all_value = True#是否是全属性值
  319. is_same_with_lastHead = True#和上一行的结构是否相同
  320. is_same_value=True#一行的item都一样
  321. #is_same_first_item = True#与上一行的第一项是否相同
  322. same_value = inner_table[h][0][0]
  323. for w in range(width):
  324. if last_head is not None:
  325. if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
  326. is_all_key = False
  327. if inner_table[h][w][0]==1:
  328. is_all_value = False
  329. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  330. is_same_with_lastHead = False
  331. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  332. is_same_value = False
  333. else:
  334. if re.search("\d+",same_value) is not None:
  335. is_same_value = False
  336. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  337. is_same_first_item = False
  338. last_head = h
  339. if last_is_same_value:
  340. last_is_same_value = is_same_value
  341. continue
  342. if is_same_value:
  343. head_list.append(h)
  344. last_is_same_value = is_same_value
  345. continue
  346. if not is_all_key:
  347. if not is_same_with_lastHead:
  348. head_list.append(h)
  349. head_list.append(height)
  350. return head_list
  351. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  352. set_item = set()
  353. height = len(inner_table)
  354. width = len(inner_table[0])
  355. empty_set = set()
  356. for i in range(height):
  357. for j in range(width):
  358. item = inner_table[i][j][0]
  359. if item.strip()=="":
  360. empty_set.add(item)
  361. else:
  362. set_item.add(item)
  363. list_item = list(set_item)
  364. if list_item:
  365. x = []
  366. for item in list_item:
  367. x.append(getPredictor("form").encode(item))
  368. predict_y = getPredictor("form").predict(np.array(x),type="item")
  369. _dict = dict()
  370. for item,values in zip(list_item,list(predict_y)):
  371. _dict[item] = values[1]
  372. # print("##",item,values)
  373. #print(_dict)
  374. for i in range(height):
  375. for j in range(width):
  376. item = inner_table[i][j][0]
  377. if item not in empty_set:
  378. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  379. # print("=====")
  380. # for item in inner_table:
  381. # print(item)
  382. # print("======")
  383. repairTable(inner_table)
  384. head_list = sliceTable(inner_table)
  385. return inner_table,head_list
  386. def set_head_model(inner_table):
  387. for i in range(len(inner_table)):
  388. for j in range(len(inner_table[i])):
  389. # 删掉单格前后符号,以免影响表头预测
  390. col = inner_table[i][j][0]
  391. col = re.sub("^[^\u4e00-\u9fa5a-zA-Z0-9]+", "", col)
  392. col = re.sub("[^\u4e00-\u9fa5a-zA-Z0-9]+$", "", col)
  393. inner_table[i][j] = col
  394. # 模型预测表头
  395. predict_list = predict(inner_table)
  396. # with open(r"C:\Users\Administrator\Desktop\table_head_test.txt", "a") as f:
  397. # for i in range(len(predict_list)):
  398. # f.write(str(i) + " " + str(inner_table[i]) + "\n")
  399. # f.write(str(i) + " " + str(predict_list[i]) + "\n")
  400. # f.write("\n")
  401. # print("table_list", inner_table)
  402. # print("predict_list", predict_list)
  403. for i in range(len(inner_table)):
  404. for j in range(len(inner_table[i])):
  405. inner_table[i][j] = [inner_table[i][j], int(predict_list[i][j])]
  406. head_list = sliceTable(inner_table)
  407. return inner_table, head_list
  408. def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  409. data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
  410. predict_y = getPredictor("form").getModel("context").predict(data_x)
  411. for _position,_y in zip(data_position,predict_y):
  412. _w = _position[0]
  413. _h = _position[1]
  414. if _y[1]>prob_min:
  415. inner_table[_h][_w][1] = 1
  416. else:
  417. inner_table[_h][_w][1] = 0
  418. _item = inner_table[_h][_w][0]
  419. if re.search(pat_head,_item) is not None and len(_item)<8:
  420. inner_table[_h][_w][1] = 1
  421. # print("=====")
  422. # for item in inner_table:
  423. # print(item)
  424. # print("======")
  425. height = len(inner_table)
  426. width = len(inner_table[0])
  427. for i in range(height):
  428. for j in range(width):
  429. if re.search("[::]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
  430. inner_table[i][j][1] = 1
  431. repairTable(inner_table)
  432. head_list = sliceTable(inner_table)
  433. # print("inner_table:",inner_table)
  434. return inner_table,head_list
  435. #设置表头
  436. def setHead_inline(inner_table,prob_min=0.64):
  437. pad_row = "@@"
  438. pad_col = "##"
  439. removePadding(inner_table, pad_row, pad_col)
  440. pad_pattern = re.compile(pad_row+"|"+pad_col)
  441. height = len(inner_table)
  442. width = len(inner_table[0])
  443. head_list = []
  444. head_list.append(0)
  445. #行表头
  446. is_head_last = False
  447. for i in range(height):
  448. is_head = False
  449. is_long_value = False
  450. #判断是否是全padding值
  451. is_same_value = True
  452. same_value = inner_table[i][0][0]
  453. for j in range(width):
  454. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  455. is_same_value = False
  456. break
  457. #predict is head or not with model
  458. temp_item = ""
  459. for j in range(width):
  460. temp_item += inner_table[i][j][0]+"|"
  461. temp_item = re.sub(pad_pattern,"",temp_item)
  462. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  463. if form_prob is not None:
  464. if form_prob[0][1]>prob_min:
  465. is_head = True
  466. else:
  467. is_head = False
  468. #print(temp_item,form_prob)
  469. if len(inner_table[i][0][0])>40:
  470. is_long_value = True
  471. if is_head or is_long_value or is_same_value:
  472. #不把连续表头分开
  473. if not is_head_last:
  474. head_list.append(i)
  475. if is_long_value or is_same_value:
  476. head_list.append(i+1)
  477. if is_head:
  478. for j in range(width):
  479. inner_table[i][j][1] = 1
  480. is_head_last = is_head
  481. head_list.append(height)
  482. #列表头
  483. for i in range(len(head_list)-1):
  484. head_begin = head_list[i]
  485. head_end = head_list[i+1]
  486. #最后一列不设置为列表头
  487. for i in range(width-1):
  488. is_head = False
  489. #predict is head or not with model
  490. temp_item = ""
  491. for j in range(head_begin,head_end):
  492. temp_item += inner_table[j][i][0]+"|"
  493. temp_item = re.sub(pad_pattern,"",temp_item)
  494. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  495. if form_prob is not None:
  496. if form_prob[0][1]>prob_min:
  497. is_head = True
  498. else:
  499. is_head = False
  500. if is_head:
  501. for j in range(head_begin,head_end):
  502. inner_table[j][i][1] = 2
  503. addPadding(inner_table, pad_row, pad_col)
  504. return inner_table,head_list
  505. #设置表头
  506. def setHead_withRule(inner_table,pattern,pat_value,count):
  507. height = len(inner_table)
  508. width = len(inner_table[0])
  509. head_list = []
  510. head_list.append(0)
  511. #行表头
  512. is_head_last = False
  513. for i in range(height):
  514. set_match = set()
  515. is_head = False
  516. is_long_value = False
  517. is_same_value = True
  518. same_value = inner_table[i][0][0]
  519. for j in range(width):
  520. if inner_table[i][j][0]!=same_value:
  521. is_same_value = False
  522. break
  523. for j in range(width):
  524. if re.search(pat_value,inner_table[i][j][0]) is not None:
  525. is_head = False
  526. break
  527. str_find = re.findall(pattern,inner_table[i][j][0])
  528. if len(str_find)>0:
  529. set_match.add(inner_table[i][j][0])
  530. if len(set_match)>=count:
  531. is_head = True
  532. if len(inner_table[i][0][0])>40:
  533. is_long_value = True
  534. if is_head or is_long_value or is_same_value:
  535. if not is_head_last:
  536. head_list.append(i)
  537. if is_head:
  538. for j in range(width):
  539. inner_table[i][j][1] = 1
  540. is_head_last = is_head
  541. head_list.append(height)
  542. #列表头
  543. for i in range(len(head_list)-1):
  544. head_begin = head_list[i]
  545. head_end = head_list[i+1]
  546. #最后一列不设置为列表头
  547. for i in range(width-1):
  548. set_match = set()
  549. is_head = False
  550. for j in range(head_begin,head_end):
  551. if re.search(pat_value,inner_table[j][i][0]) is not None:
  552. is_head = False
  553. break
  554. str_find = re.findall(pattern,inner_table[j][i][0])
  555. if len(str_find)>0:
  556. set_match.add(inner_table[j][i][0])
  557. if len(set_match)>=count:
  558. is_head = True
  559. if is_head:
  560. for j in range(head_begin,head_end):
  561. inner_table[j][i][1] = 2
  562. return inner_table,head_list
  563. #取得表格的处理方向
  564. def getDirect(inner_table,begin,end):
  565. '''
  566. column_head = set()
  567. row_head = set()
  568. widths = len(inner_table[0])
  569. for height in range(begin,end):
  570. for width in range(widths):
  571. if inner_table[height][width][1] ==1:
  572. row_head.add(height)
  573. if inner_table[height][width][1] ==2:
  574. column_head.add(width)
  575. company_pattern = re.compile("公司")
  576. if 0 in column_head and begin not in row_head:
  577. return "column"
  578. if 0 in column_head and begin in row_head:
  579. for height in range(begin,end):
  580. count = 0
  581. count_flag = True
  582. for width_index in range(width):
  583. if inner_table[height][width_index][1]==0:
  584. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  585. count += 1
  586. else:
  587. count_flag = False
  588. if count_flag and count>=2:
  589. return "column"
  590. return "row"
  591. '''
  592. count_row_keys = 0
  593. count_column_keys = 0
  594. width = len(inner_table[0])
  595. if begin<end:
  596. for w in range(len(inner_table[begin])):
  597. if inner_table[begin][w][1]!=0:
  598. count_row_keys += 1
  599. for h in range(begin,end):
  600. if inner_table[h][0][1]!=0:
  601. count_column_keys += 1
  602. company_pattern = re.compile("有限(责任)?公司")
  603. for height in range(begin,end):
  604. count_set = set()
  605. count_flag = True
  606. for width_index in range(width):
  607. if inner_table[height][width_index][1]==0:
  608. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  609. count_set.add(inner_table[height][width_index][0])
  610. else:
  611. count_flag = False
  612. if count_flag and len(count_set)>=2:
  613. return "column"
  614. # if count_column_keys>count_row_keys: #2022/2/15 此项不够严谨,造成很多错误,故取消
  615. # return "column"
  616. return "row"
  617. #根据表格处理方向生成句子,
  618. def getTableText(inner_table,head_list,key_direct=False):
  619. # packPattern = "(标包|[标包][号段名])"
  620. packPattern = "(标包|标的|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  621. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)" # 2020/11/23 大网站规则,添加序号为排序
  622. entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))"
  623. moneyPattern = "([中投]标|报价)(金额|价)"
  624. height = len(inner_table)
  625. width = len(inner_table[0])
  626. text = ""
  627. for head_i in range(len(head_list)-1):
  628. head_begin = head_list[head_i]
  629. head_end = head_list[head_i+1]
  630. direct = getDirect(inner_table, head_begin, head_end)
  631. #若只有一行,则直接按行读取
  632. if head_end-head_begin==1:
  633. text_line = ""
  634. for i in range(head_begin,head_end):
  635. for w in range(len(inner_table[i])):
  636. if inner_table[i][w][1]==1:
  637. _punctuation = ":"
  638. else:
  639. _punctuation = "," #2021/12/15 统一为中文标点,避免 206893924 国际F座1108,1,009,197.49元
  640. if w>0:
  641. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  642. text_line += inner_table[i][w][0]+_punctuation
  643. else:
  644. text_line += inner_table[i][w][0]+_punctuation
  645. text_line = text_line+"。" if text_line!="" else text_line
  646. text += text_line
  647. else:
  648. #构建一个共现矩阵
  649. table_occurence = []
  650. for i in range(head_begin,head_end):
  651. line_oc = []
  652. for j in range(width):
  653. cell = inner_table[i][j]
  654. line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
  655. table_occurence.append(line_oc)
  656. occu_height = len(table_occurence)
  657. occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
  658. #为每个属性值寻找表头
  659. for i in range(occu_height):
  660. for j in range(occu_width):
  661. cell = table_occurence[i][j]
  662. #是属性值
  663. if cell["type"]==0 and cell["text"]!="":
  664. left_head = ""
  665. top_head = ""
  666. find_flag = False
  667. temp_head = ""
  668. for loop_i in range(1,i+1):
  669. if not key_direct:
  670. key_values = [1,2]
  671. else:
  672. key_values = [1]
  673. if table_occurence[i-loop_i][j]["type"] in key_values:
  674. if find_flag:
  675. if table_occurence[i-loop_i][j]["text"]!=temp_head:
  676. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  677. else:
  678. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  679. find_flag = True
  680. temp_head = table_occurence[i-loop_i][j]["text"]
  681. table_occurence[i-loop_i][j]["occu_count"] += 1
  682. else:
  683. #找到表头后遇到属性值就返回
  684. if find_flag:
  685. break
  686. cell["top_head"] += top_head
  687. find_flag = False
  688. temp_head = ""
  689. for loop_j in range(1,j+1):
  690. if not key_direct:
  691. key_values = [1,2]
  692. else:
  693. key_values = [2]
  694. if table_occurence[i][j-loop_j]["type"] in key_values:
  695. if find_flag:
  696. if table_occurence[i][j-loop_j]["text"]!=temp_head:
  697. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  698. else:
  699. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  700. find_flag = True
  701. temp_head = table_occurence[i][j-loop_j]["text"]
  702. table_occurence[i][j-loop_j]["occu_count"] += 1
  703. else:
  704. if find_flag:
  705. break
  706. cell["left_head"] += left_head
  707. if direct=="row":
  708. for i in range(occu_height):
  709. pack_text = ""
  710. rank_text = ""
  711. entity_text = ""
  712. text_line = ""
  713. money_text = ""
  714. #在同一句话中重复的可以去掉
  715. text_set = set()
  716. for j in range(width):
  717. cell = table_occurence[i][j]
  718. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  719. cell = table_occurence[i][j]
  720. head = (cell["top_head"]+":") if len(cell["top_head"])>0 else ""
  721. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  722. head = cell["left_head"] + head
  723. else:
  724. head += cell["left_head"]
  725. if str(head+cell["text"]) in text_set:
  726. continue
  727. if re.search(packPattern,head) is not None:
  728. pack_text += head+cell["text"]+","
  729. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  730. #排名替换为同一种表达
  731. rank_text += head+cell["text"]+","
  732. #print(rank_text)
  733. elif re.search(entityPattern,head) is not None:
  734. entity_text += head+cell["text"]+","
  735. #print(entity_text)
  736. else:
  737. if re.search(moneyPattern,head) is not None and entity_text!="":
  738. money_text += head+cell["text"]+","
  739. else:
  740. text_line += head+cell["text"]+","
  741. text_set.add(str(head+cell["text"]))
  742. text += pack_text+rank_text+entity_text+money_text+text_line
  743. text = text[:-1]+"。" if len(text)>0 else text
  744. else:
  745. for j in range(occu_width):
  746. pack_text = ""
  747. rank_text = ""
  748. entity_text = ""
  749. text_line = ""
  750. text_set = set()
  751. for i in range(occu_height):
  752. cell = table_occurence[i][j]
  753. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  754. cell = table_occurence[i][j]
  755. head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
  756. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  757. head = cell["top_head"] + head
  758. else:
  759. head += cell["top_head"]
  760. if str(head+cell["text"]) in text_set:
  761. continue
  762. if re.search(packPattern,head) is not None:
  763. pack_text += head+cell["text"]+","
  764. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  765. #排名替换为同一种表达
  766. rank_text += head+cell["text"]+","
  767. #print(rank_text)
  768. elif re.search(entityPattern,head) is not None and \
  769. re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
  770. entity_text += head+cell["text"]+","
  771. #print(entity_text)
  772. else:
  773. text_line += head+cell["text"]+","
  774. text_set.add(str(head+cell["text"]))
  775. text += pack_text+rank_text+entity_text+text_line
  776. text = text[:-1]+"。" if len(text)>0 else text
  777. # if direct=="row":
  778. # for i in range(head_begin,head_end):
  779. # pack_text = ""
  780. # rank_text = ""
  781. # entity_text = ""
  782. # text_line = ""
  783. # #在同一句话中重复的可以去掉
  784. # text_set = set()
  785. # for j in range(width):
  786. # cell = inner_table[i][j]
  787. # #是属性值
  788. # if cell[1]==0 and cell[0]!="":
  789. # head = ""
  790. #
  791. # find_flag = False
  792. # temp_head = ""
  793. # for loop_i in range(0,i+1-head_begin):
  794. # if not key_direct:
  795. # key_values = [1,2]
  796. # else:
  797. # key_values = [1]
  798. # if inner_table[i-loop_i][j][1] in key_values:
  799. # if find_flag:
  800. # if inner_table[i-loop_i][j][0]!=temp_head:
  801. # head = inner_table[i-loop_i][j][0]+":"+head
  802. # else:
  803. # head = inner_table[i-loop_i][j][0]+":"+head
  804. # find_flag = True
  805. # temp_head = inner_table[i-loop_i][j][0]
  806. # else:
  807. # #找到表头后遇到属性值就返回
  808. # if find_flag:
  809. # break
  810. #
  811. # find_flag = False
  812. # temp_head = ""
  813. #
  814. #
  815. #
  816. # for loop_j in range(1,j+1):
  817. # if not key_direct:
  818. # key_values = [1,2]
  819. # else:
  820. # key_values = [2]
  821. # if inner_table[i][j-loop_j][1] in key_values:
  822. # if find_flag:
  823. # if inner_table[i][j-loop_j][0]!=temp_head:
  824. # head = inner_table[i][j-loop_j][0]+":"+head
  825. # else:
  826. # head = inner_table[i][j-loop_j][0]+":"+head
  827. # find_flag = True
  828. # temp_head = inner_table[i][j-loop_j][0]
  829. # else:
  830. # if find_flag:
  831. # break
  832. #
  833. # if str(head+inner_table[i][j][0]) in text_set:
  834. # continue
  835. # if re.search(packPattern,head) is not None:
  836. # pack_text += head+inner_table[i][j][0]+","
  837. # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  838. # #排名替换为同一种表达
  839. # rank_text += head+inner_table[i][j][0]+","
  840. # #print(rank_text)
  841. # elif re.search(entityPattern,head) is not None:
  842. # entity_text += head+inner_table[i][j][0]+","
  843. # #print(entity_text)
  844. # else:
  845. # text_line += head+inner_table[i][j][0]+","
  846. # text_set.add(str(head+inner_table[i][j][0]))
  847. # text += pack_text+rank_text+entity_text+text_line
  848. # text = text[:-1]+"。" if len(text)>0 else text
  849. # else:
  850. # for j in range(width):
  851. #
  852. # rank_text = ""
  853. # entity_text = ""
  854. # text_line = ""
  855. # text_set = set()
  856. # for i in range(head_begin,head_end):
  857. # cell = inner_table[i][j]
  858. # #是属性值
  859. # if cell[1]==0 and cell[0]!="":
  860. # find_flag = False
  861. # head = ""
  862. # temp_head = ""
  863. #
  864. # for loop_j in range(1,j+1):
  865. # if not key_direct:
  866. # key_values = [1,2]
  867. # else:
  868. # key_values = [2]
  869. # if inner_table[i][j-loop_j][1] in key_values:
  870. # if find_flag:
  871. # if inner_table[i][j-loop_j][0]!=temp_head:
  872. # head = inner_table[i][j-loop_j][0]+":"+head
  873. # else:
  874. # head = inner_table[i][j-loop_j][0]+":"+head
  875. # find_flag = True
  876. # temp_head = inner_table[i][j-loop_j][0]
  877. # else:
  878. # if find_flag:
  879. # break
  880. # find_flag = False
  881. # temp_head = ""
  882. # for loop_i in range(0,i+1-head_begin):
  883. # if not key_direct:
  884. # key_values = [1,2]
  885. # else:
  886. # key_values = [1]
  887. # if inner_table[i-loop_i][j][1] in key_values:
  888. # if find_flag:
  889. # if inner_table[i-loop_i][j][0]!=temp_head:
  890. # head = inner_table[i-loop_i][j][0]+":"+head
  891. # else:
  892. # head = inner_table[i-loop_i][j][0]+":"+head
  893. # find_flag = True
  894. # temp_head = inner_table[i-loop_i][j][0]
  895. # else:
  896. # if find_flag:
  897. # break
  898. # if str(head+inner_table[i][j][0]) in text_set:
  899. # continue
  900. # if re.search(rankPattern,head) is not None:
  901. # rank_text += head+inner_table[i][j][0]+","
  902. # #print(rank_text)
  903. # elif re.search(entityPattern,head) is not None:
  904. # entity_text += head+inner_table[i][j][0]+","
  905. # #print(entity_text)
  906. # else:
  907. # text_line += head+inner_table[i][j][0]+","
  908. # text_set.add(str(head+inner_table[i][j][0]))
  909. # text += rank_text+entity_text+text_line
  910. # text = text[:-1]+"。" if len(text)>0 else text
  911. return text
  912. def removeFix(inner_table,fix_value="~~"):
  913. height = len(inner_table)
  914. width = len(inner_table[0])
  915. for h in range(height):
  916. for w in range(width):
  917. if inner_table[h][w][0]==fix_value:
  918. inner_table[h][w][0] = ""
  919. def trunTable(tbody,in_attachment):
  920. # print(tbody.find('tbody'))
  921. # 附件中的表格,排除异常错乱的表格
  922. if in_attachment:
  923. if tbody.name=='table':
  924. _tbody = tbody.find('tbody')
  925. if _tbody is None:
  926. _tbody = tbody
  927. else:
  928. _tbody = tbody
  929. _td_len_list = []
  930. for _tr in _tbody.find_all(recursive=False):
  931. len_td = len(_tr.find_all(recursive=False))
  932. _td_len_list.append(len_td)
  933. if len(list(set(_td_len_list)))>8:
  934. return None
  935. fixSpan(tbody)
  936. inner_table = getTable(tbody)
  937. inner_table = fixTable(inner_table)
  938. if len(inner_table)>0 and len(inner_table[0])>0:
  939. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  940. #inner_table,head_list = setHead_inline(inner_table)
  941. # inner_table, head_list = setHead_initem(inner_table,pat_head)
  942. inner_table, head_list = set_head_model(inner_table)
  943. # inner_table,head_list = setHead_incontext(inner_table,pat_head)
  944. # print("table_head", inner_table)
  945. # for begin in range(len(head_list[:-1])):
  946. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  947. # print(item)
  948. # print("====")
  949. removeFix(inner_table)
  950. # print("----")
  951. # print(head_list)
  952. # for item in inner_table:
  953. # print(item)
  954. tbody.string = getTableText(inner_table,head_list)
  955. table_max_len = 30000
  956. tbody.string = tbody.string[:table_max_len]
  957. #print(tbody.string)
  958. tbody.name = "turntable"
  959. return inner_table
  960. return None
  961. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标|(需求|服务|项目|施工|采购|招租|出租|转让|出让|业主|询价|委托|权属|招标|竞得|抽取|承建)(人|方|单位)(名称)?|(供应商|供货商|服务商)(名称)?)$')
  962. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  963. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  964. list_innerTable = []
  965. # 2022/2/9 删除干扰标签
  966. for tag in soup.find_all('option'): #例子: 216661412
  967. if 'selected' not in tag.attrs:
  968. tag.extract()
  969. for ul in soup.find_all('ul'): #例子 156439663 多个不同channel 类别的标题
  970. if ul.find_all('li') == ul.findChildren(recursive=False) and len(set(re.findall(
  971. '招标公告|中标结果公示|中标候选人公示|招标答疑|开标评标|合同履?约?公示|开标评标|资格评审',
  972. ul.get_text(), re.S)))>3:
  973. ul.extract()
  974. # tbodies = soup.find_all('table')
  975. # 遍历表格中的每个tbody
  976. tbodies = []
  977. in_attachment = False
  978. for _part in soup.find_all():
  979. if _part.name=='table':
  980. tbodies.append((_part,in_attachment))
  981. elif _part.name=='div':
  982. if 'class' in _part.attrs and "richTextFetch" in _part['class']:
  983. in_attachment = True
  984. #逆序处理嵌套表格
  985. for tbody_index in range(1,len(tbodies)+1):
  986. tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
  987. inner_table = trunTable(tbody,_in_attachment)
  988. list_innerTable.append(inner_table)
  989. # tbodies = soup.find_all('tbody')
  990. # 遍历表格中的每个tbody
  991. tbodies = []
  992. in_attachment = False
  993. for _part in soup.find_all():
  994. if _part.name == 'tbody':
  995. tbodies.append((_part, in_attachment))
  996. elif _part.name == 'div':
  997. if 'class' in _part.attrs and "richTextFetch" in _part['class']:
  998. in_attachment = True
  999. #逆序处理嵌套表格
  1000. for tbody_index in range(1,len(tbodies)+1):
  1001. tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
  1002. inner_table = trunTable(tbody,_in_attachment)
  1003. list_innerTable.append(inner_table)
  1004. return soup
  1005. # return list_innerTable
  1006. re_num = re.compile("[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十]")
  1007. num_dict = {
  1008. "一": 1, "二": 2,
  1009. "三": 3, "四": 4,
  1010. "五": 5, "六": 6,
  1011. "七": 7, "八": 8,
  1012. "九": 9, "十": 10}
  1013. # 一百以内的中文大写转换为数字
  1014. def change2num(text):
  1015. result_num = -1
  1016. # text = text[:6]
  1017. match = re_num.search(text)
  1018. if match:
  1019. _num = match.group()
  1020. if num_dict.get(_num):
  1021. return num_dict.get(_num)
  1022. else:
  1023. tenths = 1
  1024. the_unit = 0
  1025. num_split = _num.split("十")
  1026. if num_dict.get(num_split[0]):
  1027. tenths = num_dict.get(num_split[0])
  1028. if num_dict.get(num_split[1]):
  1029. the_unit = num_dict.get(num_split[1])
  1030. result_num = tenths * 10 + the_unit
  1031. elif re.search("\d{1,2}",text):
  1032. _num = re.search("\d{1,2}",text).group()
  1033. result_num = int(_num)
  1034. return result_num
  1035. #大纲分段处理
  1036. def get_preprocessed_outline(soup):
  1037. pattern_0 = re.compile("^(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[、.\.]")
  1038. pattern_1 = re.compile("^[\((]?(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[\))]")
  1039. pattern_2 = re.compile("^\d{1,2}[、.\.](?=[^\d]{1,2}|$)")
  1040. pattern_3 = re.compile("^[\((]?\d{1,2}[\))]")
  1041. pattern_list = [pattern_0, pattern_1, pattern_2, pattern_3]
  1042. body = soup.find("body")
  1043. body_child = body.find_all(recursive=False)
  1044. deal_part = body
  1045. # print(body_child[0]['id'])
  1046. if 'id' in body_child[0].attrs:
  1047. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1048. deal_part = body_child[0]
  1049. if len(deal_part.find_all(recursive=False))>2:
  1050. deal_part = deal_part.parent
  1051. skip_tag = ['turntable', 'tbody', 'th', 'tr', 'td', 'table','thead','tfoot']
  1052. for part in deal_part.find_all(recursive=False):
  1053. # 查找解析文本的主干部分
  1054. is_main_text = False
  1055. through_text_num = 0
  1056. while (not is_main_text and part.find_all(recursive=False)):
  1057. while len(part.find_all(recursive=False)) == 1 and part.get_text(strip=True) == \
  1058. part.find_all(recursive=False)[0].get_text(strip=True):
  1059. part = part.find_all(recursive=False)[0]
  1060. max_len = len(part.get_text(strip=True))
  1061. is_main_text = True
  1062. for t_part in part.find_all(recursive=False):
  1063. if t_part.name not in skip_tag and t_part.get_text(strip=True)!="":
  1064. through_text_num += 1
  1065. if t_part.get_text(strip=True)!="" and len(t_part.get_text(strip=True))/max_len>=0.65:
  1066. if t_part.name not in skip_tag:
  1067. is_main_text = False
  1068. part = t_part
  1069. break
  1070. else:
  1071. while len(t_part.find_all(recursive=False)) == 1 and t_part.get_text(strip=True) == \
  1072. t_part.find_all(recursive=False)[0].get_text(strip=True):
  1073. t_part = t_part.find_all(recursive=False)[0]
  1074. if through_text_num>2:
  1075. is_table = True
  1076. for _t_part in t_part.find_all(recursive=False):
  1077. if _t_part.name not in skip_tag:
  1078. is_table = False
  1079. break
  1080. if not is_table:
  1081. is_main_text = False
  1082. part = t_part
  1083. break
  1084. else:
  1085. is_main_text = False
  1086. part = t_part
  1087. break
  1088. is_find = False
  1089. for _pattern in pattern_list:
  1090. last_index = 0
  1091. handle_list = []
  1092. for _part in part.find_all(recursive=False):
  1093. if _part.name not in skip_tag and _part.get_text(strip=True) != "":
  1094. # print('text:', _part.get_text(strip=True))
  1095. re_match = re.search(_pattern, _part.get_text(strip=True))
  1096. if re_match:
  1097. outline_index = change2num(re_match.group())
  1098. if last_index < outline_index:
  1099. # _part.insert_before("##split##")
  1100. handle_list.append(_part)
  1101. last_index = outline_index
  1102. if len(handle_list)>1:
  1103. is_find = True
  1104. for _part in handle_list:
  1105. _part.insert_before("##split##")
  1106. if is_find:
  1107. break
  1108. # print(soup)
  1109. return soup
  1110. #数据清洗
  1111. def segment(soup,final=True):
  1112. # print("==")
  1113. # print(soup)
  1114. # print("====")
  1115. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  1116. subspaceList = ["td",'a',"span","p"]
  1117. if soup.name in subspaceList:
  1118. #判断有值叶子节点数
  1119. _count = 0
  1120. for child in soup.find_all(recursive=True):
  1121. if child.get_text().strip()!="" and len(child.find_all())==0:
  1122. _count += 1
  1123. if _count<=1:
  1124. text = soup.get_text()
  1125. # 2020/11/24 大网站规则添加
  1126. if 'title' in soup.attrs:
  1127. if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']:
  1128. text = soup.attrs['title']
  1129. _list = []
  1130. for x in re.split("\s+",text):
  1131. if x.strip()!="":
  1132. _list.append(len(x))
  1133. if len(_list)>0:
  1134. _minLength = min(_list)
  1135. if _minLength>2:
  1136. _substr = ","
  1137. else:
  1138. _substr = ""
  1139. else:
  1140. _substr = ""
  1141. text = text.replace("\r\n",",").replace("\n",",")
  1142. text = re.sub("\s+",_substr,text)
  1143. # text = re.sub("\s+","##space##",text)
  1144. return text
  1145. segList = ["title"]
  1146. commaList = ["div","br","td","p","li"]
  1147. #commaList = []
  1148. spaceList = ["span"]
  1149. tbodies = soup.find_all('tbody')
  1150. if len(tbodies) == 0:
  1151. tbodies = soup.find_all('table')
  1152. # 递归遍历所有节点,插入符号
  1153. for child in soup.find_all(recursive=True):
  1154. # print(child.name,child.get_text())
  1155. if child.name in segList:
  1156. child.insert_after("。")
  1157. if child.name in commaList:
  1158. child.insert_after(",")
  1159. # if child.name == 'div' and 'class' in child.attrs:
  1160. # # 添加附件"attachment"标识
  1161. # if "richTextFetch" in child['class']:
  1162. # child.insert_before("##attachment##")
  1163. # print(child.parent)
  1164. # if child.name in subspaceList:
  1165. # child.insert_before("#subs"+str(child.name)+"#")
  1166. # child.insert_after("#sube"+str(child.name)+"#")
  1167. # if child.name in spaceList:
  1168. # child.insert_after(" ")
  1169. text = str(soup.get_text())
  1170. #替换英文冒号为中文冒号
  1171. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1172. #替换为中文逗号
  1173. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1174. #替换为中文分号
  1175. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1176. # 感叹号替换为中文句号
  1177. text = re.sub("(?<=[\u4e00-\u9fa5])[!!]|[!!](?=[\u4e00-\u9fa5])","。",text)
  1178. #替换格式未识别的问号为" " ,update:2021/7/20
  1179. text = re.sub("[?\?]{2,}"," ",text)
  1180. #替换"""为"“",否则导入deepdive出错
  1181. # text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1182. text = text.replace('"',"“").replace("\r","").replace("\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
  1183. # print('==1',text)
  1184. # text = re.sub("\s{4,}",",",text)
  1185. # 解决公告中的" "空格替换问题
  1186. if re.search("\s{4,}",text):
  1187. _text = ""
  1188. for _sent in re.split("。+",text):
  1189. for _sent2 in re.split(',+',_sent):
  1190. for _sent3 in re.split(":+",_sent2):
  1191. for _t in re.split("\s{4,}",_sent3):
  1192. if len(_t)<3:
  1193. _text += _t
  1194. else:
  1195. _text += ","+_t
  1196. _text += ":"
  1197. _text = _text[:-1]
  1198. _text += ","
  1199. _text = _text[:-1]
  1200. _text += "。"
  1201. _text = _text[:-1]
  1202. text = _text
  1203. # print('==2',text)
  1204. #替换标点
  1205. #替换连续的标点
  1206. if final:
  1207. text = re.sub("##space##"," ",text)
  1208. punc_pattern = "(?P<del>[。,;::,\s]+)"
  1209. list_punc = re.findall(punc_pattern,text)
  1210. list_punc.sort(key=lambda x:len(x),reverse=True)
  1211. for punc_del in list_punc:
  1212. if len(punc_del)>1:
  1213. if len(punc_del.strip())>0:
  1214. if ":" in punc_del.strip():
  1215. if "。" in punc_del.strip():
  1216. text = re.sub(punc_del, ":。", text)
  1217. else:
  1218. text = re.sub(punc_del,":",text)
  1219. else:
  1220. text = re.sub(punc_del,punc_del.strip()[0],text) #2021/12/09 修正由于某些标签后插入符号把原来符号替换
  1221. else:
  1222. text = re.sub(punc_del,"",text)
  1223. #将连续的中文句号替换为一个
  1224. text_split = text.split("。")
  1225. text_split = [x for x in text_split if len(x)>0]
  1226. text = "。".join(text_split)
  1227. # #删除标签中的所有空格
  1228. # for subs in subspaceList:
  1229. # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  1230. # while(True):
  1231. # oneMatch = re.search(re.compile(patten),text)
  1232. # if oneMatch is not None:
  1233. # _match = oneMatch.group(1)
  1234. # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
  1235. # else:
  1236. # break
  1237. # text过大报错
  1238. LOOP_LEN = 10000
  1239. LOOP_BEGIN = 0
  1240. _text = ""
  1241. if len(text)<10000000:
  1242. while(LOOP_BEGIN<len(text)):
  1243. _text += re.sub(")",")",re.sub("(","(",re.sub("\s+","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  1244. LOOP_BEGIN += LOOP_LEN
  1245. text = _text
  1246. # 附件标识前修改为句号,避免正文和附件内容混合在一起
  1247. text = re.sub("[^。](?=##attachment##)","。",text)
  1248. text = re.sub("[^。](?=##attachment_begin##)","。",text)
  1249. text = re.sub("[^。](?=##attachment_end##)","。",text)
  1250. text = re.sub("##attachment_begin##。","##attachment_begin##",text)
  1251. text = re.sub("##attachment_end##。","##attachment_end##",text)
  1252. return text
  1253. '''
  1254. #数据清洗
  1255. def segment(soup):
  1256. segList = ["title"]
  1257. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  1258. spaceList = ["span"]
  1259. tbodies = soup.find_all('tbody')
  1260. if len(tbodies) == 0:
  1261. tbodies = soup.find_all('table')
  1262. # 递归遍历所有节点,插入符号
  1263. for child in soup.find_all(recursive=True):
  1264. if child.name == 'br':
  1265. child.insert_before(',')
  1266. child_text = re.sub('\s', '', child.get_text())
  1267. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  1268. continue
  1269. if child.name in segList:
  1270. child.insert_after("。")
  1271. if child.name in commaList:
  1272. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  1273. child.insert_after(",")
  1274. elif len(child_text) >=50:
  1275. child.insert_after("。")
  1276. #if child.name in spaceList:
  1277. #child.insert_after(" ")
  1278. text = str(soup.get_text())
  1279. text = re.sub("\s{5,}",",",text)
  1280. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1281. #替换"""为"“",否则导入deepdive出错
  1282. text = text.replace('"',"“")
  1283. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  1284. #删除所有空格
  1285. text = re.sub("\s+","#nbsp#",text)
  1286. text_list = text.split('#nbsp#')
  1287. new_text = ''
  1288. for i in range(len(text_list)-1):
  1289. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  1290. new_text += text_list[i]
  1291. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  1292. new_text += text_list[i] + '。'
  1293. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  1294. new_text += text_list[i] + ';'
  1295. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  1296. new_text += text_list[i] + ' '
  1297. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  1298. new_text += text_list[i]
  1299. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  1300. new_text += text_list[i] + ','
  1301. else:
  1302. new_text += text_list[i]
  1303. new_text += text_list[-1]
  1304. text = new_text
  1305. #替换英文冒号为中文冒号
  1306. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1307. #替换为中文逗号
  1308. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1309. #替换为中文分号
  1310. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1311. #替换标点
  1312. while(True):
  1313. #替换连续的标点
  1314. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  1315. if punc is not None:
  1316. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  1317. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  1318. if punc is not None:
  1319. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  1320. else:
  1321. #替换标点之后的空格
  1322. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  1323. if punc is not None:
  1324. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  1325. else:
  1326. break
  1327. #将连续的中文句号替换为一个
  1328. text_split = text.split("。")
  1329. text_split = [x for x in text_split if len(x)>0]
  1330. text = "。".join(text_split)
  1331. #替换中文括号为英文括号
  1332. text = re.sub("(","(",text)
  1333. text = re.sub(")",")",text)
  1334. return text
  1335. '''
  1336. #连续实体合并(弃用)
  1337. def union_ner(list_ner):
  1338. result_list = []
  1339. union_index = []
  1340. union_index_set = set()
  1341. for i in range(len(list_ner)-1):
  1342. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  1343. if list_ner[i][1]-list_ner[i+1][0]==1:
  1344. union_index_set.add(i)
  1345. union_index_set.add(i+1)
  1346. union_index.append((i,i+1))
  1347. for i in range(len(list_ner)):
  1348. if i not in union_index_set:
  1349. result_list.append(list_ner[i])
  1350. for item in union_index:
  1351. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  1352. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  1353. return result_list
  1354. # def get_preprocessed(articles,useselffool=False):
  1355. # '''
  1356. # @summary:预处理步骤,NLP处理、实体识别
  1357. # @param:
  1358. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1359. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1360. # '''
  1361. # list_articles = []
  1362. # list_sentences = []
  1363. # list_entitys = []
  1364. # cost_time = dict()
  1365. # for article in articles:
  1366. # list_sentences_temp = []
  1367. # list_entitys_temp = []
  1368. # doc_id = article[0]
  1369. # sourceContent = article[1]
  1370. # _send_doc_id = article[3]
  1371. # _title = article[4]
  1372. # #表格处理
  1373. # key_preprocess = "tableToText"
  1374. # start_time = time.time()
  1375. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1376. #
  1377. # # log(article_processed)
  1378. #
  1379. # if key_preprocess not in cost_time:
  1380. # cost_time[key_preprocess] = 0
  1381. # cost_time[key_preprocess] += time.time()-start_time
  1382. #
  1383. # #article_processed = article[1]
  1384. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1385. # #nlp处理
  1386. # if article_processed is not None and len(article_processed)!=0:
  1387. # split_patten = "。"
  1388. # sentences = []
  1389. # _begin = 0
  1390. # for _iter in re.finditer(split_patten,article_processed):
  1391. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1392. # _begin = _iter.span()[1]
  1393. # sentences.append(article_processed[_begin:])
  1394. #
  1395. # lemmas = []
  1396. # doc_offsets = []
  1397. # dep_types = []
  1398. # dep_tokens = []
  1399. #
  1400. # time1 = time.time()
  1401. #
  1402. # '''
  1403. # tokens_all = fool.cut(sentences)
  1404. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1405. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1406. # ner_entitys_all = fool.ner(sentences)
  1407. # '''
  1408. # #限流执行
  1409. # key_nerToken = "nerToken"
  1410. # start_time = time.time()
  1411. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1412. # if key_nerToken not in cost_time:
  1413. # cost_time[key_nerToken] = 0
  1414. # cost_time[key_nerToken] += time.time()-start_time
  1415. #
  1416. #
  1417. # for sentence_index in range(len(sentences)):
  1418. #
  1419. #
  1420. #
  1421. # list_sentence_entitys = []
  1422. # sentence_text = sentences[sentence_index]
  1423. # tokens = tokens_all[sentence_index]
  1424. #
  1425. # list_tokenbegin = []
  1426. # begin = 0
  1427. # for i in range(0,len(tokens)):
  1428. # list_tokenbegin.append(begin)
  1429. # begin += len(str(tokens[i]))
  1430. # list_tokenbegin.append(begin+1)
  1431. # #pos_tag = pos_all[sentence_index]
  1432. # pos_tag = ""
  1433. #
  1434. # ner_entitys = ner_entitys_all[sentence_index]
  1435. #
  1436. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1437. #
  1438. # #识别package
  1439. #
  1440. #
  1441. # #识别实体
  1442. # for ner_entity in ner_entitys:
  1443. # begin_index_temp = ner_entity[0]
  1444. # end_index_temp = ner_entity[1]
  1445. # entity_type = ner_entity[2]
  1446. # entity_text = ner_entity[3]
  1447. #
  1448. # for j in range(len(list_tokenbegin)):
  1449. # if list_tokenbegin[j]==begin_index_temp:
  1450. # begin_index = j
  1451. # break
  1452. # elif list_tokenbegin[j]>begin_index_temp:
  1453. # begin_index = j-1
  1454. # break
  1455. # begin_index_temp += len(str(entity_text))
  1456. # for j in range(begin_index,len(list_tokenbegin)):
  1457. # if list_tokenbegin[j]>=begin_index_temp:
  1458. # end_index = j-1
  1459. # break
  1460. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1461. #
  1462. # #去掉标点符号
  1463. # entity_text = re.sub("[,,。:]","",entity_text)
  1464. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1465. #
  1466. #
  1467. # #使用正则识别金额
  1468. # entity_type = "money"
  1469. #
  1470. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1471. #
  1472. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1473. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1474. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1475. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1476. #
  1477. # set_begin = set()
  1478. # for pattern_key in list_money_pattern.keys():
  1479. # pattern = re.compile(list_money_pattern[pattern_key])
  1480. # all_match = re.findall(pattern, sentence_text)
  1481. # index = 0
  1482. # for i in range(len(all_match)):
  1483. # if len(all_match[i][0])>0:
  1484. # # print("===",all_match[i])
  1485. # #print(all_match[i][0])
  1486. # unit = ""
  1487. # entity_text = all_match[i][3]
  1488. # if pattern_key in ["key_word","front_m"]:
  1489. # unit = all_match[i][1]
  1490. # else:
  1491. # unit = all_match[i][4]
  1492. # if entity_text.find("元")>=0:
  1493. # unit = ""
  1494. #
  1495. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1496. #
  1497. # begin_index_temp = index
  1498. # for j in range(len(list_tokenbegin)):
  1499. # if list_tokenbegin[j]==index:
  1500. # begin_index = j
  1501. # break
  1502. # elif list_tokenbegin[j]>index:
  1503. # begin_index = j-1
  1504. # break
  1505. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1506. # end_index_temp = index
  1507. # #index += len(str(all_match[i][0]))
  1508. # for j in range(begin_index,len(list_tokenbegin)):
  1509. # if list_tokenbegin[j]>=index:
  1510. # end_index = j-1
  1511. # break
  1512. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1513. #
  1514. #
  1515. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1516. # if len(unit)>0:
  1517. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1518. # else:
  1519. # entity_text = str(getUnifyMoney(entity_text))
  1520. #
  1521. # _exists = False
  1522. # for item in list_sentence_entitys:
  1523. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1524. # _exists = True
  1525. # if not _exists:
  1526. # if float(entity_text)>10:
  1527. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1528. #
  1529. # else:
  1530. # index += 1
  1531. #
  1532. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1533. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1534. # list_sentences.append(list_sentences_temp)
  1535. # list_entitys.append(list_entitys_temp)
  1536. # return list_articles,list_sentences,list_entitys,cost_time
  1537. def get_preprocessed(articles, useselffool=False):
  1538. '''
  1539. @summary:预处理步骤,NLP处理、实体识别
  1540. @param:
  1541. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1542. @return:list of articles,list of each article of sentences,list of each article of entitys
  1543. '''
  1544. cost_time = dict()
  1545. list_articles = get_preprocessed_article(articles,cost_time)
  1546. list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time)
  1547. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1548. calibrateEnterprise(list_articles,list_sentences,list_entitys)
  1549. return list_articles,list_sentences,list_entitys,list_outlines,cost_time
  1550. def special_treatment(sourceContent, web_source_no):
  1551. try:
  1552. if web_source_no == 'DX000202-1':
  1553. ser = re.search('中标供应商及中标金额:【((\w{5,20}-[\d,.]+,)+)】', sourceContent)
  1554. if ser:
  1555. new = ""
  1556. l = ser.group(1).split(',')
  1557. for i in range(len(l)):
  1558. it = l[i]
  1559. if '-' in it:
  1560. role, money = it.split('-')
  1561. new += '标段%d, 中标供应商: ' % (i + 1) + role + ',中标金额:' + money + '。'
  1562. sourceContent = sourceContent.replace(ser.group(0), new, 1)
  1563. elif web_source_no == '00753-14':
  1564. body = sourceContent.find("body")
  1565. body_child = body.find_all(recursive=False)
  1566. pcontent = body
  1567. if 'id' in body_child[0].attrs:
  1568. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1569. pcontent = body_child[0]
  1570. # pcontent = sourceContent.find("div", id="pcontent")
  1571. pcontent = pcontent.find_all(recursive=False)[0]
  1572. first_table = None
  1573. for idx in range(len(pcontent.find_all(recursive=False))):
  1574. t_part = pcontent.find_all(recursive=False)[idx]
  1575. if t_part.name != "table":
  1576. break
  1577. if idx == 0:
  1578. first_table = t_part
  1579. else:
  1580. for _tr in t_part.find("tbody").find_all(recursive=False):
  1581. first_table.find("tbody").append(_tr)
  1582. t_part.clear()
  1583. elif web_source_no == 'DX008357-11':
  1584. body = sourceContent.find("body")
  1585. body_child = body.find_all(recursive=False)
  1586. pcontent = body
  1587. if 'id' in body_child[0].attrs:
  1588. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1589. pcontent = body_child[0]
  1590. # pcontent = sourceContent.find("div", id="pcontent")
  1591. pcontent = pcontent.find_all(recursive=False)[0]
  1592. error_table = []
  1593. is_error_table = False
  1594. for part in pcontent.find_all(recursive=False):
  1595. if is_error_table:
  1596. if part.name == "table":
  1597. error_table.append(part)
  1598. else:
  1599. break
  1600. if part.name == "div" and part.get_text(strip=True) == "中标候选单位:":
  1601. is_error_table = True
  1602. first_table = None
  1603. for idx in range(len(error_table)):
  1604. t_part = error_table[idx]
  1605. # if t_part.name != "table":
  1606. # break
  1607. if idx == 0:
  1608. for _tr in t_part.find("tbody").find_all(recursive=False):
  1609. if _tr.get_text(strip=True) == "":
  1610. _tr.decompose()
  1611. first_table = t_part
  1612. else:
  1613. for _tr in t_part.find("tbody").find_all(recursive=False):
  1614. if _tr.get_text(strip=True) != "":
  1615. first_table.find("tbody").append(_tr)
  1616. t_part.clear()
  1617. elif web_source_no == '18021-2':
  1618. body = sourceContent.find("body")
  1619. body_child = body.find_all(recursive=False)
  1620. pcontent = body
  1621. if 'id' in body_child[0].attrs:
  1622. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1623. pcontent = body_child[0]
  1624. # pcontent = sourceContent.find("div", id="pcontent")
  1625. td = pcontent.find_all("td")
  1626. for _td in td:
  1627. if str(_td.string).strip() == "报价金额":
  1628. _td.string = "单价"
  1629. elif web_source_no == '13740-2':
  1630. # “xxx成为成交供应商”
  1631. re_match = re.search("[^,。]+成为[^,。]*成交供应商", sourceContent)
  1632. if re_match:
  1633. sourceContent = sourceContent.replace(re_match.group(), "成交人:" + re_match.group())
  1634. elif web_source_no == '03786-10':
  1635. ser1 = re.search('中标价:([\d,.]+)', sourceContent)
  1636. ser2 = re.search('合同金额[((]万元[))]:([\d,.]+)', sourceContent)
  1637. if ser1 and ser2:
  1638. m1 = ser1.group(1).replace(',', '')
  1639. m2 = ser2.group(1).replace(',', '')
  1640. if float(m1) < 100000 and (m1.split('.')[0] == m2.split('.')[0] or m2 == '0'):
  1641. new = '中标价(万元):' + m1
  1642. sourceContent = sourceContent.replace(ser1.group(0), new, 1)
  1643. elif web_source_no=='00076-4':
  1644. ser = re.search('主要标的数量:([0-9一]+)\w{,3},主要标的单价:([\d,.]+)元?,合同金额:(.00),', sourceContent)
  1645. if ser:
  1646. num = ser.group(1).replace('一', '1')
  1647. try:
  1648. num = 1 if num == '0' else num
  1649. unit_price = ser.group(2).replace(',', '')
  1650. total_price = str(int(num) * float(unit_price))
  1651. new = '合同金额:' + total_price
  1652. sourceContent = sourceContent.replace('合同金额:.00', new, 1)
  1653. except Exception as e:
  1654. log('preprocessing.py special_treatment exception')
  1655. elif web_source_no=='DX000105-2':
  1656. if re.search("成交公示", sourceContent) and re.search(',投标人:', sourceContent) and re.search(',成交人:', sourceContent)==None:
  1657. sourceContent = sourceContent.replace(',投标人:', ',成交人:')
  1658. elif web_source_no in ['04080-3', '04080-4']:
  1659. ser = re.search('合同金额:([0-9,]+.[0-9]{3,})(.{,4})', sourceContent)
  1660. if ser and '万' not in ser.group(2):
  1661. sourceContent = sourceContent.replace('合同金额:', '合同金额(万元):')
  1662. elif web_source_no=='03761-3':
  1663. ser = re.search('中标价,([0-9]+)[.0-9]*%', sourceContent)
  1664. if ser and int(ser.group(1))>100:
  1665. sourceContent = sourceContent.replace(ser.group(0), ser.group(0)[:-1]+'元')
  1666. elif web_source_no=='00695-7':
  1667. ser = re.search('支付金额:', sourceContent)
  1668. if ser:
  1669. sourceContent = sourceContent.replace('支付金额:', '合同金额:')
  1670. return sourceContent
  1671. except Exception as e:
  1672. log('特殊数据源: %s 预处理特别修改抛出异常: %s'%(web_source_no, e))
  1673. return sourceContent
  1674. def article_limit(soup,limit_words=30000):
  1675. sub_space = re.compile("\s+")
  1676. def soup_limit(_soup,_count,max_count=30000,max_gap=500):
  1677. """
  1678. :param _soup: soup
  1679. :param _count: 当前字数
  1680. :param max_count: 字数最大限制
  1681. :param max_gap: 超过限制后的最大误差
  1682. :return:
  1683. """
  1684. _gap = _count - max_count
  1685. _is_skip = False
  1686. next_soup = None
  1687. while len(_soup.find_all(recursive=False)) == 1 and \
  1688. _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
  1689. _soup = _soup.find_all(recursive=False)[0]
  1690. if len(_soup.find_all(recursive=False)) == 0:
  1691. _soup.string = str(_soup.get_text())[:max_count-_count]
  1692. _count += len(re.sub(sub_space, "", _soup.string))
  1693. _gap = _count - max_count
  1694. next_soup = None
  1695. else:
  1696. for _soup_part in _soup.find_all(recursive=False):
  1697. if not _is_skip:
  1698. _count += len(re.sub(sub_space, "", _soup_part.get_text()))
  1699. if _count >= max_count:
  1700. _gap = _count - max_count
  1701. if _gap <= max_gap:
  1702. _is_skip = True
  1703. else:
  1704. _is_skip = True
  1705. next_soup = _soup_part
  1706. _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
  1707. continue
  1708. else:
  1709. _soup_part.decompose()
  1710. return _count,_gap,next_soup
  1711. text_count = 0
  1712. have_attachment = False
  1713. attachment_part = None
  1714. for child in soup.find_all(recursive=True):
  1715. if child.name == 'div' and 'class' in child.attrs:
  1716. if "richTextFetch" in child['class']:
  1717. child.insert_before("##attachment##。") # 句号分开,避免项目名称等提取
  1718. attachment_part = child
  1719. have_attachment = True
  1720. break
  1721. if not have_attachment:
  1722. # 无附件
  1723. if len(re.sub(sub_space, "", soup.get_text())) > limit_words:
  1724. text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500)
  1725. while n_soup:
  1726. text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
  1727. else:
  1728. # 有附件
  1729. _text = re.sub(sub_space, "", soup.get_text())
  1730. _text_split = _text.split("##attachment##")
  1731. if len(_text_split[0])>limit_words:
  1732. main_soup = attachment_part.parent
  1733. main_text = main_soup.find_all(recursive=False)[0]
  1734. text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500)
  1735. while n_soup:
  1736. text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
  1737. if len(_text_split[1])>limit_words:
  1738. # attachment_html纯文本,无子结构
  1739. if len(attachment_part.find_all(recursive=False))==0:
  1740. attachment_part.string = str(attachment_part.get_text())[:limit_words]
  1741. else:
  1742. attachment_text_nums = 0
  1743. attachment_skip = False
  1744. for part in attachment_part.find_all(recursive=False):
  1745. if not attachment_skip:
  1746. last_attachment_text_nums = attachment_text_nums
  1747. attachment_text_nums = attachment_text_nums + len(re.sub(sub_space, "", part.get_text()))
  1748. if attachment_text_nums>=limit_words:
  1749. part.string = str(part.get_text())[:limit_words-last_attachment_text_nums]
  1750. attachment_skip = True
  1751. else:
  1752. part.decompose()
  1753. return soup
  1754. def attachment_filelink(soup):
  1755. have_attachment = False
  1756. attachment_part = None
  1757. for child in soup.find_all(recursive=True):
  1758. if child.name == 'div' and 'class' in child.attrs:
  1759. if "richTextFetch" in child['class']:
  1760. attachment_part = child
  1761. have_attachment = True
  1762. break
  1763. if not have_attachment:
  1764. return soup
  1765. else:
  1766. # 附件类型:图片、表格
  1767. attachment_type = re.compile("\.(?:png|jpg|jpeg|tif|bmp|xlsx|xls)$")
  1768. attachment_dict = dict()
  1769. for _attachment in attachment_part.find_all(recursive=False):
  1770. if _attachment.name == 'div' and 'filemd5' in _attachment.attrs:
  1771. # print('filemd5',_attachment['filemd5'])
  1772. attachment_dict[_attachment['filemd5']] = _attachment
  1773. # print(attachment_dict)
  1774. for child in soup.find_all(recursive=True):
  1775. if child.name == 'div' and 'class' in child.attrs:
  1776. if "richTextFetch" in child['class']:
  1777. break
  1778. if "filelink" in child.attrs and child['filelink'] in attachment_dict:
  1779. if re.search(attachment_type,str(child.string).strip()) or \
  1780. ('original' in child.attrs and re.search(attachment_type,str(child['original']).strip())):
  1781. # 附件插入正文标识
  1782. child.insert_before("。##attachment_begin##")
  1783. child.insert_after("。##attachment_end##")
  1784. child.replace_with(attachment_dict[child['filelink']])
  1785. # print('格式化输出',soup.prettify())
  1786. return soup
  1787. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  1788. '''
  1789. :param articles: 待处理的article source html
  1790. :param useselffool: 是否使用selffool
  1791. :return: list_articles
  1792. '''
  1793. list_articles = []
  1794. for article in articles:
  1795. doc_id = article[0]
  1796. sourceContent = article[1]
  1797. sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
  1798. sourceContent = sourceContent.replace('<br/>', '<br>')
  1799. sourceContent = re.sub("<br>(\s{0,}<br>)+","<br>",sourceContent)
  1800. # for br_match in re.findall("[^>]+?<br>",sourceContent):
  1801. # _new = re.sub("<br>","",br_match)
  1802. # # <br>标签替换为<p>标签
  1803. # if not re.search("^\s+$",_new):
  1804. # _new = '<p>'+_new + '</p>'
  1805. # # print(br_match,_new)
  1806. # sourceContent = sourceContent.replace(br_match,_new,1)
  1807. _send_doc_id = article[3]
  1808. _title = article[4]
  1809. page_time = article[5]
  1810. web_source_no = article[6]
  1811. '''特别数据源对 html 做特别修改'''
  1812. if web_source_no in ['DX000202-1']:
  1813. sourceContent = special_treatment(sourceContent, web_source_no)
  1814. #表格处理
  1815. key_preprocess = "tableToText"
  1816. start_time = time.time()
  1817. # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml"))
  1818. article_processed = BeautifulSoup(sourceContent,"lxml")
  1819. '''特别数据源对 BeautifulSoup(html) 做特别修改'''
  1820. if web_source_no in ["00753-14","DX008357-11","18021-2"]:
  1821. article_processed = special_treatment(article_processed, web_source_no)
  1822. for _soup in article_processed.descendants:
  1823. # 识别无标签文本,添加<span>标签
  1824. if not _soup.name and not _soup.parent.string and _soup.string.strip()!="":
  1825. # print(_soup.parent.string,_soup.string.strip())
  1826. _soup.wrap(article_processed.new_tag("span"))
  1827. # print(article_processed)
  1828. # 正文和附件内容限制字数30000
  1829. article_processed = article_limit(article_processed, limit_words=30000)
  1830. # 把每个附件识别对应的html放回原来出现的位置
  1831. article_processed = attachment_filelink(article_processed)
  1832. article_processed = get_preprocessed_outline(article_processed)
  1833. # print('article_processed')
  1834. article_processed = tableToText(article_processed)
  1835. article_processed = segment(article_processed)
  1836. article_processed = article_processed.replace('(', '(').replace(')', ')') #2022/8/10 统一为中文括号
  1837. article_processed = article_processed.replace('.','.') # 2021/12/01 修正OCR识别PDF小数点错误问题
  1838. article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
  1839. article_processed = article_processed.replace('成交工程价款', '成交工程价') # 2021/12/21 修正为中标价
  1840. article_processed = re.sub('任务(?=编号[::])', '项目',article_processed) # 2022/08/10 修正为项目编号
  1841. article_processed = article_processed.replace('招标(建设)单位', '招标单位') #2022/8/10 修正预测不到表达
  1842. article_processed = re.sub('(招标|采购)人(概况|信息)[,。]', '采购人信息:', article_processed) # 2022/8/10统一表达
  1843. # 修复OCR金额中“,”、“。”识别错误
  1844. article_processed_list = article_processed.split("##attachment##")
  1845. if len(article_processed_list)>1:
  1846. attachment_text = article_processed_list[1]
  1847. for _match in re.finditer("\d。\d{2}",attachment_text):
  1848. _match_text = _match.group()
  1849. attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1)
  1850. for _match in re.finditer("(\d,\d{3})[,,.]",attachment_text):
  1851. _match_text = _match.group()
  1852. attachment_text = attachment_text.replace(_match_text,_match_text.replace(",",","),1)
  1853. article_processed_list[1] = attachment_text
  1854. article_processed = "##attachment##".join(article_processed_list)
  1855. '''特别数据源对 预处理后文本 做特别修改'''
  1856. if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2"]:
  1857. article_processed = special_treatment(article_processed, web_source_no)
  1858. # 提取bidway
  1859. list_bidway = extract_bidway(article_processed, _title)
  1860. if list_bidway:
  1861. bidway = list_bidway[0].get("body")
  1862. # bidway名称统一规范
  1863. bidway = bidway_integrate(bidway)
  1864. else:
  1865. bidway = ""
  1866. # 修正被","逗号分隔的时间
  1867. repair_time = re.compile("[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d|"
  1868. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[:时点],?[0-6]\d分?|"
  1869. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[时点]|"
  1870. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]|"
  1871. "[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d"
  1872. )
  1873. for _time in set(re.findall(repair_time,article_processed)):
  1874. if re.search(",",_time):
  1875. _time2 = re.sub(",", "", _time)
  1876. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time2)
  1877. if item:
  1878. _time2 = _time2.replace(item.group(),item.group() + " ")
  1879. article_processed = article_processed.replace(_time, _time2)
  1880. else:
  1881. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time)
  1882. if item:
  1883. _time2 = _time.replace(item.group(),item.group() + " ")
  1884. article_processed = article_processed.replace(_time, _time2)
  1885. # print('re_rtime',re.findall(repair_time,article_processed))
  1886. # log(article_processed)
  1887. if key_preprocess not in cost_time:
  1888. cost_time[key_preprocess] = 0
  1889. cost_time[key_preprocess] += round(time.time()-start_time,2)
  1890. #article_processed = article[1]
  1891. _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
  1892. bidway=bidway)
  1893. _article.fingerprint = getFingerprint(_title+sourceContent)
  1894. _article.page_time = page_time
  1895. list_articles.append(_article)
  1896. return list_articles
  1897. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  1898. '''
  1899. :param list_articles: 经过预处理的article text
  1900. :return: list_sentences
  1901. '''
  1902. list_sentences = []
  1903. list_outlines = []
  1904. for article in list_articles:
  1905. list_sentences_temp = []
  1906. list_entitys_temp = []
  1907. doc_id = article.id
  1908. _send_doc_id = article.doc_id
  1909. _title = article.title
  1910. #表格处理
  1911. key_preprocess = "tableToText"
  1912. start_time = time.time()
  1913. article_processed = article.content
  1914. if len(_title)<100 and _title not in article_processed: # 把标题放到正文
  1915. article_processed = _title + article_processed
  1916. attachment_begin_index = -1
  1917. if key_preprocess not in cost_time:
  1918. cost_time[key_preprocess] = 0
  1919. cost_time[key_preprocess] += time.time()-start_time
  1920. #nlp处理
  1921. if article_processed is not None and len(article_processed)!=0:
  1922. split_patten = "。"
  1923. sentences = []
  1924. _begin = 0
  1925. sentences_set = set()
  1926. for _iter in re.finditer(split_patten,article_processed):
  1927. _sen = article_processed[_begin:_iter.span()[1]]
  1928. if len(_sen)>0 and _sen not in sentences_set:
  1929. # 标识在附件里的句子
  1930. if re.search("##attachment##",_sen):
  1931. attachment_begin_index = len(sentences)
  1932. # _sen = re.sub("##attachment##","",_sen)
  1933. sentences.append(_sen)
  1934. sentences_set.add(_sen)
  1935. _begin = _iter.span()[1]
  1936. _sen = article_processed[_begin:]
  1937. if re.search("##attachment##", _sen):
  1938. # _sen = re.sub("##attachment##", "", _sen)
  1939. attachment_begin_index = len(sentences)
  1940. if len(_sen)>0 and _sen not in sentences_set:
  1941. sentences.append(_sen)
  1942. sentences_set.add(_sen)
  1943. # 解析outline大纲分段
  1944. outline_list = []
  1945. if re.search("##split##",article.content):
  1946. temp_sentences = []
  1947. last_sentence_index = (-1,-1)
  1948. outline_index = 0
  1949. for sentence_index in range(len(sentences)):
  1950. sentence_text = sentences[sentence_index]
  1951. for _ in re.findall("##split##", sentence_text):
  1952. _match = re.search("##split##", sentence_text)
  1953. if last_sentence_index[0] > -1:
  1954. sentence_begin_index,wordOffset_begin = last_sentence_index
  1955. sentence_end_index = sentence_index
  1956. wordOffset_end = _match.start()
  1957. if sentence_begin_index<attachment_begin_index and sentence_end_index>=attachment_begin_index:
  1958. outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,attachment_begin_index-1,wordOffset_begin,len(sentences[attachment_begin_index-1])))
  1959. else:
  1960. outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,sentence_end_index,wordOffset_begin,wordOffset_end))
  1961. outline_index += 1
  1962. sentence_text = re.sub("##split##", "", sentence_text,count=1)
  1963. last_sentence_index = (sentence_index,_match.start())
  1964. temp_sentences.append(sentence_text)
  1965. if attachment_begin_index>-1 and last_sentence_index[0]<attachment_begin_index:
  1966. outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],attachment_begin_index-1,last_sentence_index[1],len(temp_sentences[attachment_begin_index-1])))
  1967. else:
  1968. outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],len(sentences)-1,last_sentence_index[1],len(temp_sentences[-1])))
  1969. sentences = temp_sentences
  1970. #解析outline的outline_text内容
  1971. for _outline in outline_list:
  1972. if _outline.sentence_begin_index==_outline.sentence_end_index:
  1973. _text = sentences[_outline.sentence_begin_index][_outline.wordOffset_begin:_outline.wordOffset_end]
  1974. else:
  1975. _text = ""
  1976. for idx in range(_outline.sentence_begin_index,_outline.sentence_end_index+1):
  1977. if idx==_outline.sentence_begin_index:
  1978. _text += sentences[idx][_outline.wordOffset_begin:]
  1979. elif idx==_outline.sentence_end_index:
  1980. _text += sentences[idx][:_outline.wordOffset_end]
  1981. else:
  1982. _text += sentences[idx]
  1983. _outline.outline_text = _text
  1984. _outline_summary = re.split("[::,]",_text,1)[0]
  1985. if len(_outline_summary)<20:
  1986. _outline.outline_summary = _outline_summary
  1987. # print(_outline.outline_index,_outline.outline_text)
  1988. article.content = "".join(sentences)
  1989. # sentences.append(article_processed[_begin:])
  1990. lemmas = []
  1991. doc_offsets = []
  1992. dep_types = []
  1993. dep_tokens = []
  1994. time1 = time.time()
  1995. '''
  1996. tokens_all = fool.cut(sentences)
  1997. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1998. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1999. ner_entitys_all = fool.ner(sentences)
  2000. '''
  2001. #限流执行
  2002. key_nerToken = "nerToken"
  2003. start_time = time.time()
  2004. # tokens_all = getTokens(sentences,useselffool=useselffool)
  2005. tokens_all = getTokens([re.sub("##attachment_begin##|##attachment_end##","",_sen) for _sen in sentences],useselffool=useselffool)
  2006. if key_nerToken not in cost_time:
  2007. cost_time[key_nerToken] = 0
  2008. cost_time[key_nerToken] += round(time.time()-start_time,2)
  2009. in_attachment = False
  2010. for sentence_index in range(len(sentences)):
  2011. sentence_text = sentences[sentence_index]
  2012. if re.search("##attachment_begin##",sentence_text):
  2013. in_attachment = True
  2014. sentence_text = re.sub("##attachment_begin##","",sentence_text)
  2015. elif re.search("##attachment_end##",sentence_text):
  2016. in_attachment = False
  2017. sentence_text = re.sub("##attachment_end##", "", sentence_text)
  2018. if sentence_index >= attachment_begin_index and attachment_begin_index!=-1:
  2019. in_attachment = True
  2020. tokens = tokens_all[sentence_index]
  2021. #pos_tag = pos_all[sentence_index]
  2022. pos_tag = ""
  2023. ner_entitys = ""
  2024. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys,in_attachment=in_attachment))
  2025. if len(list_sentences_temp)==0:
  2026. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  2027. list_sentences.append(list_sentences_temp)
  2028. list_outlines.append(outline_list)
  2029. return list_sentences,list_outlines
  2030. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  2031. '''
  2032. :param list_sentences:分局情况
  2033. :param cost_time:
  2034. :return: list_entitys
  2035. '''
  2036. list_entitys = []
  2037. for list_sentence in list_sentences:
  2038. sentences = []
  2039. list_entitys_temp = []
  2040. for _sentence in list_sentence:
  2041. sentences.append(_sentence.sentence_text)
  2042. lemmas = []
  2043. doc_offsets = []
  2044. dep_types = []
  2045. dep_tokens = []
  2046. time1 = time.time()
  2047. '''
  2048. tokens_all = fool.cut(sentences)
  2049. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  2050. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  2051. ner_entitys_all = fool.ner(sentences)
  2052. '''
  2053. #限流执行
  2054. key_nerToken = "nerToken"
  2055. start_time = time.time()
  2056. found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
  2057. # found_pingbiao = False
  2058. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  2059. if key_nerToken not in cost_time:
  2060. cost_time[key_nerToken] = 0
  2061. cost_time[key_nerToken] += round(time.time()-start_time,2)
  2062. company_dict = set()
  2063. company_index = dict((i,set()) for i in range(len(list_sentence)))
  2064. for sentence_index in range(len(list_sentence)):
  2065. list_sentence_entitys = []
  2066. sentence_text = list_sentence[sentence_index].sentence_text
  2067. tokens = list_sentence[sentence_index].tokens
  2068. doc_id = list_sentence[sentence_index].doc_id
  2069. in_attachment = list_sentence[sentence_index].in_attachment
  2070. list_tokenbegin = []
  2071. begin = 0
  2072. for i in range(0,len(tokens)):
  2073. list_tokenbegin.append(begin)
  2074. begin += len(str(tokens[i]))
  2075. list_tokenbegin.append(begin+1)
  2076. #pos_tag = pos_all[sentence_index]
  2077. pos_tag = ""
  2078. ner_entitys = ner_entitys_all[sentence_index]
  2079. '''正则识别角色实体 经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
  2080. for it in re.finditer(
  2081. '(?P<text_key_word>(((单一来源|中标|中选|中价|成交)(供应商|供货商|服务商|候选人|单位|人))|(供应商|供货商|服务商|候选人))(名称)?[为::]+)(?P<text>([^,。、;《::]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[,。]',
  2082. sentence_text):
  2083. for k, v in it.groupdict().items():
  2084. if k == 'text_key_word':
  2085. keyword = v
  2086. if k == 'text':
  2087. entity = v
  2088. b = it.start() + len(keyword)
  2089. e = it.end() - 1
  2090. if (b, e, 'location', entity) in ner_entitys:
  2091. ner_entitys.remove((b, e, 'location', entity))
  2092. ner_entitys.append((b, e, 'company', entity))
  2093. elif (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  2094. ner_entitys.append((b, e, 'company', entity))
  2095. for it in re.finditer(
  2096. '(?P<text_key_word>((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为::]+)(?P<text>\w{2,4}[省市县区镇]([^,。、;《]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[,。]',
  2097. sentence_text):
  2098. for k, v in it.groupdict().items():
  2099. if k == 'text_key_word':
  2100. keyword = v
  2101. if k == 'text':
  2102. entity = v
  2103. b = it.start() + len(keyword)
  2104. e = it.end() - 1
  2105. if (b, e, 'location', entity) in ner_entitys:
  2106. ner_entitys.remove((b, e, 'location', entity))
  2107. ner_entitys.append((b, e, 'org', entity))
  2108. if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  2109. ner_entitys.append((b, e, 'org', entity))
  2110. for ner_entity in ner_entitys:
  2111. if ner_entity[2] in ['company','org']:
  2112. company_dict.add((ner_entity[2],ner_entity[3]))
  2113. company_index[sentence_index].add((ner_entity[0],ner_entity[1]))
  2114. #识别package
  2115. #识别实体
  2116. for ner_entity in ner_entitys:
  2117. begin_index_temp = ner_entity[0]
  2118. end_index_temp = ner_entity[1]
  2119. entity_type = ner_entity[2]
  2120. entity_text = ner_entity[3]
  2121. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  2122. continue
  2123. for j in range(len(list_tokenbegin)):
  2124. if list_tokenbegin[j]==begin_index_temp:
  2125. begin_index = j
  2126. break
  2127. elif list_tokenbegin[j]>begin_index_temp:
  2128. begin_index = j-1
  2129. break
  2130. begin_index_temp += len(str(entity_text))
  2131. for j in range(begin_index,len(list_tokenbegin)):
  2132. if list_tokenbegin[j]>=begin_index_temp:
  2133. end_index = j-1
  2134. break
  2135. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2136. #去掉标点符号
  2137. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  2138. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  2139. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
  2140. # 标记文章末尾的"发布人”、“发布时间”实体
  2141. if sentence_index==len(list_sentence)-1:
  2142. if len(list_sentence_entitys[-2:])>2:
  2143. second2last = list_sentence_entitys[-2]
  2144. last = list_sentence_entitys[-1]
  2145. if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
  2146. second2last.entity_type=="time" and last.entity_type in ["company",'org']):
  2147. if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
  2148. last.is_tail = True
  2149. second2last.is_tail = True
  2150. #使用正则识别金额
  2151. entity_type = "money"
  2152. #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  2153. # list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  2154. # "key_word":"((?P<text_key_word>(?:[¥¥]+,?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果:)(?:[,(\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[)\)]?))",
  2155. # "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())",
  2156. # "behind_m":"(()()(?P<money_behind_m>[0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\))]?)"}
  2157. list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  2158. "key_word": "((?P<text_key_word>(?:[¥¥]+,?|[单报标限总]价|金额|成交报?价|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果|成交额|中标额)(?:[,,(\(]*\s*(人民币)?(?P<unit_key_word_before>[万亿]?元?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[::])?(\d+(\*\d+%)+=)?(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]{,1})(?:[(\(]?(?P<filter_>[%])*\s*(单位[::])?(?P<unit_key_word_behind>[万亿]?元?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[)\)]?))",
  2159. "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万亿]?元)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)())",
  2160. "behind_m":"(()()(?P<money_behind_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)(人民币)?[\((]?(?P<unit_behind_m>[万亿]?元(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\))]?)"}
  2161. # 2021/7/19 调整金额,单位提取正则,修复部分金额因为单位提取失败被过滤问题。
  2162. pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
  2163. set_begin = set()
  2164. # for pattern_key in list_money_pattern.keys():
  2165. # for pattern_key in ["cn","key_word","behind_m","front_m"]:
  2166. # # pattern = re.compile(list_money_pattern[pattern_key])
  2167. # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?\s*([万元]*)\s*[)\)]?))*|(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*|((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*")
  2168. # all_match = re.findall(pattern, sentence_text)
  2169. # index = 0
  2170. # for i in range(len(all_match)):
  2171. # if len(all_match[i][0])>0:
  2172. # print("===",all_match[i])
  2173. # #print(all_match[i][0])
  2174. # unit = ""
  2175. # entity_text = all_match[i][3]
  2176. # if pattern_key in ["key_word","front_m"]:
  2177. # unit = all_match[i][1]
  2178. # if pattern_key=="key_word":
  2179. # if all_match[i][1]=="" and all_match[i][4]!="":
  2180. # unit = all_match[i][4]
  2181. # else:
  2182. # unit = all_match[i][4]
  2183. # if entity_text.find("元")>=0:
  2184. # unit = ""
  2185. #
  2186. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  2187. # begin_index_temp = index
  2188. # for j in range(len(list_tokenbegin)):
  2189. # if list_tokenbegin[j]==index:
  2190. # begin_index = j
  2191. # break
  2192. # elif list_tokenbegin[j]>index:
  2193. # begin_index = j-1
  2194. # break
  2195. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  2196. # end_index_temp = index
  2197. # #index += len(str(all_match[i][0]))
  2198. # for j in range(begin_index,len(list_tokenbegin)):
  2199. # if list_tokenbegin[j]>=index:
  2200. # end_index = j-1
  2201. # break
  2202. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2203. #
  2204. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  2205. # if len(unit)>0:
  2206. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  2207. # else:
  2208. # entity_text = str(getUnifyMoney(entity_text))
  2209. #
  2210. # _exists = False
  2211. # for item in list_sentence_entitys:
  2212. # if item.entity_id==entity_id and item.entity_type==entity_type:
  2213. # _exists = True
  2214. # if not _exists:
  2215. # if float(entity_text)>1:
  2216. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  2217. #
  2218. # else:
  2219. # index += 1
  2220. # if re.search('评标结果|候选人公示', sentence_text):
  2221. # found_pingbiao = True
  2222. if re.search('业绩', sentence_text):
  2223. found_yeji += 1
  2224. if found_yeji >= 2: # 过滤掉业绩后面的所有金额
  2225. all_match = []
  2226. else:
  2227. all_match = re.finditer(pattern_money, sentence_text)
  2228. index = 0
  2229. for _match in all_match:
  2230. if len(_match.group())>0:
  2231. # print("===",_match.group())
  2232. # # print(_match.groupdict())
  2233. notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
  2234. unit = ""
  2235. entity_text = ""
  2236. text_beforeMoney = ""
  2237. filter = ""
  2238. filter_unit = False
  2239. notSure = False
  2240. if re.search('业绩', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额
  2241. # print('金额在业绩后面: ', _match.group(0))
  2242. found_yeji += 1
  2243. break
  2244. for k,v in _match.groupdict().items():
  2245. if v!="" and v is not None:
  2246. if k=='text_key_word':
  2247. notSure = True
  2248. if k.split("_")[0]=="money":
  2249. entity_text = v
  2250. if k.split("_")[0]=="unit":
  2251. if v=='万元' or unit=="": # 处理 预算金额(元):160万元 这种出现前后单位不一致情况
  2252. unit = v
  2253. if k.split("_")[0]=="text":
  2254. text_beforeMoney = v
  2255. if k.split("_")[0]=="filter":
  2256. filter = v
  2257. if re.search("filter_unit",k) is not None:
  2258. filter_unit = True
  2259. # print(_match.group())
  2260. # print(entity_text,unit,text_beforeMoney,filter,filter_unit)
  2261. if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()): # 2021/7/19 修正OCR识别小数点为逗号
  2262. if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]):
  2263. entity_text = re.sub('\d+,', '', entity_text)
  2264. else:
  2265. entity_text = entity_text.replace(',', '.')
  2266. # print(' 修正OCR识别小数点为逗号')
  2267. if entity_text.find("元")>=0:
  2268. unit = ""
  2269. if unit == "": #2021/7/21 有明显金额特征的补充单位,避免被过滤
  2270. if ('¥' in text_beforeMoney or '¥' in text_beforeMoney):
  2271. unit = '元'
  2272. # print('明显金额特征补充单位 元')
  2273. elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[::为]+$', text_beforeMoney.strip()) and \
  2274. re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None:
  2275. unit = '元'
  2276. # print('明显金额特征补充单位 元')
  2277. elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7},?)$)|(^\d{,3}(,\d{3})+,?$)',entity_text):
  2278. unit = '元'
  2279. # print('明显金额特征补充单位 元')
  2280. if unit.find("万") >= 0 and entity_text.find("万") >= 0: #2021/7/19修改为金额文本有万,不计算单位
  2281. # print('修正金额及单位都有万, 金额:',entity_text, '单位:',unit)
  2282. unit = "元"
  2283. if re.search('.*万元万元', entity_text): #2021/7/19 修正两个万元
  2284. # print(' 修正两个万元',entity_text)
  2285. entity_text = entity_text.replace('万元万元','万元')
  2286. else:
  2287. if filter_unit:
  2288. continue
  2289. if filter!="":
  2290. continue
  2291. index = _match.span()[0]+len(text_beforeMoney)
  2292. begin_index_temp = index
  2293. for j in range(len(list_tokenbegin)):
  2294. if list_tokenbegin[j]==index:
  2295. begin_index = j
  2296. break
  2297. elif list_tokenbegin[j]>index:
  2298. begin_index = j-1
  2299. break
  2300. index = _match.span()[1]
  2301. end_index_temp = index
  2302. #index += len(str(all_match[i][0]))
  2303. for j in range(begin_index,len(list_tokenbegin)):
  2304. if list_tokenbegin[j]>=index:
  2305. end_index = j-1
  2306. break
  2307. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2308. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
  2309. # print('转换前金额:', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
  2310. if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/8/5过滤掉总投资金额
  2311. # print('总投资金额: ', _match.group(0))
  2312. notes = '总投资'
  2313. elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额
  2314. notes = '投资'
  2315. elif re.search('工程造价', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额
  2316. notes = '工程造价'
  2317. elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
  2318. or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\((]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\))]*[::为]',
  2319. sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
  2320. or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
  2321. sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
  2322. notes = '保证金'
  2323. # print('保证金信息:', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
  2324. elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
  2325. sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
  2326. notes = '成本警戒线'
  2327. elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为:]', sentence_text[_match.span()[0]:_match.span()[1]]):
  2328. cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
  2329. notes = cost_re.group(1)
  2330. elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
  2331. notes = '单价'
  2332. elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
  2333. notes = '大写'
  2334. if entity_text[0] == "拾": # 2021/12/16 修正大写金额省略了数字转换错误问题
  2335. entity_text = "壹"+entity_text
  2336. # print("补充备注:notes = 大写")
  2337. if len(unit)>0:
  2338. if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况
  2339. # print('修正单位万元金额过大的情况 金额:', entity_text, '单位:', unit)
  2340. entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000)
  2341. unit = '元' # 修正金额后单位 重置为元
  2342. else:
  2343. # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
  2344. entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  2345. else:
  2346. if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8:
  2347. entity_text = str(getUnifyMoney(entity_text)/10000)
  2348. # print('修正金额字段含万 过大的情况')
  2349. else:
  2350. entity_text = str(getUnifyMoney(entity_text))
  2351. if float(entity_text)>100000000000 or float(entity_text)<100: # float(entity_text)<100 or 2022/3/4 取消最小金额限制
  2352. # print('过滤掉金额:float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
  2353. continue
  2354. if notSure and unit=="" and float(entity_text)>100*10000:
  2355. # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000:', entity_text, unit)
  2356. continue
  2357. _exists = False
  2358. for item in list_sentence_entitys:
  2359. if item.entity_id==entity_id and item.entity_type==entity_type:
  2360. _exists = True
  2361. if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index):
  2362. _exists = True
  2363. if not _exists:
  2364. if float(entity_text)>1:
  2365. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp,in_attachment=in_attachment))
  2366. list_sentence_entitys[-1].notes = notes # 2021/7/20 新增金额备注
  2367. list_sentence_entitys[-1].money_unit = unit # 2021/7/20 新增金额备注
  2368. # print('预处理中的 金额:%s, 单位:%s'%(entity_text,unit))
  2369. # print(entity_text,unit,notes)
  2370. else:
  2371. index += 1
  2372. # "联系人"正则补充提取 2021/11/15 新增
  2373. list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
  2374. error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
  2375. '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理',
  2376. '代理人','采购','附件','注意','登录','报名','踏勘',"测试"]
  2377. list_person_text = set(list_person_text + error_text)
  2378. re_person = re.compile("联系人[::]([\u4e00-\u9fa5]工)|"
  2379. "联系人[::]([\u4e00-\u9fa5]{2,3})(?=联系)|"
  2380. "联系人[::]([\u4e00-\u9fa5]{2,3})")
  2381. list_person = []
  2382. if not in_attachment:
  2383. for match_result in re_person.finditer(sentence_text):
  2384. match_text = match_result.group()
  2385. entity_text = match_text[4:]
  2386. wordOffset_begin = match_result.start() + 4
  2387. wordOffset_end = match_result.end()
  2388. # print(text[wordOffset_begin:wordOffset_end])
  2389. # 排除一些不为人名的实体
  2390. if re.search("^[\u4e00-\u9fa5]{7,}([,。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
  2391. continue
  2392. if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
  2393. _person = dict()
  2394. _person['body'] = entity_text
  2395. _person['begin_index'] = wordOffset_begin
  2396. _person['end_index'] = wordOffset_end
  2397. list_person.append(_person)
  2398. entity_type = "person"
  2399. for person in list_person:
  2400. begin_index_temp = person['begin_index']
  2401. for j in range(len(list_tokenbegin)):
  2402. if list_tokenbegin[j] == begin_index_temp:
  2403. begin_index = j
  2404. break
  2405. elif list_tokenbegin[j] > begin_index_temp:
  2406. begin_index = j - 1
  2407. break
  2408. index = person['end_index']
  2409. end_index_temp = index
  2410. for j in range(begin_index, len(list_tokenbegin)):
  2411. if list_tokenbegin[j] >= index:
  2412. end_index = j - 1
  2413. break
  2414. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2415. entity_text = person['body']
  2416. list_sentence_entitys.append(
  2417. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2418. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2419. # 资金来源提取 2020/12/30 新增
  2420. list_moneySource = extract_moneySource(sentence_text)
  2421. entity_type = "moneysource"
  2422. for moneySource in list_moneySource:
  2423. begin_index_temp = moneySource['begin_index']
  2424. for j in range(len(list_tokenbegin)):
  2425. if list_tokenbegin[j] == begin_index_temp:
  2426. begin_index = j
  2427. break
  2428. elif list_tokenbegin[j] > begin_index_temp:
  2429. begin_index = j - 1
  2430. break
  2431. index = moneySource['end_index']
  2432. end_index_temp = index
  2433. for j in range(begin_index, len(list_tokenbegin)):
  2434. if list_tokenbegin[j] >= index:
  2435. end_index = j - 1
  2436. break
  2437. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2438. entity_text = moneySource['body']
  2439. list_sentence_entitys.append(
  2440. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2441. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2442. # 电子邮箱提取 2021/11/04 新增
  2443. list_email = extract_email(sentence_text)
  2444. entity_type = "email" # 电子邮箱
  2445. for email in list_email:
  2446. begin_index_temp = email['begin_index']
  2447. for j in range(len(list_tokenbegin)):
  2448. if list_tokenbegin[j] == begin_index_temp:
  2449. begin_index = j
  2450. break
  2451. elif list_tokenbegin[j] > begin_index_temp:
  2452. begin_index = j - 1
  2453. break
  2454. index = email['end_index']
  2455. end_index_temp = index
  2456. for j in range(begin_index, len(list_tokenbegin)):
  2457. if list_tokenbegin[j] >= index:
  2458. end_index = j - 1
  2459. break
  2460. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2461. entity_text = email['body']
  2462. list_sentence_entitys.append(
  2463. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2464. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2465. # 服务期限提取 2020/12/30 新增
  2466. list_servicetime = extract_servicetime(sentence_text)
  2467. entity_type = "serviceTime"
  2468. for servicetime in list_servicetime:
  2469. begin_index_temp = servicetime['begin_index']
  2470. for j in range(len(list_tokenbegin)):
  2471. if list_tokenbegin[j] == begin_index_temp:
  2472. begin_index = j
  2473. break
  2474. elif list_tokenbegin[j] > begin_index_temp:
  2475. begin_index = j - 1
  2476. break
  2477. index = servicetime['end_index']
  2478. end_index_temp = index
  2479. for j in range(begin_index, len(list_tokenbegin)):
  2480. if list_tokenbegin[j] >= index:
  2481. end_index = j - 1
  2482. break
  2483. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2484. entity_text = servicetime['body']
  2485. list_sentence_entitys.append(
  2486. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2487. begin_index_temp, end_index_temp,in_attachment=in_attachment, prob=servicetime["prob"]))
  2488. # 招标方式提取 2020/12/30 新增
  2489. # list_bidway = extract_bidway(sentence_text, )
  2490. # entity_type = "bidway"
  2491. # for bidway in list_bidway:
  2492. # begin_index_temp = bidway['begin_index']
  2493. # end_index_temp = bidway['end_index']
  2494. # begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
  2495. # end_index = changeIndexFromWordToWords(tokens, end_index_temp)
  2496. # if begin_index is None or end_index is None:
  2497. # continue
  2498. # print(begin_index_temp,end_index_temp,begin_index,end_index)
  2499. # entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2500. # entity_text = bidway['body']
  2501. # list_sentence_entitys.append(
  2502. # Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2503. # begin_index_temp, end_index_temp))
  2504. # 2021/12/29 新增比率提取
  2505. list_ratio = extract_ratio(sentence_text)
  2506. entity_type = "ratio"
  2507. for ratio in list_ratio:
  2508. # print("ratio", ratio)
  2509. begin_index_temp = ratio['begin_index']
  2510. for j in range(len(list_tokenbegin)):
  2511. if list_tokenbegin[j] == begin_index_temp:
  2512. begin_index = j
  2513. break
  2514. elif list_tokenbegin[j] > begin_index_temp:
  2515. begin_index = j - 1
  2516. break
  2517. index = ratio['end_index']
  2518. end_index_temp = index
  2519. for j in range(begin_index, len(list_tokenbegin)):
  2520. if list_tokenbegin[j] >= index:
  2521. end_index = j - 1
  2522. break
  2523. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2524. entity_text = ratio['body']
  2525. list_sentence_entitys.append(
  2526. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2527. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2528. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  2529. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  2530. # 补充ner模型未识别全的company/org实体
  2531. for sentence_index in range(len(list_sentence)):
  2532. sentence_text = list_sentence[sentence_index].sentence_text
  2533. tokens = list_sentence[sentence_index].tokens
  2534. doc_id = list_sentence[sentence_index].doc_id
  2535. in_attachment = list_sentence[sentence_index].in_attachment
  2536. list_tokenbegin = []
  2537. begin = 0
  2538. for i in range(0, len(tokens)):
  2539. list_tokenbegin.append(begin)
  2540. begin += len(str(tokens[i]))
  2541. list_tokenbegin.append(begin + 1)
  2542. add_sentence_entitys = []
  2543. company_dict = sorted(list(company_dict),key=lambda x:len(x[1]),reverse=True)
  2544. for company_type,company_text in company_dict:
  2545. begin_index_list = findAllIndex(company_text,sentence_text)
  2546. for begin_index in begin_index_list:
  2547. is_continue = False
  2548. for t_begin,t_end in list(company_index[sentence_index]):
  2549. if begin_index>=t_begin and begin_index+len(company_text)<=t_end:
  2550. is_continue = True
  2551. break
  2552. if not is_continue:
  2553. add_sentence_entitys.append((begin_index,begin_index+len(company_text),company_type,company_text))
  2554. company_index[sentence_index].add((begin_index,begin_index+len(company_text)))
  2555. else:
  2556. continue
  2557. for ner_entity in add_sentence_entitys:
  2558. begin_index_temp = ner_entity[0]
  2559. end_index_temp = ner_entity[1]
  2560. entity_type = ner_entity[2]
  2561. entity_text = ner_entity[3]
  2562. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  2563. continue
  2564. for j in range(len(list_tokenbegin)):
  2565. if list_tokenbegin[j]==begin_index_temp:
  2566. begin_index = j
  2567. break
  2568. elif list_tokenbegin[j]>begin_index_temp:
  2569. begin_index = j-1
  2570. break
  2571. begin_index_temp += len(str(entity_text))
  2572. for j in range(begin_index,len(list_tokenbegin)):
  2573. if list_tokenbegin[j]>=begin_index_temp:
  2574. end_index = j-1
  2575. break
  2576. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2577. #去掉标点符号
  2578. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  2579. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  2580. list_entitys_temp.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
  2581. list_entitys_temp.sort(key=lambda x:(x.sentence_index,x.begin_index))
  2582. list_entitys.append(list_entitys_temp)
  2583. return list_entitys
  2584. def union_result(codeName,prem):
  2585. '''
  2586. @summary:模型的结果拼成字典
  2587. @param:
  2588. codeName:编号名称模型的结果字典
  2589. prem:拿到属性的角色的字典
  2590. @return:拼接起来的字典
  2591. '''
  2592. result = []
  2593. assert len(codeName)==len(prem)
  2594. for item_code,item_prem in zip(codeName,prem):
  2595. result.append(dict(item_code,**item_prem))
  2596. return result
  2597. def persistenceData(data):
  2598. '''
  2599. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  2600. '''
  2601. import psycopg2
  2602. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  2603. cursor = conn.cursor()
  2604. for item_index in range(len(data)):
  2605. item = data[item_index]
  2606. doc_id = item[0]
  2607. dic = item[1]
  2608. code = dic['code']
  2609. name = dic['name']
  2610. prem = dic['prem']
  2611. if len(code)==0:
  2612. code_insert = ""
  2613. else:
  2614. code_insert = ";".join(code)
  2615. prem_insert = ""
  2616. for item in prem:
  2617. for x in item:
  2618. if isinstance(x, list):
  2619. if len(x)>0:
  2620. for x1 in x:
  2621. prem_insert+="/".join(x1)+","
  2622. prem_insert+="$"
  2623. else:
  2624. prem_insert+=str(x)+"$"
  2625. prem_insert+=";"
  2626. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  2627. cursor.execute(sql)
  2628. conn.commit()
  2629. conn.close()
  2630. def persistenceData1(list_entitys,list_sentences):
  2631. '''
  2632. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  2633. '''
  2634. import psycopg2
  2635. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  2636. cursor = conn.cursor()
  2637. for list_entity in list_entitys:
  2638. for entity in list_entity:
  2639. if entity.values is not None:
  2640. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  2641. else:
  2642. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  2643. cursor.execute(sql)
  2644. for list_sentence in list_sentences:
  2645. for sentence in list_sentence:
  2646. str_tokens = "["
  2647. for item in sentence.tokens:
  2648. str_tokens += "'"
  2649. if item=="'":
  2650. str_tokens += "''"
  2651. else:
  2652. str_tokens += item
  2653. str_tokens += "',"
  2654. str_tokens = str_tokens[:-1]+"]"
  2655. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  2656. cursor.execute(sql)
  2657. conn.commit()
  2658. conn.close()
  2659. def _handle(item,result_queue):
  2660. dochtml = item["dochtml"]
  2661. docid = item["docid"]
  2662. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  2663. flag = False
  2664. if list_innerTable:
  2665. flag = True
  2666. for table in list_innerTable:
  2667. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  2668. def getPredictTable():
  2669. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  2670. import pandas as pd
  2671. import json
  2672. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  2673. df = pd.read_csv(filename)
  2674. df_data = {"json_table":[],"docid":[]}
  2675. _count = 0
  2676. _sum = len(df["docid"])
  2677. task_queue = Queue()
  2678. result_queue = Queue()
  2679. _index = 0
  2680. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  2681. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  2682. _index += 1
  2683. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  2684. mh.run()
  2685. while True:
  2686. try:
  2687. item = result_queue.get(block=True,timeout=1)
  2688. df_data["docid"].append(item["docid"])
  2689. df_data["json_table"].append(item["json_table"])
  2690. except Exception as e:
  2691. print(e)
  2692. break
  2693. df_1 = pd.DataFrame(df_data)
  2694. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  2695. if __name__=="__main__":
  2696. '''
  2697. import glob
  2698. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  2699. file_txt = str(file).replace("html","txt")
  2700. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  2701. f.write("\n================\n")
  2702. content = codecs.open(file,"r",encoding="utf8").read()
  2703. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2704. '''
  2705. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  2706. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2707. # getPredictTable()
  2708. with open('D:/138786703.html', 'r', encoding='utf-8') as f:
  2709. sourceContent = f.read()
  2710. # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
  2711. # print(article_processed)
  2712. list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
  2713. for entity in list_entitys[0]:
  2714. print(entity.entity_type, entity.entity_text)