Preprocessing.py 173 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344
  1. # -*- coding: utf-8 -*-
  2. from bs4 import BeautifulSoup, Comment
  3. import copy
  4. import sys
  5. import os
  6. import time
  7. import codecs
  8. from BiddingKG.dl.ratio.re_ratio import extract_ratio
  9. from BiddingKG.dl.table_head.predict import predict
  10. sys.setrecursionlimit(1000000)
  11. sys.path.append(os.path.abspath("../.."))
  12. sys.path.append(os.path.abspath(".."))
  13. from BiddingKG.dl.common.Utils import *
  14. from BiddingKG.dl.interface.Entitys import *
  15. from BiddingKG.dl.interface.predictor import getPredictor, TableTag2List
  16. from BiddingKG.dl.common.nerUtils import *
  17. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  18. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  19. from BiddingKG.dl.relation_extraction.re_email import extract_email
  20. from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
  21. from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
  22. from BiddingKG.dl.entityLink.entityLink import *
  23. #
  24. def tableToText(soup):
  25. '''
  26. @param:
  27. soup:网页html的soup
  28. @return:处理完表格信息的网页text
  29. '''
  30. def getTrs(tbody):
  31. #获取所有的tr
  32. trs = []
  33. objs = tbody.find_all(recursive=False)
  34. for obj in objs:
  35. if obj.name=="tr":
  36. trs.append(obj)
  37. if obj.name=="tbody":
  38. for tr in obj.find_all("tr",recursive=False):
  39. trs.append(tr)
  40. return trs
  41. def fixSpan(tbody):
  42. # 处理colspan, rowspan信息补全问题
  43. #trs = tbody.findChildren('tr', recursive=False)
  44. trs = getTrs(tbody)
  45. ths_len = 0
  46. ths = list()
  47. trs_set = set()
  48. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  49. # 遍历每一个tr
  50. for indtr, tr in enumerate(trs):
  51. ths_tmp = tr.findChildren('th', recursive=False)
  52. #不补全含有表格的tr
  53. if len(tr.findChildren('table'))>0:
  54. continue
  55. if len(ths_tmp) > 0:
  56. ths_len = ths_len + len(ths_tmp)
  57. for th in ths_tmp:
  58. ths.append(th)
  59. trs_set.add(tr)
  60. # 遍历每行中的element
  61. tds = tr.findChildren(recursive=False)
  62. for indtd, td in enumerate(tds):
  63. # 若有colspan 则补全同一行下一个位置
  64. if 'colspan' in td.attrs:
  65. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  66. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  67. if col<100 and len(td.get_text())<1000:
  68. td['colspan'] = 1
  69. for i in range(1, col, 1):
  70. td.insert_after(copy.copy(td))
  71. for indtr, tr in enumerate(trs):
  72. ths_tmp = tr.findChildren('th', recursive=False)
  73. #不补全含有表格的tr
  74. if len(tr.findChildren('table'))>0:
  75. continue
  76. if len(ths_tmp) > 0:
  77. ths_len = ths_len + len(ths_tmp)
  78. for th in ths_tmp:
  79. ths.append(th)
  80. trs_set.add(tr)
  81. # 遍历每行中的element
  82. tds = tr.findChildren(recursive=False)
  83. for indtd, td in enumerate(tds):
  84. # 若有rowspan 则补全下一行同样位置
  85. if 'rowspan' in td.attrs:
  86. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  87. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  88. td['rowspan'] = 1
  89. for i in range(1, row, 1):
  90. # 获取下一行的所有td, 在对应的位置插入
  91. if indtr+i<len(trs):
  92. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  93. if len(tds1) >= (indtd) and len(tds1)>0:
  94. if indtd > 0:
  95. tds1[indtd - 1].insert_after(copy.copy(td))
  96. else:
  97. tds1[0].insert_before(copy.copy(td))
  98. elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全
  99. tds1[indtd-2].insert_after(copy.copy(td))
  100. def getTable(tbody):
  101. #trs = tbody.findChildren('tr', recursive=False)
  102. trs = getTrs(tbody)
  103. inner_table = []
  104. for tr in trs:
  105. tr_line = []
  106. tds = tr.findChildren(['td','th'], recursive=False)
  107. if len(tds)==0:
  108. tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
  109. for td in tds:
  110. tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
  111. #tr_line.append([td.get_text(),0])
  112. inner_table.append(tr_line)
  113. return inner_table
  114. #处理表格不对齐的问题
  115. def fixTable(inner_table,fix_value="~~"):
  116. maxWidth = 0
  117. for item in inner_table:
  118. if len(item)>maxWidth:
  119. maxWidth = len(item)
  120. if maxWidth > 100:
  121. # log('表格列数大于100,表格异常不做处理。')
  122. return []
  123. for i in range(len(inner_table)):
  124. if len(inner_table[i])<maxWidth:
  125. for j in range(maxWidth-len(inner_table[i])):
  126. inner_table[i].append([fix_value,0])
  127. return inner_table
  128. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  129. height = len(inner_table)
  130. width = len(inner_table[0])
  131. for i in range(height):
  132. point = ""
  133. for j in range(width):
  134. if inner_table[i][j][0]==point and point!="":
  135. inner_table[i][j][0] = pad_row
  136. else:
  137. if inner_table[i][j][0] not in [pad_row,pad_col]:
  138. point = inner_table[i][j][0]
  139. for j in range(width):
  140. point = ""
  141. for i in range(height):
  142. if inner_table[i][j][0]==point and point!="":
  143. inner_table[i][j][0] = pad_col
  144. else:
  145. if inner_table[i][j][0] not in [pad_row,pad_col]:
  146. point = inner_table[i][j][0]
  147. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  148. height = len(inner_table)
  149. width = len(inner_table[0])
  150. for i in range(height):
  151. for j in range(width):
  152. if inner_table[i][j][0]==pad_row:
  153. inner_table[i][j][0] = inner_table[i][j-1][0]
  154. inner_table[i][j][1] = inner_table[i][j-1][1]
  155. if inner_table[i][j][0]==pad_col:
  156. inner_table[i][j][0] = inner_table[i-1][j][0]
  157. inner_table[i][j][1] = inner_table[i-1][j][1]
  158. def repairTable(inner_table, dye_set=set(), key_set=set(), fix_value="~~"):
  159. """
  160. @summary: 修复表头识别,将明显错误的进行修正
  161. """
  162. def repairNeeded(line):
  163. first_1 = -1
  164. last_1 = -1
  165. first_0 = -1
  166. last_0 = -1
  167. count_1 = 0
  168. count_0 = 0
  169. for i in range(len(line)):
  170. if line[i][0]==fix_value:
  171. continue
  172. if line[i][1]==1:
  173. if first_1==-1:
  174. first_1 = i
  175. last_1 = i
  176. count_1 += 1
  177. if line[i][1]==0:
  178. if first_0 == -1:
  179. first_0 = i
  180. last_0 = i
  181. count_0 += 1
  182. if first_1 ==-1 or last_0 == -1:
  183. return False
  184. # 异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  185. if first_1-0 > 0 or last_0-len(line)+1 < 0 or last_1 == len(line)-1 or count_1-count_0 >= 3:
  186. return True
  187. return False
  188. def getsimilarity(line, line1):
  189. same_count = 0
  190. for item, item1 in zip(line,line1):
  191. if item[1] == item1[1]:
  192. same_count += 1
  193. return same_count/len(line)
  194. def selfrepair(inner_table,index,dye_set,key_set):
  195. """
  196. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  197. """
  198. #print("B",inner_table[index])
  199. min_presure = 3
  200. list_dye = []
  201. first = None
  202. count = 0
  203. # temp_set = set()
  204. temp_set = set(['~~']) # 2023/10/10纠正236239652 受让单位识别不到表头; 受让单位,明细用途:用途名称:陵川县民政局,
  205. _index = 0
  206. for item in inner_table[index]:
  207. if first is None:
  208. first = item[1]
  209. if item[0] not in temp_set:
  210. count += 1
  211. temp_set.add(item[0])
  212. else:
  213. if first == item[1]:
  214. if item[0] not in temp_set:
  215. temp_set.add(item[0])
  216. count += 1
  217. else:
  218. list_dye.append([first,count,_index])
  219. first = item[1]
  220. temp_set.add(item[0])
  221. count = 1
  222. _index += 1
  223. list_dye.append([first,count,_index])
  224. if len(list_dye)>1:
  225. begin = 0
  226. end = 0
  227. for i in range(len(list_dye)):
  228. end = list_dye[i][2]
  229. dye_flag = False
  230. # 首尾要求压力减一
  231. if i==0:
  232. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  233. dye_flag = True
  234. dye_type = list_dye[i+1][0]
  235. elif i==len(list_dye)-1:
  236. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  237. dye_flag = True
  238. dye_type = list_dye[i-1][0]
  239. else:
  240. if list_dye[i][1]>1:
  241. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  242. dye_flag = True
  243. dye_type = list_dye[i+1][0]
  244. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  245. dye_flag = True
  246. dye_type = list_dye[i-1][0]
  247. else:
  248. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  249. dye_flag = True
  250. dye_type = list_dye[i+1][0]
  251. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  252. dye_flag = True
  253. dye_type = list_dye[i-1][0]
  254. if dye_flag:
  255. for h in range(begin,end):
  256. inner_table[index][h][1] = dye_type
  257. dye_set.add((inner_table[index][h][0],dye_type))
  258. key_set.add(inner_table[index][h][0])
  259. begin = end
  260. #print("E",inner_table[index])
  261. def otherrepair(inner_table,index,dye_set,key_set):
  262. list_provide_repair = []
  263. if index==0 and len(inner_table)>1:
  264. list_provide_repair.append(index+1)
  265. elif index==len(inner_table)-1:
  266. list_provide_repair.append(index-1)
  267. else:
  268. list_provide_repair.append(index+1)
  269. list_provide_repair.append(index-1)
  270. for provide_index in list_provide_repair:
  271. if not repairNeeded(inner_table[provide_index]):
  272. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  273. if same_prob>=0.8:
  274. for i in range(len(inner_table[provide_index])):
  275. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  276. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  277. key_set.add(inner_table[index][i][0])
  278. inner_table[index][i][1] = inner_table[provide_index][i][1]
  279. elif same_prob<=0.2:
  280. for i in range(len(inner_table[provide_index])):
  281. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  282. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  283. key_set.add(inner_table[index][i][0])
  284. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  285. len_dye_set = len(dye_set)
  286. height = len(inner_table)
  287. for i in range(height):
  288. if repairNeeded(inner_table[i]):
  289. selfrepair(inner_table, i, dye_set, key_set)
  290. #otherrepair(inner_table,i,dye_set,key_set)
  291. for h in range(len(inner_table)):
  292. for w in range(len(inner_table[0])):
  293. if inner_table[h][w][0] in key_set:
  294. for item in dye_set:
  295. if inner_table[h][w][0] == item[0]:
  296. inner_table[h][w][1] = item[1]
  297. # 如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  298. if len(dye_set) != len(key_set):
  299. for i in range(height):
  300. if repairNeeded(inner_table[i]):
  301. selfrepair(inner_table,i,dye_set,key_set)
  302. #otherrepair(inner_table,i,dye_set,key_set)
  303. return
  304. if len(dye_set) == len_dye_set:
  305. '''
  306. for i in range(height):
  307. if repairNeeded(inner_table[i]):
  308. otherrepair(inner_table,i,dye_set,key_set)
  309. '''
  310. return
  311. repairTable(inner_table, dye_set, key_set)
  312. def repair_table2(inner_table):
  313. """
  314. @summary: 修复表头识别,将明显错误的进行修正
  315. """
  316. # 修复第一第二第三中标候选人作为列表头
  317. if len(inner_table) >= 2 and len(inner_table[0]) >= 3:
  318. for i in range(len(inner_table[:3])):
  319. for j in range(len(inner_table[i]) - 2):
  320. if inner_table[i][j][0] == '第一中标候选人' \
  321. and inner_table[i][j + 1][0] == '第二中标候选人' \
  322. and inner_table[i][j + 2][0] == '第三中标候选人' \
  323. and i + 1 < len(inner_table) \
  324. and inner_table[i + 1][j][1] == 0 \
  325. and inner_table[i + 1][j + 1][1] == 0 \
  326. and inner_table[i + 1][j + 2][1] == 0:
  327. inner_table[i][j][1] = 1
  328. inner_table[i][j + 1][1] = 1
  329. inner_table[i][j + 2][1] = 1
  330. break
  331. # 修复连续的第一第二第三候选人行表头
  332. for i in range(len(inner_table)):
  333. for j in range(len(inner_table[i])):
  334. only_chinese1 = ''.join(re.findall('[\u4e00-\u9fa5]+', inner_table[i][j][0]))
  335. if only_chinese1 in ['第一候选人', '第一中标候选人'] and inner_table[i][j][1] == 0:
  336. if j + 1 < len(inner_table[i]) and ''.join(
  337. re.findall('[\u4e00-\u9fa5]+', inner_table[i][j + 1][0])) in ['第二候选人', '第二中标候选人']:
  338. inner_table[i][j][1] = 1
  339. inner_table[i][j + 1][1] = 1
  340. if j + 2 < len(inner_table[i]) and ''.join(
  341. re.findall('[\u4e00-\u9fa5]+', inner_table[i][j + 2][0])) in ['第三候选人', '第三中标候选人']:
  342. inner_table[i][j + 2][1] = 1
  343. # 修复多个重复的单元格表头不一致
  344. for i in range(len(inner_table)):
  345. for j in range(len(inner_table[i]) - 1):
  346. only_chinese1 = ''.join(re.findall('[\u4e00-\u9fa5]+', inner_table[i][j][0]))
  347. only_chinese2 = ''.join(re.findall('[\u4e00-\u9fa5]+', inner_table[i][j + 1][0]))
  348. if only_chinese1 == only_chinese2 and inner_table[i][j][1] != inner_table[i][j + 1][1]:
  349. inner_table[i][j][1] = 1
  350. inner_table[i][j + 1][1] = 1
  351. # 修复一行几乎都是表头,个别不是;或者一行几乎都是非表头,个别是
  352. for i in range(len(inner_table)):
  353. head_dict = {}
  354. not_head_dict = {}
  355. for j in range(len(inner_table[i])):
  356. if inner_table[i][j][1] == 1:
  357. if inner_table[i][j][0] not in head_dict:
  358. head_dict[inner_table[i][j][0]] = 1
  359. else:
  360. if inner_table[i][j][0] not in not_head_dict:
  361. not_head_dict[inner_table[i][j][0]] = 1
  362. # 非表头:表头 <= 1:3
  363. if len(head_dict.keys()) > 0 and len(not_head_dict.keys()) / len(head_dict.keys()) <= 1 / 3 and len(
  364. head_dict.keys()) >= 3:
  365. for j in range(len(inner_table[i])):
  366. if len(re.sub(' ', '', inner_table[i][j][0])) > 0:
  367. inner_table[i][j][1] = 1
  368. # 表头数一个且非表头数大于2且上一行都是表头
  369. if i > 0 and len(head_dict.keys()) == 1 and len(not_head_dict.keys()) >= 2 and inner_table[i][0][1] == 0:
  370. last_row = inner_table[i - 1]
  371. col_list = []
  372. for j in range(len(last_row)):
  373. if len(re.sub(' ', '', last_row[j][0])) > 0:
  374. if last_row[j][1] == 0:
  375. col_list = []
  376. break
  377. col_list.append(last_row[j][0])
  378. if col_list:
  379. col_list = list(set(col_list))
  380. if len(col_list) > 2:
  381. for j in range(len(inner_table[i])):
  382. if inner_table[i][j][1] == 1:
  383. inner_table[i][j][1] = 0
  384. # 修复冒号在文本中间的,不能作为表头
  385. for i in range(len(inner_table)):
  386. for j in range(len(inner_table[i])):
  387. _text = inner_table[i][j][0]
  388. if len(_text) >= 3 and inner_table[i][j][1] == 1:
  389. match = re.search('[::]', _text)
  390. if match:
  391. start_index, end_index = match.span()
  392. if start_index == 0 or end_index == len(_text):
  393. continue
  394. if re.search('[\u4e00-\u9fa50-9a-zA-Z]', _text[:start_index]) and re.search(
  395. '[\u4e00-\u9fa50-9a-zA-Z]', _text[end_index:]):
  396. inner_table[i][j][1] = 0
  397. # 修复表头关键词未作为表头
  398. head_keyword = ['供应商']
  399. for i in range(len(inner_table)):
  400. for j in range(len(inner_table[i])):
  401. match = re.search('[\u4e00-\u9fa50-9a-zA-Z::]+', inner_table[i][j][0])
  402. if inner_table[i][j][1] == 0 and match and match.group() in head_keyword:
  403. inner_table[i][j][1] = 1
  404. # 修复姓名被作为表头 # 2023-02-10 取消修复,避免项目名称、编号,单位、单价等作为了非表头
  405. # surname = [
  406. # "赵", "钱", "孙", "李", "周", "吴", "郑", "王", "冯", "陈", "褚", "卫", "蒋", "沈", "韩", "杨", "朱", "秦", "尤", "许", "何", "吕", "施", "张", "孔", "曹", "严", "华", "金", "魏", "陶", "姜", "戚", "谢", "邹", "喻", "柏", "水", "窦", "章", "云", "苏", "潘", "葛", "奚", "范", "彭", "郎", "鲁", "韦", "昌", "马", "苗", "凤", "花", "方", "俞", "任", "袁", "柳", "酆", "鲍", "史", "唐", "费", "廉", "岑", "薛", "雷", "贺", "倪", "汤", "滕", "殷", "罗", "毕", "郝", "邬", "安", "常", "乐", "于", "时", "傅", "皮", "卞", "齐", "康", "伍", "余", "元", "卜", "顾", "孟", "平", "黄", "和", "穆", "萧", "尹", "姚", "邵", "湛", "汪", "祁", "毛", "禹", "狄", "米", "贝", "明", "臧", "计", "伏", "成", "戴", "谈", "宋", "茅", "庞", "熊", "纪", "舒", "屈", "项", "祝", "董", "梁", "杜", "阮", "蓝", "闵", "席", "季", "麻", "强", "贾", "路", "娄", "危", "江", "童", "颜", "郭", "梅", "盛", "林", "刁", "钟", "徐", "邱", "骆", "高", "夏", "蔡", "田", "樊", "胡", "凌", "霍", "虞", "万", "支", "柯", "昝", "管", "卢", "莫", "经", "房", "裘", "缪", "干", "解", "应", "宗", "丁", "宣", "贲", "邓", "郁", "单", "杭", "洪", "包", "诸", "左", "石", "崔", "吉", "钮", "龚", "程", "嵇", "邢", "滑", "裴", "陆", "荣", "翁", "荀", "羊", "於", "惠", "甄", "麴", "家", "封", "芮", "羿", "储", "靳", "汲", "邴", "糜", "松", "井", "段", "富", "巫", "乌", "焦", "巴", "弓", "牧", "隗", "山", "谷", "车", "侯", "宓", "蓬", "全", "郗", "班", "仰", "秋", "仲", "伊", "宫", "宁", "仇", "栾", "暴", "甘", "钭", "厉", "戎", "祖", "武", "符", "刘", "景", "詹", "束", "龙", "叶", "幸", "司", "韶", "郜", "黎", "蓟", "薄", "印", "宿", "白", "怀", "蒲", "邰", "从", "鄂", "索", "咸", "籍", "赖", "卓", "蔺", "屠", "蒙", "池", "乔", "阴", "欎", "胥", "能", "苍", "双", "闻", "莘", "党", "翟", "谭", "贡", "劳", "逄", "姬", "申", "扶", "堵", "冉", "宰", "郦", "雍", "舄", "璩", "桑", "桂", "濮", "牛", "寿", "通", "边", "扈", "燕", "冀", "郏", "浦", "尚", "农", "温", "别", "庄", "晏", "柴", "瞿", "阎", "充", "慕", "连", "茹", "习", "宦", "艾", "鱼", "容", "向", "古", "易", "慎", "戈", "廖", "庾", "终", "暨", "居", "衡", "步", "都", "耿", "满", "弘", "匡", "国", "文", "寇", "广", "禄", "阙", "东", "殴", "殳", "沃", "利", "蔚", "越", "夔", "隆", "师", "巩", "厍", "聂", "晁", "勾", "敖", "融", "冷", "訾", "辛", "阚", "那", "简", "饶", "空", "曾", "毋", "沙", "乜", "养", "鞠", "须", "丰", "巢", "关", "蒯", "相", "查", "後", "荆", "红", "游", "竺", "权", "逯", "盖", "益", "桓", "公", "万俟", "司马", "上官", "欧阳", "夏侯", "诸葛", "闻人", "东方", "赫连", "皇甫", "尉迟", "公羊", "澹台", "公冶", "宗政", "濮阳", "淳于", "单于", "太叔", "申屠", "公孙", "仲孙", "轩辕", "令狐", "钟离", "宇文", "长孙", "慕容", "鲜于", "闾丘", "司徒", "司空", "亓官", "司寇", "仉", "督", "子车", "颛孙", "端木", "巫马", "公西", "漆雕", "乐正", "壤驷", "公良", "拓跋", "夹谷", "宰父", "谷梁", "晋", "楚", "闫", "法", "汝", "鄢", "涂", "钦", "段干", "百里", "东郭", "南门", "呼延", "归", "海", "羊舌", "微生", "岳", "帅", "缑", "亢", "况", "后", "有", "琴", "梁丘", "左丘", "东门", "西门", "商", "牟", "佘", "佴", "伯", "赏", "南宫", "墨", "哈", "谯", "笪", "年", "爱", "阳", "佟", "第五", "言", "福",
  407. # ]
  408. # for i in range(len(inner_table)):
  409. # for j in range(len(inner_table[i])):
  410. # if inner_table[i][j][1] == 1 \
  411. # and 2 <= len(inner_table[i][j][0]) <= 4 \
  412. # and (inner_table[i][j][0][0] in surname or inner_table[i][j][0][:2] in surname) \
  413. # and re.search("[^\u4e00-\u9fa5]", inner_table[i][j][0]) is None:
  414. # inner_table[i][j][1] = 0
  415. return inner_table
  416. def sliceTable(inner_table,fix_value="~~"):
  417. #进行分块
  418. height = len(inner_table)
  419. width = len(inner_table[0])
  420. head_list = []
  421. head_list.append(0)
  422. last_head = None
  423. last_is_same_value = False
  424. for h in range(height):
  425. is_all_key = True#是否是全表头行
  426. is_all_value = True#是否是全属性值
  427. is_same_with_lastHead = True#和上一行的结构是否相同
  428. is_same_value=True#一行的item都一样
  429. #is_same_first_item = True#与上一行的第一项是否相同
  430. same_value = inner_table[h][0][0]
  431. for w in range(width):
  432. if last_head is not None:
  433. if inner_table[h-1][w][0] != fix_value and inner_table[h-1][w][0] != "" and inner_table[h-1][w][1] == 0:
  434. is_all_key = False
  435. if inner_table[h][w][0]==1:
  436. is_all_value = False
  437. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  438. is_same_with_lastHead = False
  439. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  440. is_same_value = False
  441. else:
  442. if re.search("\d+",same_value) is not None:
  443. is_same_value = False
  444. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  445. is_same_first_item = False
  446. last_head = h
  447. if last_is_same_value:
  448. last_is_same_value = is_same_value
  449. continue
  450. if is_same_value:
  451. # 该块只有表头一行不合法
  452. if h - head_list[-1] > 1:
  453. head_list.append(h)
  454. last_is_same_value = is_same_value
  455. continue
  456. if not is_all_key:
  457. if not is_same_with_lastHead:
  458. # 该块只有表头一行不合法
  459. if h - head_list[-1] > 1:
  460. head_list.append(h)
  461. head_list.append(height)
  462. return head_list
  463. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  464. set_item = set()
  465. height = len(inner_table)
  466. width = len(inner_table[0])
  467. empty_set = set()
  468. for i in range(height):
  469. for j in range(width):
  470. item = inner_table[i][j][0]
  471. if item.strip()=="":
  472. empty_set.add(item)
  473. else:
  474. set_item.add(item)
  475. list_item = list(set_item)
  476. if list_item:
  477. x = []
  478. for item in list_item:
  479. x.append(getPredictor("form").encode(item))
  480. predict_y = getPredictor("form").predict(np.array(x),type="item")
  481. _dict = dict()
  482. for item,values in zip(list_item,list(predict_y)):
  483. _dict[item] = values[1]
  484. # print("##",item,values)
  485. #print(_dict)
  486. for i in range(height):
  487. for j in range(width):
  488. item = inner_table[i][j][0]
  489. if item not in empty_set:
  490. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  491. # print("=====")
  492. # for item in inner_table:
  493. # print(item)
  494. # print("======")
  495. repairTable(inner_table)
  496. head_list = sliceTable(inner_table)
  497. return inner_table,head_list
  498. def set_head_model(inner_table):
  499. origin_inner_table = copy.deepcopy(inner_table)
  500. for i in range(len(inner_table)):
  501. for j in range(len(inner_table[i])):
  502. # 删掉单格前后符号,以免影响表头预测
  503. col = inner_table[i][j][0]
  504. col = re.sub("^[^\u4e00-\u9fa5a-zA-Z0-9]+", "", col)
  505. col = re.sub("[^\u4e00-\u9fa5a-zA-Z0-9]+$", "", col)
  506. inner_table[i][j] = col
  507. # 模型预测表头
  508. predict_list = predict(inner_table)
  509. # 组合结果
  510. for i in range(len(inner_table)):
  511. for j in range(len(inner_table[i])):
  512. inner_table[i][j] = [origin_inner_table[i][j][0], int(predict_list[i][j])]
  513. # print("table_head before repair", inner_table)
  514. # 表头修正
  515. # repairTable(inner_table)
  516. inner_table = repair_table2(inner_table)
  517. # 按表头分割表格
  518. head_list = sliceTable(inner_table)
  519. return inner_table, head_list
  520. def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  521. data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
  522. predict_y = getPredictor("form").getModel("context").predict(data_x)
  523. for _position,_y in zip(data_position,predict_y):
  524. _w = _position[0]
  525. _h = _position[1]
  526. if _y[1]>prob_min:
  527. inner_table[_h][_w][1] = 1
  528. else:
  529. inner_table[_h][_w][1] = 0
  530. _item = inner_table[_h][_w][0]
  531. if re.search(pat_head,_item) is not None and len(_item)<8:
  532. inner_table[_h][_w][1] = 1
  533. # print("=====")
  534. # for item in inner_table:
  535. # print(item)
  536. # print("======")
  537. height = len(inner_table)
  538. width = len(inner_table[0])
  539. for i in range(height):
  540. for j in range(width):
  541. if re.search("[::]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
  542. inner_table[i][j][1] = 1
  543. repairTable(inner_table)
  544. head_list = sliceTable(inner_table)
  545. # print("inner_table:",inner_table)
  546. return inner_table,head_list
  547. #设置表头
  548. def setHead_inline(inner_table,prob_min=0.64):
  549. pad_row = "@@"
  550. pad_col = "##"
  551. removePadding(inner_table, pad_row, pad_col)
  552. pad_pattern = re.compile(pad_row+"|"+pad_col)
  553. height = len(inner_table)
  554. width = len(inner_table[0])
  555. head_list = []
  556. head_list.append(0)
  557. #行表头
  558. is_head_last = False
  559. for i in range(height):
  560. is_head = False
  561. is_long_value = False
  562. #判断是否是全padding值
  563. is_same_value = True
  564. same_value = inner_table[i][0][0]
  565. for j in range(width):
  566. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  567. is_same_value = False
  568. break
  569. #predict is head or not with model
  570. temp_item = ""
  571. for j in range(width):
  572. temp_item += inner_table[i][j][0]+"|"
  573. temp_item = re.sub(pad_pattern,"",temp_item)
  574. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  575. if form_prob is not None:
  576. if form_prob[0][1]>prob_min:
  577. is_head = True
  578. else:
  579. is_head = False
  580. #print(temp_item,form_prob)
  581. if len(inner_table[i][0][0])>40:
  582. is_long_value = True
  583. if is_head or is_long_value or is_same_value:
  584. #不把连续表头分开
  585. if not is_head_last:
  586. head_list.append(i)
  587. if is_long_value or is_same_value:
  588. head_list.append(i+1)
  589. if is_head:
  590. for j in range(width):
  591. inner_table[i][j][1] = 1
  592. is_head_last = is_head
  593. head_list.append(height)
  594. #列表头
  595. for i in range(len(head_list)-1):
  596. head_begin = head_list[i]
  597. head_end = head_list[i+1]
  598. #最后一列不设置为列表头
  599. for i in range(width-1):
  600. is_head = False
  601. #predict is head or not with model
  602. temp_item = ""
  603. for j in range(head_begin,head_end):
  604. temp_item += inner_table[j][i][0]+"|"
  605. temp_item = re.sub(pad_pattern,"",temp_item)
  606. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  607. if form_prob is not None:
  608. if form_prob[0][1]>prob_min:
  609. is_head = True
  610. else:
  611. is_head = False
  612. if is_head:
  613. for j in range(head_begin,head_end):
  614. inner_table[j][i][1] = 2
  615. addPadding(inner_table, pad_row, pad_col)
  616. return inner_table,head_list
  617. #设置表头
  618. def setHead_withRule(inner_table,pattern,pat_value,count):
  619. height = len(inner_table)
  620. width = len(inner_table[0])
  621. head_list = []
  622. head_list.append(0)
  623. #行表头
  624. is_head_last = False
  625. for i in range(height):
  626. set_match = set()
  627. is_head = False
  628. is_long_value = False
  629. is_same_value = True
  630. same_value = inner_table[i][0][0]
  631. for j in range(width):
  632. if inner_table[i][j][0]!=same_value:
  633. is_same_value = False
  634. break
  635. for j in range(width):
  636. if re.search(pat_value,inner_table[i][j][0]) is not None:
  637. is_head = False
  638. break
  639. str_find = re.findall(pattern,inner_table[i][j][0])
  640. if len(str_find)>0:
  641. set_match.add(inner_table[i][j][0])
  642. if len(set_match)>=count:
  643. is_head = True
  644. if len(inner_table[i][0][0])>40:
  645. is_long_value = True
  646. if is_head or is_long_value or is_same_value:
  647. if not is_head_last:
  648. head_list.append(i)
  649. if is_head:
  650. for j in range(width):
  651. inner_table[i][j][1] = 1
  652. is_head_last = is_head
  653. head_list.append(height)
  654. #列表头
  655. for i in range(len(head_list)-1):
  656. head_begin = head_list[i]
  657. head_end = head_list[i+1]
  658. #最后一列不设置为列表头
  659. for i in range(width-1):
  660. set_match = set()
  661. is_head = False
  662. for j in range(head_begin,head_end):
  663. if re.search(pat_value,inner_table[j][i][0]) is not None:
  664. is_head = False
  665. break
  666. str_find = re.findall(pattern,inner_table[j][i][0])
  667. if len(str_find)>0:
  668. set_match.add(inner_table[j][i][0])
  669. if len(set_match)>=count:
  670. is_head = True
  671. if is_head:
  672. for j in range(head_begin,head_end):
  673. inner_table[j][i][1] = 2
  674. return inner_table,head_list
  675. #取得表格的处理方向
  676. def getDirect(inner_table,begin,end):
  677. '''
  678. column_head = set()
  679. row_head = set()
  680. widths = len(inner_table[0])
  681. for height in range(begin,end):
  682. for width in range(widths):
  683. if inner_table[height][width][1] ==1:
  684. row_head.add(height)
  685. if inner_table[height][width][1] ==2:
  686. column_head.add(width)
  687. company_pattern = re.compile("公司")
  688. if 0 in column_head and begin not in row_head:
  689. return "column"
  690. if 0 in column_head and begin in row_head:
  691. for height in range(begin,end):
  692. count = 0
  693. count_flag = True
  694. for width_index in range(width):
  695. if inner_table[height][width_index][1]==0:
  696. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  697. count += 1
  698. else:
  699. count_flag = False
  700. if count_flag and count>=2:
  701. return "column"
  702. return "row"
  703. '''
  704. count_row_keys = 0
  705. count_column_keys = 0
  706. width = len(inner_table[0])
  707. if begin<end:
  708. for w in range(len(inner_table[begin])):
  709. if inner_table[begin][w][1]!=0:
  710. count_row_keys += 1
  711. for h in range(begin,end):
  712. if inner_table[h][0][1]!=0:
  713. count_column_keys += 1
  714. company_pattern = re.compile("有限(责任)?公司")
  715. for height in range(begin,end):
  716. count_set = set()
  717. count_flag = True
  718. for width_index in range(width):
  719. if inner_table[height][width_index][1]==0:
  720. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  721. count_set.add(inner_table[height][width_index][0])
  722. else:
  723. count_flag = False
  724. if count_flag and len(count_set)>=2:
  725. return "column"
  726. # if count_column_keys>count_row_keys: #2022/2/15 此项不够严谨,造成很多错误,故取消
  727. # return "column"
  728. return "row"
  729. #根据表格处理方向生成句子,
  730. def getTableText(inner_table,head_list,key_direct=False):
  731. # packPattern = "(标包|[标包][号段名])"
  732. packPattern = "(标包|标的|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  733. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见|评标情况|推荐顺序)" # 2020/11/23 大网站规则,添加序号为排序
  734. entityPattern = "((候选|[中投]标|报价)(单位|公司|人|供应商))|供应商名称"
  735. moneyPattern = "([中投]标|报价)(金额|价)"
  736. height = len(inner_table)
  737. width = len(inner_table[0])
  738. text = ""
  739. for head_i in range(len(head_list)-1):
  740. head_begin = head_list[head_i]
  741. head_end = head_list[head_i+1]
  742. direct = getDirect(inner_table, head_begin, head_end)
  743. #若只有一行,则直接按行读取
  744. if head_end-head_begin==1:
  745. text_line = ""
  746. for i in range(head_begin,head_end):
  747. for w in range(len(inner_table[i])):
  748. if inner_table[i][w][1]==1:
  749. _punctuation = ":"
  750. else:
  751. _punctuation = "," #2021/12/15 统一为中文标点,避免 206893924 国际F座1108,1,009,197.49元
  752. if w>0:
  753. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  754. text_line += inner_table[i][w][0]+_punctuation
  755. else:
  756. text_line += inner_table[i][w][0]+_punctuation
  757. text_line = text_line+"。" if text_line!="" else text_line
  758. text += text_line
  759. else:
  760. #构建一个共现矩阵
  761. table_occurence = []
  762. for i in range(head_begin,head_end):
  763. line_oc = []
  764. for j in range(width):
  765. cell = inner_table[i][j]
  766. line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
  767. table_occurence.append(line_oc)
  768. occu_height = len(table_occurence)
  769. occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
  770. #为每个属性值寻找表头
  771. for i in range(occu_height):
  772. for j in range(occu_width):
  773. cell = table_occurence[i][j]
  774. #是属性值
  775. if cell["type"]==0 and cell["text"]!="":
  776. left_head = ""
  777. top_head = ""
  778. find_flag = False
  779. temp_head = ""
  780. for loop_i in range(1,i+1):
  781. if not key_direct:
  782. key_values = [1,2]
  783. else:
  784. key_values = [1]
  785. if table_occurence[i-loop_i][j]["type"] in key_values:
  786. if find_flag:
  787. if table_occurence[i-loop_i][j]["text"]!=temp_head:
  788. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  789. else:
  790. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  791. find_flag = True
  792. temp_head = table_occurence[i-loop_i][j]["text"]
  793. table_occurence[i-loop_i][j]["occu_count"] += 1
  794. else:
  795. #找到表头后遇到属性值就返回
  796. if find_flag:
  797. break
  798. cell["top_head"] += top_head
  799. find_flag = False
  800. temp_head = ""
  801. for loop_j in range(1,j+1):
  802. if not key_direct:
  803. key_values = [1,2]
  804. else:
  805. key_values = [2]
  806. if table_occurence[i][j-loop_j]["type"] in key_values:
  807. if find_flag:
  808. if table_occurence[i][j-loop_j]["text"]!=temp_head:
  809. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  810. else:
  811. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  812. find_flag = True
  813. temp_head = table_occurence[i][j-loop_j]["text"]
  814. table_occurence[i][j-loop_j]["occu_count"] += 1
  815. else:
  816. if find_flag:
  817. break
  818. cell["left_head"] += left_head
  819. if direct=="row":
  820. for i in range(occu_height):
  821. pack_text = ""
  822. rank_text = ""
  823. entity_text = ""
  824. text_line = ""
  825. money_text = ""
  826. #在同一句话中重复的可以去掉
  827. text_set = set()
  828. for j in range(width):
  829. cell = table_occurence[i][j]
  830. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  831. cell = table_occurence[i][j]
  832. head = (cell["top_head"]+":") if len(cell["top_head"])>0 else ""
  833. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  834. head = cell["left_head"] + head
  835. else:
  836. head += cell["left_head"]
  837. if str(head+cell["text"]) in text_set:
  838. continue
  839. if re.search(packPattern,head) is not None:
  840. pack_text += head+cell["text"]+","
  841. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  842. #排名替换为同一种表达
  843. rank_text += head+cell["text"]+","
  844. #print(rank_text)
  845. elif re.search(entityPattern,head) is not None:
  846. entity_text += head+cell["text"]+","
  847. #print(entity_text)
  848. else:
  849. if re.search(moneyPattern,head) is not None and entity_text!="":
  850. money_text += head+cell["text"]+","
  851. else:
  852. text_line += head+cell["text"]+","
  853. text_set.add(str(head+cell["text"]))
  854. text += pack_text+rank_text+entity_text+money_text+text_line
  855. text = text[:-1]+"。" if len(text)>0 else text
  856. else:
  857. for j in range(occu_width):
  858. pack_text = ""
  859. rank_text = ""
  860. entity_text = ""
  861. text_line = ""
  862. text_set = set()
  863. for i in range(occu_height):
  864. cell = table_occurence[i][j]
  865. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  866. cell = table_occurence[i][j]
  867. head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
  868. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  869. head = cell["top_head"] + head
  870. else:
  871. head += cell["top_head"]
  872. if str(head+cell["text"]) in text_set:
  873. continue
  874. if re.search(packPattern,head) is not None:
  875. pack_text += head+cell["text"]+","
  876. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  877. #排名替换为同一种表达
  878. rank_text += head+cell["text"]+","
  879. #print(rank_text)
  880. elif re.search(entityPattern,head) is not None and \
  881. re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
  882. entity_text += head+cell["text"]+","
  883. #print(entity_text)
  884. else:
  885. text_line += head+cell["text"]+","
  886. text_set.add(str(head+cell["text"]))
  887. text += pack_text+rank_text+entity_text+text_line
  888. text = text[:-1]+"。" if len(text)>0 else text
  889. # if direct=="row":
  890. # for i in range(head_begin,head_end):
  891. # pack_text = ""
  892. # rank_text = ""
  893. # entity_text = ""
  894. # text_line = ""
  895. # #在同一句话中重复的可以去掉
  896. # text_set = set()
  897. # for j in range(width):
  898. # cell = inner_table[i][j]
  899. # #是属性值
  900. # if cell[1]==0 and cell[0]!="":
  901. # head = ""
  902. #
  903. # find_flag = False
  904. # temp_head = ""
  905. # for loop_i in range(0,i+1-head_begin):
  906. # if not key_direct:
  907. # key_values = [1,2]
  908. # else:
  909. # key_values = [1]
  910. # if inner_table[i-loop_i][j][1] in key_values:
  911. # if find_flag:
  912. # if inner_table[i-loop_i][j][0]!=temp_head:
  913. # head = inner_table[i-loop_i][j][0]+":"+head
  914. # else:
  915. # head = inner_table[i-loop_i][j][0]+":"+head
  916. # find_flag = True
  917. # temp_head = inner_table[i-loop_i][j][0]
  918. # else:
  919. # #找到表头后遇到属性值就返回
  920. # if find_flag:
  921. # break
  922. #
  923. # find_flag = False
  924. # temp_head = ""
  925. #
  926. #
  927. #
  928. # for loop_j in range(1,j+1):
  929. # if not key_direct:
  930. # key_values = [1,2]
  931. # else:
  932. # key_values = [2]
  933. # if inner_table[i][j-loop_j][1] in key_values:
  934. # if find_flag:
  935. # if inner_table[i][j-loop_j][0]!=temp_head:
  936. # head = inner_table[i][j-loop_j][0]+":"+head
  937. # else:
  938. # head = inner_table[i][j-loop_j][0]+":"+head
  939. # find_flag = True
  940. # temp_head = inner_table[i][j-loop_j][0]
  941. # else:
  942. # if find_flag:
  943. # break
  944. #
  945. # if str(head+inner_table[i][j][0]) in text_set:
  946. # continue
  947. # if re.search(packPattern,head) is not None:
  948. # pack_text += head+inner_table[i][j][0]+","
  949. # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  950. # #排名替换为同一种表达
  951. # rank_text += head+inner_table[i][j][0]+","
  952. # #print(rank_text)
  953. # elif re.search(entityPattern,head) is not None:
  954. # entity_text += head+inner_table[i][j][0]+","
  955. # #print(entity_text)
  956. # else:
  957. # text_line += head+inner_table[i][j][0]+","
  958. # text_set.add(str(head+inner_table[i][j][0]))
  959. # text += pack_text+rank_text+entity_text+text_line
  960. # text = text[:-1]+"。" if len(text)>0 else text
  961. # else:
  962. # for j in range(width):
  963. #
  964. # rank_text = ""
  965. # entity_text = ""
  966. # text_line = ""
  967. # text_set = set()
  968. # for i in range(head_begin,head_end):
  969. # cell = inner_table[i][j]
  970. # #是属性值
  971. # if cell[1]==0 and cell[0]!="":
  972. # find_flag = False
  973. # head = ""
  974. # temp_head = ""
  975. #
  976. # for loop_j in range(1,j+1):
  977. # if not key_direct:
  978. # key_values = [1,2]
  979. # else:
  980. # key_values = [2]
  981. # if inner_table[i][j-loop_j][1] in key_values:
  982. # if find_flag:
  983. # if inner_table[i][j-loop_j][0]!=temp_head:
  984. # head = inner_table[i][j-loop_j][0]+":"+head
  985. # else:
  986. # head = inner_table[i][j-loop_j][0]+":"+head
  987. # find_flag = True
  988. # temp_head = inner_table[i][j-loop_j][0]
  989. # else:
  990. # if find_flag:
  991. # break
  992. # find_flag = False
  993. # temp_head = ""
  994. # for loop_i in range(0,i+1-head_begin):
  995. # if not key_direct:
  996. # key_values = [1,2]
  997. # else:
  998. # key_values = [1]
  999. # if inner_table[i-loop_i][j][1] in key_values:
  1000. # if find_flag:
  1001. # if inner_table[i-loop_i][j][0]!=temp_head:
  1002. # head = inner_table[i-loop_i][j][0]+":"+head
  1003. # else:
  1004. # head = inner_table[i-loop_i][j][0]+":"+head
  1005. # find_flag = True
  1006. # temp_head = inner_table[i-loop_i][j][0]
  1007. # else:
  1008. # if find_flag:
  1009. # break
  1010. # if str(head+inner_table[i][j][0]) in text_set:
  1011. # continue
  1012. # if re.search(rankPattern,head) is not None:
  1013. # rank_text += head+inner_table[i][j][0]+","
  1014. # #print(rank_text)
  1015. # elif re.search(entityPattern,head) is not None:
  1016. # entity_text += head+inner_table[i][j][0]+","
  1017. # #print(entity_text)
  1018. # else:
  1019. # text_line += head+inner_table[i][j][0]+","
  1020. # text_set.add(str(head+inner_table[i][j][0]))
  1021. # text += rank_text+entity_text+text_line
  1022. # text = text[:-1]+"。" if len(text)>0 else text
  1023. return text
  1024. def removeFix(inner_table,fix_value="~~"):
  1025. height = len(inner_table)
  1026. width = len(inner_table[0])
  1027. for h in range(height):
  1028. for w in range(width):
  1029. if inner_table[h][w][0]==fix_value:
  1030. inner_table[h][w][0] = ""
  1031. def trunTable(tbody,in_attachment):
  1032. # print(tbody.find('tbody'))
  1033. # 附件中的表格,排除异常错乱的表格
  1034. if in_attachment:
  1035. if tbody.name=='table':
  1036. _tbody = tbody.find('tbody')
  1037. if _tbody is None:
  1038. _tbody = tbody
  1039. else:
  1040. _tbody = tbody
  1041. _td_len_list = []
  1042. for _tr in _tbody.find_all(recursive=False):
  1043. len_td = len(_tr.find_all(recursive=False))
  1044. _td_len_list.append(len_td)
  1045. if _td_len_list:
  1046. if len(list(set(_td_len_list))) >= 8 or max(_td_len_list) > 100:
  1047. string_list = [re.sub("\s+","",i)for i in tbody.strings if i and i!='\n']
  1048. tbody.string = ",".join(string_list)
  1049. table_max_len = 30000
  1050. tbody.string = tbody.string[:table_max_len]
  1051. tbody.name = "turntable"
  1052. return None
  1053. # fixSpan(tbody)
  1054. # inner_table = getTable(tbody)
  1055. # inner_table = fixTable(inner_table)
  1056. table2list = TableTag2List()
  1057. inner_table = table2list.table2list(tbody, segment)
  1058. inner_table = fixTable(inner_table)
  1059. if inner_table == []:
  1060. string_list = [re.sub("\s+", "", i) for i in tbody.strings if i and i != '\n']
  1061. tbody.string = ",".join(string_list)
  1062. table_max_len = 30000
  1063. tbody.string = tbody.string[:table_max_len]
  1064. # log('异常表格直接取全文')
  1065. tbody.name = "turntable"
  1066. return None
  1067. if len(inner_table)>0 and len(inner_table[0])>0:
  1068. for tr in inner_table:
  1069. for td in tr:
  1070. if isinstance(td, str):
  1071. tbody.string = segment(tbody,final=False)
  1072. table_max_len = 30000
  1073. tbody.string = tbody.string[:table_max_len]
  1074. # log('异常表格,不做表格处理,直接取全文')
  1075. tbody.name = "turntable"
  1076. return None
  1077. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  1078. #inner_table,head_list = setHead_inline(inner_table)
  1079. # inner_table, head_list = setHead_initem(inner_table,pat_head)
  1080. inner_table, head_list = set_head_model(inner_table)
  1081. # inner_table,head_list = setHead_incontext(inner_table,pat_head)
  1082. # print("table_head", inner_table)
  1083. # print("head_list", head_list)
  1084. # for begin in range(len(head_list[:-1])):
  1085. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  1086. # print(item)
  1087. # print("====")
  1088. removeFix(inner_table)
  1089. # print("----")
  1090. # print(head_list)
  1091. # for item in inner_table:
  1092. # print(item)
  1093. tbody.string = getTableText(inner_table,head_list)
  1094. table_max_len = 30000
  1095. tbody.string = tbody.string[:table_max_len]
  1096. # print(tbody.string)
  1097. tbody.name = "turntable"
  1098. return inner_table
  1099. return None
  1100. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标|(需求|服务|项目|施工|采购|招租|出租|转让|出让|业主|询价|委托|权属|招标|竞得|抽取|承建)(人|方|单位)(名称)?|(供应商|供货商|服务商)(名称)?)$')
  1101. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  1102. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  1103. list_innerTable = []
  1104. # 2022/2/9 删除干扰标签
  1105. for tag in soup.find_all('option'): #例子: 216661412
  1106. if 'selected' not in tag.attrs:
  1107. tag.extract()
  1108. for ul in soup.find_all('ul'): #例子 156439663 多个不同channel 类别的标题
  1109. if ul.find_all('li') == ul.findChildren(recursive=False) and len(set(re.findall(
  1110. '招标公告|中标结果公示|中标候选人公示|招标答疑|开标评标|合同履?约?公示|资格评审',
  1111. ul.get_text(), re.S)))>3:
  1112. ul.extract()
  1113. # tbodies = soup.find_all('table')
  1114. # 遍历表格中的每个tbody
  1115. tbodies = []
  1116. in_attachment = False
  1117. for _part in soup.find_all():
  1118. if _part.name=='table':
  1119. tbodies.append((_part,in_attachment))
  1120. elif _part.name=='div':
  1121. if 'class' in _part.attrs and "richTextFetch" in _part['class']:
  1122. in_attachment = True
  1123. #逆序处理嵌套表格
  1124. for tbody_index in range(1,len(tbodies)+1):
  1125. tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
  1126. inner_table = trunTable(tbody,_in_attachment)
  1127. list_innerTable.append(inner_table)
  1128. # tbodies = soup.find_all('tbody')
  1129. # 遍历表格中的每个tbody
  1130. tbodies = []
  1131. in_attachment = False
  1132. for _part in soup.find_all():
  1133. if _part.name == 'tbody':
  1134. tbodies.append((_part, in_attachment))
  1135. elif _part.name == 'div':
  1136. if 'class' in _part.attrs and "richTextFetch" in _part['class']:
  1137. in_attachment = True
  1138. #逆序处理嵌套表格
  1139. for tbody_index in range(1,len(tbodies)+1):
  1140. tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
  1141. inner_table = trunTable(tbody,_in_attachment)
  1142. list_innerTable.append(inner_table)
  1143. return soup
  1144. # return list_innerTable
  1145. re_num = re.compile("[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十]")
  1146. num_dict = {
  1147. "一": 1, "二": 2,
  1148. "三": 3, "四": 4,
  1149. "五": 5, "六": 6,
  1150. "七": 7, "八": 8,
  1151. "九": 9, "十": 10}
  1152. # 一百以内的中文大写转换为数字
  1153. def change2num(text):
  1154. result_num = -1
  1155. # text = text[:6]
  1156. match = re_num.search(text)
  1157. if match:
  1158. _num = match.group()
  1159. if num_dict.get(_num):
  1160. return num_dict.get(_num)
  1161. else:
  1162. tenths = 1
  1163. the_unit = 0
  1164. num_split = _num.split("十")
  1165. if num_dict.get(num_split[0]):
  1166. tenths = num_dict.get(num_split[0])
  1167. if num_dict.get(num_split[1]):
  1168. the_unit = num_dict.get(num_split[1])
  1169. result_num = tenths * 10 + the_unit
  1170. elif re.search("\d{1,2}",text):
  1171. _num = re.search("\d{1,2}",text).group()
  1172. result_num = int(_num)
  1173. return result_num
  1174. #大纲分段处理
  1175. def get_preprocessed_outline(soup):
  1176. pattern_0 = re.compile("^(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[、.\.]")
  1177. pattern_1 = re.compile("^[\((]?(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[\))]")
  1178. pattern_2 = re.compile("^\d{1,2}[、.\.](?=[^\d]{1,2}|$)")
  1179. pattern_3 = re.compile("^[\((]?\d{1,2}[\))]")
  1180. pattern_list = [pattern_0, pattern_1, pattern_2, pattern_3]
  1181. body = soup.find("body")
  1182. if body == None:
  1183. return soup # 修复 无body的报错 例子:264419050
  1184. body_child = body.find_all(recursive=False)
  1185. deal_part = body
  1186. # print(body_child[0]['id'])
  1187. if 'id' in body_child[0].attrs:
  1188. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1189. deal_part = body_child[0]
  1190. if len(deal_part.find_all(recursive=False))>2:
  1191. deal_part = deal_part.parent
  1192. skip_tag = ['turntable', 'tbody', 'th', 'tr', 'td', 'table','thead','tfoot']
  1193. for part in deal_part.find_all(recursive=False):
  1194. # 查找解析文本的主干部分
  1195. is_main_text = False
  1196. through_text_num = 0
  1197. while (not is_main_text and part.find_all(recursive=False)):
  1198. while len(part.find_all(recursive=False)) == 1 and part.get_text(strip=True) == \
  1199. part.find_all(recursive=False)[0].get_text(strip=True):
  1200. part = part.find_all(recursive=False)[0]
  1201. max_len = len(part.get_text(strip=True))
  1202. is_main_text = True
  1203. for t_part in part.find_all(recursive=False):
  1204. if t_part.name not in skip_tag and t_part.get_text(strip=True)!="":
  1205. through_text_num += 1
  1206. if t_part.get_text(strip=True)!="" and len(t_part.get_text(strip=True))/max_len>=0.65:
  1207. if t_part.name not in skip_tag:
  1208. is_main_text = False
  1209. part = t_part
  1210. break
  1211. else:
  1212. while len(t_part.find_all(recursive=False)) == 1 and t_part.get_text(strip=True) == \
  1213. t_part.find_all(recursive=False)[0].get_text(strip=True):
  1214. t_part = t_part.find_all(recursive=False)[0]
  1215. if through_text_num>2:
  1216. is_table = True
  1217. for _t_part in t_part.find_all(recursive=False):
  1218. if _t_part.name not in skip_tag:
  1219. is_table = False
  1220. break
  1221. if not is_table:
  1222. is_main_text = False
  1223. part = t_part
  1224. break
  1225. else:
  1226. is_main_text = False
  1227. part = t_part
  1228. break
  1229. is_find = False
  1230. for _pattern in pattern_list:
  1231. last_index = 0
  1232. handle_list = []
  1233. for _part in part.find_all(recursive=False):
  1234. if _part.name not in skip_tag and _part.get_text(strip=True) != "":
  1235. # print('text:', _part.get_text(strip=True))
  1236. re_match = re.search(_pattern, _part.get_text(strip=True))
  1237. if re_match:
  1238. outline_index = change2num(re_match.group())
  1239. if last_index < outline_index:
  1240. # _part.insert_before("##split##")
  1241. handle_list.append(_part)
  1242. last_index = outline_index
  1243. if len(handle_list)>1:
  1244. is_find = True
  1245. for _part in handle_list:
  1246. _part.insert_before("##split##")
  1247. if is_find:
  1248. break
  1249. # print(soup)
  1250. return soup
  1251. #数据清洗
  1252. def segment(soup,final=True):
  1253. # print("==")
  1254. # print(soup)
  1255. # print("====")
  1256. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  1257. subspaceList = ["td",'a',"span","p"]
  1258. if soup.name in subspaceList:
  1259. #判断有值叶子节点数
  1260. _count = 0
  1261. for child in soup.find_all(recursive=True):
  1262. if child.get_text().strip()!="" and len(child.find_all())==0:
  1263. _count += 1
  1264. if _count<=1:
  1265. text = soup.get_text()
  1266. # 2020/11/24 大网站规则添加
  1267. if 'title' in soup.attrs:
  1268. if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']:
  1269. text = soup.attrs['title']
  1270. _list = []
  1271. for x in re.split("\s+",text):
  1272. if x.strip()!="":
  1273. _list.append(len(x))
  1274. if len(_list)>0:
  1275. _minLength = min(_list)
  1276. if _minLength>2:
  1277. _substr = ","
  1278. else:
  1279. _substr = ""
  1280. else:
  1281. _substr = ""
  1282. text = text.replace("\r\n",",").replace("\n",",")
  1283. text = re.sub("\s+",_substr,text)
  1284. # text = re.sub("\s+","##space##",text)
  1285. return text
  1286. segList = ["title"]
  1287. commaList = ["div","br","td","p","li"]
  1288. #commaList = []
  1289. spaceList = ["span"]
  1290. tbodies = soup.find_all('tbody')
  1291. if len(tbodies) == 0:
  1292. tbodies = soup.find_all('table')
  1293. # 递归遍历所有节点,插入符号
  1294. for child in soup.find_all(recursive=True):
  1295. # print(child.name,child.get_text())
  1296. if child.name in segList:
  1297. child.insert_after("。")
  1298. if child.name in commaList:
  1299. child.insert_after(",")
  1300. # if child.name == 'div' and 'class' in child.attrs:
  1301. # # 添加附件"attachment"标识
  1302. # if "richTextFetch" in child['class']:
  1303. # child.insert_before("##attachment##")
  1304. # print(child.parent)
  1305. # if child.name in subspaceList:
  1306. # child.insert_before("#subs"+str(child.name)+"#")
  1307. # child.insert_after("#sube"+str(child.name)+"#")
  1308. # if child.name in spaceList:
  1309. # child.insert_after(" ")
  1310. text = str(soup.get_text())
  1311. #替换英文冒号为中文冒号
  1312. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1313. #替换为中文逗号
  1314. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1315. #替换为中文分号
  1316. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1317. # 感叹号替换为中文句号
  1318. text = re.sub("(?<=[\u4e00-\u9fa5])[!!]|[!!](?=[\u4e00-\u9fa5])","。",text)
  1319. #替换格式未识别的问号为" " ,update:2021/7/20
  1320. text = re.sub("[?\?]{2,}|\n"," ",text)
  1321. #替换"""为"“",否则导入deepdive出错
  1322. # text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1323. text = text.replace('"',"“").replace("\r","").replace("\n","").replace("\\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
  1324. # print('==1',text)
  1325. # text = re.sub("\s{4,}",",",text)
  1326. # 解决公告中的" "空格替换问题
  1327. if re.search("\s{4,}",text):
  1328. _text = ""
  1329. for _sent in re.split("。+",text):
  1330. for _sent2 in re.split(',+',_sent):
  1331. for _sent3 in re.split(":+",_sent2):
  1332. for _t in re.split("\s{4,}",_sent3):
  1333. if len(_t)<3:
  1334. _text += _t
  1335. else:
  1336. _text += ","+_t
  1337. _text += ":"
  1338. _text = _text[:-1]
  1339. _text += ","
  1340. _text = _text[:-1]
  1341. _text += "。"
  1342. _text = _text[:-1]
  1343. text = _text
  1344. # print('==2',text)
  1345. #替换标点
  1346. #替换连续的标点
  1347. if final:
  1348. text = re.sub("##space##"," ",text)
  1349. punc_pattern = "(?P<del>[。,;::,\s]+)"
  1350. list_punc = re.findall(punc_pattern,text)
  1351. list_punc.sort(key=lambda x:len(x),reverse=True)
  1352. for punc_del in list_punc:
  1353. if len(punc_del)>1:
  1354. if len(punc_del.strip())>0:
  1355. if ":" in punc_del.strip():
  1356. if "。" in punc_del.strip():
  1357. text = re.sub(punc_del, ":。", text)
  1358. else:
  1359. text = re.sub(punc_del,":",text)
  1360. else:
  1361. text = re.sub(punc_del,punc_del.strip()[0],text) #2021/12/09 修正由于某些标签后插入符号把原来符号替换
  1362. else:
  1363. text = re.sub(punc_del,"",text)
  1364. #将连续的中文句号替换为一个
  1365. text_split = text.split("。")
  1366. text_split = [x for x in text_split if len(x)>0]
  1367. text = "。".join(text_split)
  1368. # #删除标签中的所有空格
  1369. # for subs in subspaceList:
  1370. # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  1371. # while(True):
  1372. # oneMatch = re.search(re.compile(patten),text)
  1373. # if oneMatch is not None:
  1374. # _match = oneMatch.group(1)
  1375. # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
  1376. # else:
  1377. # break
  1378. # text过大报错
  1379. LOOP_LEN = 10000
  1380. LOOP_BEGIN = 0
  1381. _text = ""
  1382. if len(text)<10000000:
  1383. while(LOOP_BEGIN<len(text)):
  1384. _text += re.sub(")",")",re.sub("(","(",re.sub("\s(?!\d{2}:\d{2})","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  1385. LOOP_BEGIN += LOOP_LEN
  1386. text = _text
  1387. # 附件标识前修改为句号,避免正文和附件内容混合在一起
  1388. text = re.sub("[^。](?=##attachment##)","。",text)
  1389. text = re.sub("[^。](?=##attachment_begin##)","。",text)
  1390. text = re.sub("[^。](?=##attachment_end##)","。",text)
  1391. text = re.sub("##attachment_begin##。","##attachment_begin##",text)
  1392. text = re.sub("##attachment_end##。","##attachment_end##",text)
  1393. return text
  1394. '''
  1395. #数据清洗
  1396. def segment(soup):
  1397. segList = ["title"]
  1398. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  1399. spaceList = ["span"]
  1400. tbodies = soup.find_all('tbody')
  1401. if len(tbodies) == 0:
  1402. tbodies = soup.find_all('table')
  1403. # 递归遍历所有节点,插入符号
  1404. for child in soup.find_all(recursive=True):
  1405. if child.name == 'br':
  1406. child.insert_before(',')
  1407. child_text = re.sub('\s', '', child.get_text())
  1408. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  1409. continue
  1410. if child.name in segList:
  1411. child.insert_after("。")
  1412. if child.name in commaList:
  1413. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  1414. child.insert_after(",")
  1415. elif len(child_text) >=50:
  1416. child.insert_after("。")
  1417. #if child.name in spaceList:
  1418. #child.insert_after(" ")
  1419. text = str(soup.get_text())
  1420. text = re.sub("\s{5,}",",",text)
  1421. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1422. #替换"""为"“",否则导入deepdive出错
  1423. text = text.replace('"',"“")
  1424. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  1425. #删除所有空格
  1426. text = re.sub("\s+","#nbsp#",text)
  1427. text_list = text.split('#nbsp#')
  1428. new_text = ''
  1429. for i in range(len(text_list)-1):
  1430. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  1431. new_text += text_list[i]
  1432. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  1433. new_text += text_list[i] + '。'
  1434. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  1435. new_text += text_list[i] + ';'
  1436. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  1437. new_text += text_list[i] + ' '
  1438. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  1439. new_text += text_list[i]
  1440. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  1441. new_text += text_list[i] + ','
  1442. else:
  1443. new_text += text_list[i]
  1444. new_text += text_list[-1]
  1445. text = new_text
  1446. #替换英文冒号为中文冒号
  1447. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1448. #替换为中文逗号
  1449. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1450. #替换为中文分号
  1451. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1452. #替换标点
  1453. while(True):
  1454. #替换连续的标点
  1455. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  1456. if punc is not None:
  1457. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  1458. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  1459. if punc is not None:
  1460. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  1461. else:
  1462. #替换标点之后的空格
  1463. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  1464. if punc is not None:
  1465. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  1466. else:
  1467. break
  1468. #将连续的中文句号替换为一个
  1469. text_split = text.split("。")
  1470. text_split = [x for x in text_split if len(x)>0]
  1471. text = "。".join(text_split)
  1472. #替换中文括号为英文括号
  1473. text = re.sub("(","(",text)
  1474. text = re.sub(")",")",text)
  1475. return text
  1476. '''
  1477. #连续实体合并(弃用)
  1478. def union_ner(list_ner):
  1479. result_list = []
  1480. union_index = []
  1481. union_index_set = set()
  1482. for i in range(len(list_ner)-1):
  1483. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  1484. if list_ner[i][1]-list_ner[i+1][0]==1:
  1485. union_index_set.add(i)
  1486. union_index_set.add(i+1)
  1487. union_index.append((i,i+1))
  1488. for i in range(len(list_ner)):
  1489. if i not in union_index_set:
  1490. result_list.append(list_ner[i])
  1491. for item in union_index:
  1492. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  1493. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  1494. return result_list
  1495. # def get_preprocessed(articles,useselffool=False):
  1496. # '''
  1497. # @summary:预处理步骤,NLP处理、实体识别
  1498. # @param:
  1499. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1500. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1501. # '''
  1502. # list_articles = []
  1503. # list_sentences = []
  1504. # list_entitys = []
  1505. # cost_time = dict()
  1506. # for article in articles:
  1507. # list_sentences_temp = []
  1508. # list_entitys_temp = []
  1509. # doc_id = article[0]
  1510. # sourceContent = article[1]
  1511. # _send_doc_id = article[3]
  1512. # _title = article[4]
  1513. # #表格处理
  1514. # key_preprocess = "tableToText"
  1515. # start_time = time.time()
  1516. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1517. #
  1518. # # log(article_processed)
  1519. #
  1520. # if key_preprocess not in cost_time:
  1521. # cost_time[key_preprocess] = 0
  1522. # cost_time[key_preprocess] += time.time()-start_time
  1523. #
  1524. # #article_processed = article[1]
  1525. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1526. # #nlp处理
  1527. # if article_processed is not None and len(article_processed)!=0:
  1528. # split_patten = "。"
  1529. # sentences = []
  1530. # _begin = 0
  1531. # for _iter in re.finditer(split_patten,article_processed):
  1532. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1533. # _begin = _iter.span()[1]
  1534. # sentences.append(article_processed[_begin:])
  1535. #
  1536. # lemmas = []
  1537. # doc_offsets = []
  1538. # dep_types = []
  1539. # dep_tokens = []
  1540. #
  1541. # time1 = time.time()
  1542. #
  1543. # '''
  1544. # tokens_all = fool.cut(sentences)
  1545. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1546. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1547. # ner_entitys_all = fool.ner(sentences)
  1548. # '''
  1549. # #限流执行
  1550. # key_nerToken = "nerToken"
  1551. # start_time = time.time()
  1552. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1553. # if key_nerToken not in cost_time:
  1554. # cost_time[key_nerToken] = 0
  1555. # cost_time[key_nerToken] += time.time()-start_time
  1556. #
  1557. #
  1558. # for sentence_index in range(len(sentences)):
  1559. #
  1560. #
  1561. #
  1562. # list_sentence_entitys = []
  1563. # sentence_text = sentences[sentence_index]
  1564. # tokens = tokens_all[sentence_index]
  1565. #
  1566. # list_tokenbegin = []
  1567. # begin = 0
  1568. # for i in range(0,len(tokens)):
  1569. # list_tokenbegin.append(begin)
  1570. # begin += len(str(tokens[i]))
  1571. # list_tokenbegin.append(begin+1)
  1572. # #pos_tag = pos_all[sentence_index]
  1573. # pos_tag = ""
  1574. #
  1575. # ner_entitys = ner_entitys_all[sentence_index]
  1576. #
  1577. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1578. #
  1579. # #识别package
  1580. #
  1581. #
  1582. # #识别实体
  1583. # for ner_entity in ner_entitys:
  1584. # begin_index_temp = ner_entity[0]
  1585. # end_index_temp = ner_entity[1]
  1586. # entity_type = ner_entity[2]
  1587. # entity_text = ner_entity[3]
  1588. #
  1589. # for j in range(len(list_tokenbegin)):
  1590. # if list_tokenbegin[j]==begin_index_temp:
  1591. # begin_index = j
  1592. # break
  1593. # elif list_tokenbegin[j]>begin_index_temp:
  1594. # begin_index = j-1
  1595. # break
  1596. # begin_index_temp += len(str(entity_text))
  1597. # for j in range(begin_index,len(list_tokenbegin)):
  1598. # if list_tokenbegin[j]>=begin_index_temp:
  1599. # end_index = j-1
  1600. # break
  1601. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1602. #
  1603. # #去掉标点符号
  1604. # entity_text = re.sub("[,,。:]","",entity_text)
  1605. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1606. #
  1607. #
  1608. # #使用正则识别金额
  1609. # entity_type = "money"
  1610. #
  1611. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1612. #
  1613. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1614. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1615. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1616. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1617. #
  1618. # set_begin = set()
  1619. # for pattern_key in list_money_pattern.keys():
  1620. # pattern = re.compile(list_money_pattern[pattern_key])
  1621. # all_match = re.findall(pattern, sentence_text)
  1622. # index = 0
  1623. # for i in range(len(all_match)):
  1624. # if len(all_match[i][0])>0:
  1625. # # print("===",all_match[i])
  1626. # #print(all_match[i][0])
  1627. # unit = ""
  1628. # entity_text = all_match[i][3]
  1629. # if pattern_key in ["key_word","front_m"]:
  1630. # unit = all_match[i][1]
  1631. # else:
  1632. # unit = all_match[i][4]
  1633. # if entity_text.find("元")>=0:
  1634. # unit = ""
  1635. #
  1636. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1637. #
  1638. # begin_index_temp = index
  1639. # for j in range(len(list_tokenbegin)):
  1640. # if list_tokenbegin[j]==index:
  1641. # begin_index = j
  1642. # break
  1643. # elif list_tokenbegin[j]>index:
  1644. # begin_index = j-1
  1645. # break
  1646. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1647. # end_index_temp = index
  1648. # #index += len(str(all_match[i][0]))
  1649. # for j in range(begin_index,len(list_tokenbegin)):
  1650. # if list_tokenbegin[j]>=index:
  1651. # end_index = j-1
  1652. # break
  1653. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1654. #
  1655. #
  1656. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1657. # if len(unit)>0:
  1658. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1659. # else:
  1660. # entity_text = str(getUnifyMoney(entity_text))
  1661. #
  1662. # _exists = False
  1663. # for item in list_sentence_entitys:
  1664. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1665. # _exists = True
  1666. # if not _exists:
  1667. # if float(entity_text)>10:
  1668. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1669. #
  1670. # else:
  1671. # index += 1
  1672. #
  1673. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1674. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1675. # list_sentences.append(list_sentences_temp)
  1676. # list_entitys.append(list_entitys_temp)
  1677. # return list_articles,list_sentences,list_entitys,cost_time
  1678. def get_preprocessed(articles, useselffool=False):
  1679. '''
  1680. @summary:预处理步骤,NLP处理、实体识别
  1681. @param:
  1682. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1683. @return:list of articles,list of each article of sentences,list of each article of entitys
  1684. '''
  1685. cost_time = dict()
  1686. list_articles = get_preprocessed_article(articles,cost_time)
  1687. list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time)
  1688. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1689. calibrateEnterprise(list_articles,list_sentences,list_entitys)
  1690. return list_articles,list_sentences,list_entitys,list_outlines,cost_time
  1691. def special_treatment(sourceContent, web_source_no):
  1692. try:
  1693. if web_source_no == 'DX000202-1':
  1694. ser = re.search('中标供应商及中标金额:【(([\w()]{5,20}-[\d,.]+,)+)】', sourceContent)
  1695. if ser:
  1696. new = ""
  1697. l = ser.group(1).split(',')
  1698. for i in range(len(l)):
  1699. it = l[i]
  1700. if '-' in it:
  1701. role, money = it.split('-')
  1702. new += '标段%d, 中标供应商: ' % (i + 1) + role + ',中标金额:' + money + '。'
  1703. sourceContent = sourceContent.replace(ser.group(0), new, 1)
  1704. elif web_source_no == '00753-14':
  1705. body = sourceContent.find("body")
  1706. body_child = body.find_all(recursive=False)
  1707. pcontent = body
  1708. if 'id' in body_child[0].attrs:
  1709. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1710. pcontent = body_child[0]
  1711. # pcontent = sourceContent.find("div", id="pcontent")
  1712. pcontent = pcontent.find_all(recursive=False)[0]
  1713. first_table = None
  1714. for idx in range(len(pcontent.find_all(recursive=False))):
  1715. t_part = pcontent.find_all(recursive=False)[idx]
  1716. if t_part.name != "table":
  1717. break
  1718. if idx == 0:
  1719. first_table = t_part
  1720. else:
  1721. for _tr in t_part.find("tbody").find_all(recursive=False):
  1722. first_table.find("tbody").append(_tr)
  1723. t_part.clear()
  1724. elif web_source_no == 'DX008357-11':
  1725. body = sourceContent.find("body")
  1726. body_child = body.find_all(recursive=False)
  1727. pcontent = body
  1728. if 'id' in body_child[0].attrs:
  1729. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1730. pcontent = body_child[0]
  1731. # pcontent = sourceContent.find("div", id="pcontent")
  1732. pcontent = pcontent.find_all(recursive=False)[0]
  1733. error_table = []
  1734. is_error_table = False
  1735. for part in pcontent.find_all(recursive=False):
  1736. if is_error_table:
  1737. if part.name == "table":
  1738. error_table.append(part)
  1739. else:
  1740. break
  1741. if part.name == "div" and part.get_text(strip=True) == "中标候选单位:":
  1742. is_error_table = True
  1743. first_table = None
  1744. for idx in range(len(error_table)):
  1745. t_part = error_table[idx]
  1746. # if t_part.name != "table":
  1747. # break
  1748. if idx == 0:
  1749. for _tr in t_part.find("tbody").find_all(recursive=False):
  1750. if _tr.get_text(strip=True) == "":
  1751. _tr.decompose()
  1752. first_table = t_part
  1753. else:
  1754. for _tr in t_part.find("tbody").find_all(recursive=False):
  1755. if _tr.get_text(strip=True) != "":
  1756. first_table.find("tbody").append(_tr)
  1757. t_part.clear()
  1758. elif web_source_no == '18021-2':
  1759. body = sourceContent.find("body")
  1760. body_child = body.find_all(recursive=False)
  1761. pcontent = body
  1762. if 'id' in body_child[0].attrs:
  1763. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1764. pcontent = body_child[0]
  1765. # pcontent = sourceContent.find("div", id="pcontent")
  1766. td = pcontent.find_all("td")
  1767. for _td in td:
  1768. if str(_td.string).strip() == "报价金额":
  1769. _td.string = "单价"
  1770. elif web_source_no == '13740-2':
  1771. # “xxx成为成交供应商”
  1772. re_match = re.search("[^,。]+成为[^,。]*成交供应商", sourceContent)
  1773. if re_match:
  1774. sourceContent = sourceContent.replace(re_match.group(), "成交人:" + re_match.group())
  1775. elif web_source_no == '03786-10':
  1776. ser1 = re.search('中标价:([\d,.]+)', sourceContent)
  1777. ser2 = re.search('合同金额[((]万元[))]:([\d,.]+)', sourceContent)
  1778. if ser1 and ser2:
  1779. m1 = ser1.group(1).replace(',', '')
  1780. m2 = ser2.group(1).replace(',', '')
  1781. if float(m1) < 100000 and (m1.split('.')[0] == m2.split('.')[0] or m2 == '0'):
  1782. new = '中标价(万元):' + m1
  1783. sourceContent = sourceContent.replace(ser1.group(0), new, 1)
  1784. elif web_source_no=='00076-4':
  1785. ser = re.search('主要标的数量:([0-9一]+)\w{,3},主要标的单价:([\d,.]+)元?,合同金额:(.00),', sourceContent)
  1786. if ser:
  1787. num = ser.group(1).replace('一', '1')
  1788. try:
  1789. num = 1 if num == '0' else num
  1790. unit_price = ser.group(2).replace(',', '')
  1791. total_price = str(int(num) * float(unit_price))
  1792. new = '合同金额:' + total_price
  1793. sourceContent = sourceContent.replace('合同金额:.00', new, 1)
  1794. except Exception as e:
  1795. log('preprocessing.py special_treatment exception')
  1796. elif web_source_no=='DX000105-2':
  1797. if re.search("成交公示", sourceContent) and re.search(',投标人:', sourceContent) and re.search(',成交人:', sourceContent)==None:
  1798. sourceContent = sourceContent.replace(',投标人:', ',成交人:')
  1799. elif web_source_no in ['03795-1', '03795-2']:
  1800. if re.search('中标单位如下', sourceContent) and re.search(',投标人:', sourceContent) and re.search(',中标人:', sourceContent)==None:
  1801. sourceContent = sourceContent.replace(',投标人:', ',中标人:')
  1802. elif web_source_no in ['04080-3', '04080-4']:
  1803. ser = re.search('合同金额:([0-9,]+.[0-9]{3,})(.{,4})', sourceContent)
  1804. if ser and '万' not in ser.group(2):
  1805. sourceContent = sourceContent.replace('合同金额:', '合同金额(万元):')
  1806. elif web_source_no=='03761-3':
  1807. ser = re.search('中标价,([0-9]+)[.0-9]*%', sourceContent)
  1808. if ser and int(ser.group(1))>100:
  1809. sourceContent = sourceContent.replace(ser.group(0), ser.group(0)[:-1]+'元')
  1810. elif web_source_no=='00695-7':
  1811. ser = re.search('支付金额:', sourceContent)
  1812. if ser:
  1813. sourceContent = sourceContent.replace('支付金额:', '合同金额:')
  1814. elif web_source_no=='00811-8':
  1815. if re.search('是否中标:是', sourceContent) and re.search('排名:\d,', sourceContent):
  1816. sourceContent = re.sub('排名:\d,', '候选', sourceContent)
  1817. elif web_source_no=='DX000726-6':
  1818. sourceContent = re.sub('卖方[::\s]+宝山钢铁股份有限公司', '招标单位:宝山钢铁股份有限公司', sourceContent)
  1819. return sourceContent
  1820. except Exception as e:
  1821. log('特殊数据源: %s 预处理特别修改抛出异常: %s'%(web_source_no, e))
  1822. return sourceContent
  1823. def article_limit(soup,limit_words=30000):
  1824. sub_space = re.compile("\s+")
  1825. def soup_limit(_soup,_count,max_count=30000,max_gap=500):
  1826. """
  1827. :param _soup: soup
  1828. :param _count: 当前字数
  1829. :param max_count: 字数最大限制
  1830. :param max_gap: 超过限制后的最大误差
  1831. :return:
  1832. """
  1833. _gap = _count - max_count
  1834. _is_skip = False
  1835. next_soup = None
  1836. while len(_soup.find_all(recursive=False)) == 1 and \
  1837. _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
  1838. _soup = _soup.find_all(recursive=False)[0]
  1839. if len(_soup.find_all(recursive=False)) == 0:
  1840. _soup.string = str(_soup.get_text())[:max_count-_count]
  1841. _count += len(re.sub(sub_space, "", _soup.string))
  1842. _gap = _count - max_count
  1843. next_soup = None
  1844. else:
  1845. for _soup_part in _soup.find_all(recursive=False):
  1846. if not _is_skip:
  1847. _count += len(re.sub(sub_space, "", _soup_part.get_text()))
  1848. if _count >= max_count:
  1849. _gap = _count - max_count
  1850. if _gap <= max_gap:
  1851. _is_skip = True
  1852. else:
  1853. _is_skip = True
  1854. next_soup = _soup_part
  1855. _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
  1856. continue
  1857. else:
  1858. _soup_part.decompose()
  1859. return _count,_gap,next_soup
  1860. text_count = 0
  1861. have_attachment = False
  1862. attachment_part = None
  1863. for child in soup.find_all(recursive=True):
  1864. if child.name == 'div' and 'class' in child.attrs:
  1865. if "richTextFetch" in child['class']:
  1866. child.insert_before("##attachment##。") # 句号分开,避免项目名称等提取
  1867. attachment_part = child
  1868. have_attachment = True
  1869. break
  1870. if not have_attachment:
  1871. # 无附件
  1872. if len(re.sub(sub_space, "", soup.get_text())) > limit_words:
  1873. text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500)
  1874. while n_soup:
  1875. text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
  1876. else:
  1877. # 有附件
  1878. _text = re.sub(sub_space, "", soup.get_text())
  1879. _text_split = _text.split("##attachment##")
  1880. if len(_text_split[0])>limit_words:
  1881. main_soup = attachment_part.parent
  1882. main_text = main_soup.find_all(recursive=False)[0]
  1883. text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500)
  1884. while n_soup:
  1885. text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
  1886. if len(_text_split[1])>limit_words:
  1887. # attachment_html纯文本,无子结构
  1888. if len(attachment_part.find_all(recursive=False))==0:
  1889. attachment_part.string = str(attachment_part.get_text())[:limit_words]
  1890. else:
  1891. attachment_text_nums = 0
  1892. attachment_skip = False
  1893. for part in attachment_part.find_all(recursive=False):
  1894. if not attachment_skip:
  1895. last_attachment_text_nums = attachment_text_nums
  1896. attachment_text_nums = attachment_text_nums + len(re.sub(sub_space, "", part.get_text()))
  1897. if attachment_text_nums>=limit_words:
  1898. part.string = str(part.get_text())[:limit_words-last_attachment_text_nums]
  1899. attachment_skip = True
  1900. else:
  1901. part.decompose()
  1902. return soup
  1903. def attachment_filelink(soup):
  1904. have_attachment = False
  1905. attachment_part = None
  1906. for child in soup.find_all(recursive=True):
  1907. if child.name == 'div' and 'class' in child.attrs:
  1908. if "richTextFetch" in child['class']:
  1909. attachment_part = child
  1910. have_attachment = True
  1911. break
  1912. if not have_attachment:
  1913. return soup
  1914. else:
  1915. # 附件类型:图片、表格
  1916. attachment_type = re.compile("\.(?:png|jpg|jpeg|tif|bmp|xlsx|xls)$")
  1917. attachment_dict = dict()
  1918. for _attachment in attachment_part.find_all(recursive=False):
  1919. if _attachment.name == 'div' and 'filemd5' in _attachment.attrs:
  1920. # print('filemd5',_attachment['filemd5'])
  1921. attachment_dict[_attachment['filemd5']] = _attachment
  1922. # print(attachment_dict)
  1923. for child in soup.find_all(recursive=True):
  1924. if child.name == 'div' and 'class' in child.attrs:
  1925. if "richTextFetch" in child['class']:
  1926. break
  1927. if "filelink" in child.attrs and child['filelink'] in attachment_dict:
  1928. if re.search(attachment_type,str(child.string).strip()) or \
  1929. ('original' in child.attrs and re.search(attachment_type,str(child['original']).strip())) or \
  1930. ('href' in child.attrs and re.search(attachment_type,str(child['href']).strip())):
  1931. # 附件插入正文标识
  1932. child.insert_before("。##attachment_begin##")
  1933. child.insert_after("。##attachment_end##")
  1934. child.replace_with(attachment_dict[child['filelink']])
  1935. # print('格式化输出',soup.prettify())
  1936. return soup
  1937. def del_achievement(text):
  1938. if re.search('中标|成交|入围|结果|评标|开标|候选人', text[:500]) == None or re.search('业绩', text) == None:
  1939. return text
  1940. p0 = '[,。;]((\d{1,2})|\d{1,2}、)[\w、]{,8}:|((\d{1,2})|\d{1,2}、)|。' # 例子 264392818
  1941. p1 = '业绩[:,](\d、[-\w()、]{6,30}(工程|项目|勘察|设计|施工|监理|总承包|采购|更新)[\w()]{,10}[,;])+' # 例子 257717618
  1942. p2 = '(类似业绩情况:|业绩:)(\w{,20}:)?(((\d)|\d、)项目名称:[-\w(),;、\d\s:]{5,100}[;。])+' # 例子 264345826
  1943. p3 = '(投标|类似|(类似)?项目|合格|有效|企业|工程)?业绩(名称|信息|\d)?:(项目名称:)?[-\w()、]{6,50}(项目|工程|勘察|设计|施工|监理|总承包|采购|更新)'
  1944. l = []
  1945. tmp = []
  1946. for it in re.finditer(p0, text):
  1947. if it.group(0)[-3:] in ['业绩:', '荣誉:']:
  1948. if tmp != []:
  1949. del_text = text[tmp[0]:it.start()]
  1950. l.append(del_text)
  1951. tmp = []
  1952. tmp.append(it.start())
  1953. elif tmp != []:
  1954. del_text = text[tmp[0]:it.start()]
  1955. l.append(del_text)
  1956. tmp = []
  1957. if tmp != []:
  1958. del_text = text[tmp[0]:]
  1959. l.append(del_text)
  1960. for del_text in l:
  1961. text = text.replace(del_text, '')
  1962. # print('删除业绩信息:', del_text)
  1963. for rs in re.finditer(p1, text):
  1964. # print('删除业绩信息:', rs.group(0))
  1965. text = text.replace(rs.group(0), '')
  1966. for rs in re.finditer(p2, text):
  1967. # print('删除业绩信息:', rs.group(0))
  1968. text = text.replace(rs.group(0), '')
  1969. for rs in re.finditer(p3, text):
  1970. # print('删除业绩信息:', rs.group(0))
  1971. text = text.replace(rs.group(0), '')
  1972. return text
  1973. def del_tabel_achievement(soup):
  1974. if re.search('中标|成交|入围|结果|评标|开标|候选人', soup.text[:800]) == None or re.search('业绩', soup.text)==None:
  1975. return None
  1976. p1 = '(中标|成交)(单位|候选人)的?(企业|项目|项目负责人|\w{,5})?业绩|类似(项目)?业绩|\w{,10}业绩$|业绩(公示|情况|荣誉)'
  1977. '''删除前面标签 命中业绩规则;当前标签为表格且公布业绩相关信息的去除'''
  1978. for tag in soup.find_all('table'):
  1979. pre_text = tag.findPreviousSibling().text.strip() if tag.findPreviousSibling() != None else ""
  1980. tr_text = tag.find('tr').text.strip() if tag.find('tr') != None else ""
  1981. # print(re.search(p1, pre_text),pre_text, len(pre_text), re.findall('序号|中标候选人名称|项目名称|工程名称|合同金额|建设单位|业主', tr_text))
  1982. if re.search(p1, pre_text) and len(pre_text) < 20 and tag.find('tr') != None and len(tr_text)<100:
  1983. _count = 0
  1984. for td in tag.find('tr').find_all('td'):
  1985. td_text = td.text.strip()
  1986. if len(td_text) > 25:
  1987. break
  1988. if len(td_text) < 25 and re.search('中标候选人|(项目|业绩|工程)名称|\w{,10}业绩$|合同金额|建设单位|采购单位|业主|甲方', td_text):
  1989. _count += 1
  1990. if _count >=2:
  1991. pre_tag = tag.findPreviousSibling().extract()
  1992. del_tag = tag.extract()
  1993. # print('删除表格业绩内容', pre_tag.text + del_tag.text)
  1994. break
  1995. elif re.search('业绩名称', tr_text) and re.search('建设单位|采购单位|业主', tr_text) and len(tr_text)<100:
  1996. del_tag = tag.extract()
  1997. # print('删除表格业绩内容', del_tag.text)
  1998. del_trs = []
  1999. '''删除表格某些行公布的业绩信息'''
  2000. for tag in soup.find_all('table'):
  2001. text = tag.text
  2002. if re.search('业绩', text) == None:
  2003. continue
  2004. # for tr in tag.find_all('tr'):
  2005. trs = tag.find_all('tr')
  2006. i = 0
  2007. while i < len(trs):
  2008. tr = trs[i]
  2009. if len(tr.find_all('td'))==2 and tr.td!=None and tr.td.findNextSibling()!=None:
  2010. td1_text =tr.td.text
  2011. td2_text =tr.td.findNextSibling().text
  2012. if re.search('业绩', td1_text)!=None and len(td1_text)<10 and len(re.findall('(\d、|(\d))?[-\w()、]+(工程|项目|勘察|设计|施工|监理|总承包|采购|更新)', td2_text))>=2:
  2013. # del_tag = tr.extract()
  2014. # print('删除表格业绩内容', del_tag.text)
  2015. del_trs.append(tr)
  2016. elif tr.td != None and re.search('^业绩|业绩$', tr.td.text.strip()) and len(tr.td.text.strip())<25:
  2017. rows = tr.td.attrs.get('rowspan', '')
  2018. cols = tr.td.attrs.get('colspan', '')
  2019. if rows.isdigit() and int(rows)>2:
  2020. for j in range(int(rows)):
  2021. if i+j < len(trs):
  2022. del_trs.append(trs[i+j])
  2023. i += j
  2024. elif cols.isdigit() and int(cols)>3 and len(tr.find_all('td'))==1 and i+2 < len(trs):
  2025. next_tr_cols = 0
  2026. td_num = 0
  2027. for td in trs[i+1].find_all('td'):
  2028. td_num += 1
  2029. if td.attrs.get('colspan', '').isdigit():
  2030. next_tr_cols += int(td.attrs.get('colspan', ''))
  2031. if next_tr_cols == int(cols):
  2032. del_trs.append(tr)
  2033. for j in range(1,len(trs)-i):
  2034. if len(trs[i+j].find_all('td')) == 1:
  2035. break
  2036. elif len(trs[i+j].find_all('td')) >= td_num-1:
  2037. del_trs.append(trs[i+j])
  2038. else:
  2039. break
  2040. i += j
  2041. i += 1
  2042. for tr in del_trs:
  2043. del_tag = tr.extract()
  2044. # print('删除表格业绩内容', del_tag.text)
  2045. def split_header(soup):
  2046. '''
  2047. 处理 空格分割多个表头的情况 : 主要标的名称 规格型号(或服务要求) 主要标的数量 主要标的单价 合同金额(万元)
  2048. :param soup: bs4 soup 对象
  2049. :return:
  2050. '''
  2051. header = []
  2052. attrs = []
  2053. flag = 0
  2054. tag = None
  2055. for p in soup.find_all('p'):
  2056. text = p.get_text()
  2057. if re.search('主要标的数量\s+主要标的单价((万?元))?\s+合同金额', text):
  2058. header = re.split('\s{3,}', text) if re.search('\s{3,}', text) else re.split('\s+', text)
  2059. flag = 1
  2060. tag = p
  2061. tag.string = ''
  2062. continue
  2063. if flag:
  2064. attrs = re.split('\s{3,}', text) if re.search('\s{3,}', text) else re.split('\s+', text)
  2065. if header and len(header) == len(attrs) and tag:
  2066. s = ""
  2067. for head, attr in zip(header, attrs):
  2068. s += head + ':' + attr + ','
  2069. # tag.string = s
  2070. # p.extract()
  2071. p.string = s
  2072. else:
  2073. break
  2074. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  2075. '''
  2076. :param articles: 待处理的article source html
  2077. :param useselffool: 是否使用selffool
  2078. :return: list_articles
  2079. '''
  2080. list_articles = []
  2081. for article in articles:
  2082. doc_id = article[0]
  2083. sourceContent = article[1]
  2084. sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
  2085. sourceContent = re.sub("##attachment##","",sourceContent)
  2086. sourceContent = sourceContent.replace('<br/>', '<br>')
  2087. sourceContent = re.sub("<br>(\s{0,}<br>)+","<br>",sourceContent)
  2088. # for br_match in re.findall("[^>]+?<br>",sourceContent):
  2089. # _new = re.sub("<br>","",br_match)
  2090. # # <br>标签替换为<p>标签
  2091. # if not re.search("^\s+$",_new):
  2092. # _new = '<p>'+_new + '</p>'
  2093. # # print(br_match,_new)
  2094. # sourceContent = sourceContent.replace(br_match,_new,1)
  2095. _send_doc_id = article[3]
  2096. _title = article[4]
  2097. page_time = article[5]
  2098. web_source_no = article[6]
  2099. '''特别数据源对 html 做特别修改'''
  2100. if web_source_no in ['DX000202-1']:
  2101. sourceContent = special_treatment(sourceContent, web_source_no)
  2102. #表格处理
  2103. key_preprocess = "tableToText"
  2104. start_time = time.time()
  2105. # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml"))
  2106. article_processed = BeautifulSoup(sourceContent,"lxml")
  2107. if re.search('主要标的数量(&nbsp;|\s)+主要标的单价((万?元))?(&nbsp;|\s)+合同金额', sourceContent): #处理 空格分割多个表头的情况
  2108. split_header(article_processed)
  2109. '''表格业绩内容删除'''
  2110. del_tabel_achievement(article_processed)
  2111. '''特别数据源对 BeautifulSoup(html) 做特别修改'''
  2112. if web_source_no in ["00753-14","DX008357-11","18021-2"]:
  2113. article_processed = special_treatment(article_processed, web_source_no)
  2114. for _soup in article_processed.descendants:
  2115. # 识别无标签文本,添加<span>标签
  2116. if not _soup.name and not _soup.parent.string and _soup.string.strip()!="":
  2117. # print(_soup.parent.string,_soup.string.strip())
  2118. _soup.wrap(article_processed.new_tag("span"))
  2119. # print(article_processed)
  2120. # 正文和附件内容限制字数30000
  2121. article_processed = article_limit(article_processed, limit_words=30000)
  2122. # 把每个附件识别对应的html放回原来出现的位置
  2123. article_processed = attachment_filelink(article_processed)
  2124. article_processed = get_preprocessed_outline(article_processed)
  2125. # print('article_processed')
  2126. article_processed = tableToText(article_processed)
  2127. article_processed = segment(article_processed)
  2128. article_processed = article_processed.replace('(', '(').replace(')', ')') #2022/8/10 统一为中文括号
  2129. # article_processed = article_processed.replace(':', ':') #2023/1/5 统一为中文冒号
  2130. article_processed = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])", ":", article_processed)
  2131. article_processed = article_processed.replace('.','.').replace('-', '-') # 2021/12/01 修正OCR识别PDF小数点错误问题
  2132. article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
  2133. article_processed = article_processed.replace('成交工程价款', '成交工程价') # 2021/12/21 修正为中标价
  2134. article_processed = re.sub('任务(?=编号[::])', '项目',article_processed) # 2022/08/10 修正为项目编号
  2135. article_processed = article_processed.replace('招标(建设)单位', '招标单位') #2022/8/10 修正预测不到表达
  2136. article_processed = re.sub("采购商(?=[^\u4e00-\u9fa5]|名称)", "招标人", article_processed)
  2137. article_processed = re.sub('(招标|采购)人(概况|信息):?[,。]', '采购人信息:', article_processed) # 2022/8/10统一表达
  2138. article_processed = article_processed.replace('\(%)', '') # 中标(成交)金额(元)\(%):498888.00, 处理 江西省政府采购网 金额特殊问题
  2139. article_processed = re.sub('金额:?((可填写下浮率?、折扣率?或费率|拟签含税总单价总计|[^万元()\d]{8,20})):?', '金额:', article_processed) # 中标(成交)金额:(可填写下浮率、折扣率或费率):29.3万元 金额特殊问题
  2140. article_processed = re.sub('(不?含(可抵扣增值|\w{,8})税)', '', article_processed) # 120637247 投标报价(元),(含可抵扣增值税):277,560.00。
  2141. article_processed = re.sub('供应商的?(名称[及其、]{1,2}地址|联系方式:名称)', '供应商名称', article_processed) # 18889217, 84422177
  2142. article_processed = re.sub(',最高有效报价者:', ',中标人名称:', article_processed) # 224678159 # 2023/7/4 四川站源特殊中标修改
  2143. article_processed = re.sub(',最高有效报价:', ',投标报价:', article_processed) # 224678159 # 2023/7/4 四川站源特殊中标修改
  2144. article_processed = re.sub('备选中标人', '第二候选人', article_processed) # 341344142 # 2023/7/17 特殊表达修改
  2145. ser = re.search('(采购|招标|比选)人(名称)?/(采购|招标|比选)?代理机构(名称)?:(?P<tenderee>[\w()]{4,25}(/[\w()]{4,25})?)/(?P<agency>[\w()]{4,25})[,。]', article_processed)
  2146. if ser:
  2147. article_processed = article_processed.replace(ser.group(0), '采购人名称:%s,采购代理机构名称:%s,' % (ser.group('tenderee'), ser.group('agency')))
  2148. ser2 = re.search('(采购|招标)人(名称)?/(采购|招标)?代理机构(名称)?:(?P<tenderee>[\w()]{4,25})[,。/]', article_processed)
  2149. if ser2:
  2150. article_processed = article_processed.replace(ser2.group(0), '采购人名称:%s,采购代理机构名称:,' % (
  2151. ser2.group('tenderee')))
  2152. if re.search('中标单位名称:[\w()]{5,25},中标候选人名次:\d,', article_processed) and re.search('中标候选人名次:\d,中标单位名称:[\w()]{5,25},', article_processed)==None: # 处理类似 304706608 此篇的数据源正文特殊表达
  2153. for it in re.finditer('(?P<tenderer>(中标单位名称:[\w()]{5,25},))(?P<rank>(中标候选人名次:\d,))', article_processed):
  2154. article_processed = article_processed.replace(it.group(0), it.group('rank')+it.group('tenderer'))
  2155. '''去除业绩内容'''
  2156. article_processed = del_achievement(article_processed)
  2157. # 修复OCR金额中“,”、“。”识别错误
  2158. article_processed_list = article_processed.split("##attachment##")
  2159. if len(article_processed_list)>1:
  2160. attachment_text = article_processed_list[1]
  2161. for _match in re.finditer("\d。\d{2}",attachment_text):
  2162. _match_text = _match.group()
  2163. attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1)
  2164. # for _match in re.finditer("(\d,\d{3})[,,.]",attachment_text):
  2165. for _match in re.finditer("\d,(?=\d{3}[^\d])",attachment_text):
  2166. _match_text = _match.group()
  2167. attachment_text = attachment_text.replace(_match_text,_match_text.replace(",",","),1)
  2168. article_processed_list[1] = attachment_text
  2169. article_processed = "##attachment##".join(article_processed_list)
  2170. '''特别数据源对 预处理后文本 做特别修改'''
  2171. if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2", '00811-8', '03795-1', '03795-2', 'DX000726-6']:
  2172. article_processed = special_treatment(article_processed, web_source_no)
  2173. # 提取bidway
  2174. list_bidway = extract_bidway(article_processed, _title)
  2175. if list_bidway:
  2176. bidway = list_bidway[0].get("body")
  2177. # bidway名称统一规范
  2178. bidway = bidway_integrate(bidway)
  2179. else:
  2180. bidway = ""
  2181. # 修正被","逗号分隔的时间
  2182. repair_time = re.compile("[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d|"
  2183. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[:时点],?[0-6]\d分?|"
  2184. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[时点]|"
  2185. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]|"
  2186. "[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d"
  2187. )
  2188. for _time in set(re.findall(repair_time,article_processed)):
  2189. if re.search(",",_time):
  2190. _time2 = re.sub(",", "", _time)
  2191. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time2)
  2192. if item:
  2193. _time2 = _time2.replace(item.group(),item.group() + " ")
  2194. article_processed = article_processed.replace(_time, _time2)
  2195. else:
  2196. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time)
  2197. if item:
  2198. _time2 = _time.replace(item.group(),item.group() + " ")
  2199. article_processed = article_processed.replace(_time, _time2)
  2200. # print('re_rtime',re.findall(repair_time,article_processed))
  2201. # log(article_processed)
  2202. if key_preprocess not in cost_time:
  2203. cost_time[key_preprocess] = 0
  2204. cost_time[key_preprocess] += round(time.time()-start_time,2)
  2205. #article_processed = article[1]
  2206. _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
  2207. bidway=bidway)
  2208. _article.fingerprint = getFingerprint(_title+sourceContent)
  2209. _article.page_time = page_time
  2210. list_articles.append(_article)
  2211. return list_articles
  2212. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  2213. '''
  2214. :param list_articles: 经过预处理的article text
  2215. :return: list_sentences
  2216. '''
  2217. list_sentences = []
  2218. list_outlines = []
  2219. for article in list_articles:
  2220. list_sentences_temp = []
  2221. list_entitys_temp = []
  2222. doc_id = article.id
  2223. _send_doc_id = article.doc_id
  2224. _title = article.title
  2225. #表格处理
  2226. key_preprocess = "tableToText"
  2227. start_time = time.time()
  2228. article_processed = article.content
  2229. if len(_title)<100 and _title not in article_processed: # 把标题放到正文
  2230. article_processed = _title + ',' + article_processed # 2023/01/06 标题正文加逗号分割,预防标题后面是产品,正文开头是公司实体,实体识别把产品和公司作为整个角色实体
  2231. attachment_begin_index = -1
  2232. if key_preprocess not in cost_time:
  2233. cost_time[key_preprocess] = 0
  2234. cost_time[key_preprocess] += time.time()-start_time
  2235. #nlp处理
  2236. if article_processed is not None and len(article_processed)!=0:
  2237. split_patten = "。"
  2238. sentences = []
  2239. _begin = 0
  2240. sentences_set = set()
  2241. for _iter in re.finditer(split_patten,article_processed):
  2242. _sen = article_processed[_begin:_iter.span()[1]]
  2243. if len(_sen)>0 and _sen not in sentences_set:
  2244. # 标识在附件里的句子
  2245. if re.search("##attachment##",_sen):
  2246. attachment_begin_index = len(sentences)
  2247. # _sen = re.sub("##attachment##","",_sen)
  2248. sentences.append(_sen)
  2249. sentences_set.add(_sen)
  2250. _begin = _iter.span()[1]
  2251. _sen = article_processed[_begin:]
  2252. if re.search("##attachment##", _sen):
  2253. # _sen = re.sub("##attachment##", "", _sen)
  2254. attachment_begin_index = len(sentences)
  2255. if len(_sen)>0 and _sen not in sentences_set:
  2256. sentences.append(_sen)
  2257. sentences_set.add(_sen)
  2258. # 解析outline大纲分段
  2259. outline_list = []
  2260. if re.search("##split##",article.content):
  2261. temp_sentences = []
  2262. last_sentence_index = (-1,-1)
  2263. outline_index = 0
  2264. for sentence_index in range(len(sentences)):
  2265. sentence_text = sentences[sentence_index]
  2266. for _ in re.findall("##split##", sentence_text):
  2267. _match = re.search("##split##", sentence_text)
  2268. if last_sentence_index[0] > -1:
  2269. sentence_begin_index,wordOffset_begin = last_sentence_index
  2270. sentence_end_index = sentence_index
  2271. wordOffset_end = _match.start()
  2272. if sentence_begin_index<attachment_begin_index and sentence_end_index>=attachment_begin_index:
  2273. outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,attachment_begin_index-1,wordOffset_begin,len(sentences[attachment_begin_index-1])))
  2274. else:
  2275. outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,sentence_end_index,wordOffset_begin,wordOffset_end))
  2276. outline_index += 1
  2277. sentence_text = re.sub("##split##,?", "", sentence_text,count=1)
  2278. last_sentence_index = (sentence_index,_match.start())
  2279. temp_sentences.append(sentence_text)
  2280. if attachment_begin_index>-1 and last_sentence_index[0]<attachment_begin_index:
  2281. outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],attachment_begin_index-1,last_sentence_index[1],len(temp_sentences[attachment_begin_index-1])))
  2282. else:
  2283. outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],len(sentences)-1,last_sentence_index[1],len(temp_sentences[-1])))
  2284. sentences = temp_sentences
  2285. #解析outline的outline_text内容
  2286. for _outline in outline_list:
  2287. if _outline.sentence_begin_index==_outline.sentence_end_index:
  2288. _text = sentences[_outline.sentence_begin_index][_outline.wordOffset_begin:_outline.wordOffset_end]
  2289. else:
  2290. _text = ""
  2291. for idx in range(_outline.sentence_begin_index,_outline.sentence_end_index+1):
  2292. if idx==_outline.sentence_begin_index:
  2293. _text += sentences[idx][_outline.wordOffset_begin:]
  2294. elif idx==_outline.sentence_end_index:
  2295. _text += sentences[idx][:_outline.wordOffset_end]
  2296. else:
  2297. _text += sentences[idx]
  2298. _outline.outline_text = _text
  2299. _outline_summary = re.split("[::,]",_text,1)[0]
  2300. if len(_outline_summary)<30:
  2301. _outline.outline_summary = _outline_summary
  2302. # print(_outline.outline_index,_outline.outline_text)
  2303. article.content = "".join(sentences)
  2304. # sentences.append(article_processed[_begin:])
  2305. lemmas = []
  2306. doc_offsets = []
  2307. dep_types = []
  2308. dep_tokens = []
  2309. time1 = time.time()
  2310. '''
  2311. tokens_all = fool.cut(sentences)
  2312. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  2313. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  2314. ner_entitys_all = fool.ner(sentences)
  2315. '''
  2316. #限流执行
  2317. key_nerToken = "nerToken"
  2318. start_time = time.time()
  2319. # tokens_all = getTokens(sentences,useselffool=useselffool)
  2320. tokens_all = getTokens([re.sub("##attachment_begin##|##attachment_end##","",_sen) for _sen in sentences],useselffool=useselffool)
  2321. if key_nerToken not in cost_time:
  2322. cost_time[key_nerToken] = 0
  2323. cost_time[key_nerToken] += round(time.time()-start_time,2)
  2324. in_attachment = False
  2325. for sentence_index in range(len(sentences)):
  2326. sentence_text = sentences[sentence_index]
  2327. if re.search("##attachment_begin##",sentence_text):
  2328. in_attachment = True
  2329. sentence_text = re.sub("##attachment_begin##","",sentence_text)
  2330. if re.search("##attachment_end##",sentence_text):
  2331. in_attachment = False
  2332. sentence_text = re.sub("##attachment_end##", "", sentence_text)
  2333. if sentence_index >= attachment_begin_index and attachment_begin_index!=-1:
  2334. in_attachment = True
  2335. tokens = tokens_all[sentence_index]
  2336. #pos_tag = pos_all[sentence_index]
  2337. pos_tag = ""
  2338. ner_entitys = ""
  2339. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys,in_attachment=in_attachment))
  2340. if len(list_sentences_temp)==0:
  2341. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  2342. list_sentences.append(list_sentences_temp)
  2343. list_outlines.append(outline_list)
  2344. article.content = re.sub("##attachment_begin##|##attachment_end##", "", article.content)
  2345. return list_sentences,list_outlines
  2346. def get_money_entity(sentence_text, found_yeji, in_attachment=False):
  2347. money_list = []
  2348. # 使用正则识别金额
  2349. entity_type = "money"
  2350. list_money_pattern = {"cn": "(()(?P<filter_kw>百分之)?(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  2351. "key_word": "((?P<text_key_word>(?:[¥¥]+,?|[单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资))?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果)(?:[,,\[(\(]*\s*(人民币|单位:)?/?(?P<unit_key_word_before>[万亿]?(?:[美日欧]元|元(/(M2|[\u4e00-\u9fa5]{1,3}))?)?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[\])\)]?)\s*[,,::]*(RMB|USD|EUR|JPY|CNY)?[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,8}?))(第[123一二三]名[::])?(\d+(\*\d+%)+=)?(?P<money_key_word>\d{1,3}([,,]\d{3})+(\.\d+)?|\d+(\.\d+)?[百千]{,1})(?P<science_key_word>(E-?\d+))?(?:[(\(]?(?P<filter_>[%%‰折])*\s*,?((金额)?单位[::])?(?P<unit_key_word_behind>[万亿]?(?:[美日欧]元|元)?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[)\)]?))",
  2352. "front_m": "((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万亿]?(?:[美日欧]元|元))\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,7}?))(?P<money_front_m>\d{1,3}([,,]\d{3})+(\.\d+)?|\d+(\.\d+)?(?:,?)[百千]*)(?P<science_front_m>(E-?\d+))?())",
  2353. "behind_m": "(()()(?P<money_behind_m>\d{1,3}([,,]\d{3})+(\.\d+)?|\d+(\.\d+)?(?:,?)[百千]*)(?P<science_behind_m>(E-?\d+))?(人民币)?[\((]?(?P<unit_behind_m>[万亿]?(?:[美日欧]元|元)(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\))]?)"}
  2354. # 2021/7/19 调整金额,单位提取正则,修复部分金额因为单位提取失败被过滤问题。
  2355. pattern_money = re.compile("%s|%s|%s|%s" % (
  2356. list_money_pattern["cn"], list_money_pattern["key_word"], list_money_pattern["behind_m"],
  2357. list_money_pattern["front_m"]))
  2358. if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text):
  2359. found_yeji += 1
  2360. if found_yeji >= 2: # 过滤掉业绩后面的所有金额
  2361. all_match = []
  2362. else:
  2363. ser = re.search('((收费标准|计算[方公]?式):|\w{3,5}\s*=)+\s*[中标投标成交金额招标人预算价格万元\s()()\[\]【】\d\.%%‰\+\-*×/]{20,}[,。]?', sentence_text) # 过滤掉收费标准里面的金额
  2364. if ser:
  2365. all_match = re.finditer(pattern_money, sentence_text.replace(ser.group(0), ' ' * len(ser.group(0))))
  2366. else:
  2367. all_match = re.finditer(pattern_money, sentence_text)
  2368. for _match in all_match:
  2369. # print('_match: ', _match.group())
  2370. if len(_match.group()) > 0:
  2371. # print("===",_match.group())
  2372. # # print(_match.groupdict())
  2373. notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
  2374. unit = ""
  2375. entity_text = ""
  2376. start_index = ""
  2377. end_index = ""
  2378. text_beforeMoney = ""
  2379. filter = ""
  2380. filter_unit = False
  2381. notSure = False
  2382. science = ""
  2383. if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额
  2384. # print('金额在业绩后面: ', _match.group(0))
  2385. found_yeji += 1
  2386. break
  2387. for k, v in _match.groupdict().items():
  2388. if v != "" and v is not None:
  2389. if k == 'text_key_word':
  2390. notSure = True
  2391. if k.split("_")[0] == "money":
  2392. entity_text = v
  2393. # print(_match.group(k), 'entity_text: ', sentence_text[_match.start(k): _match.end(k)])
  2394. if entity_text.endswith(',00'): # 金额逗号后面不可能为两个0结尾,应该小数点识别错,直接去掉
  2395. entity_text = entity_text[:-3]
  2396. if k.split("_")[0] == "unit":
  2397. if v == '万元' or unit == "": # 处理 预算金额(元):160万元 这种出现前后单位不一致情况
  2398. unit = v
  2399. if k.split("_")[0] == "text":
  2400. # print('text_before: ', _match.group(k))
  2401. text_beforeMoney = v
  2402. if k.split("_")[0] == "filter":
  2403. filter = v
  2404. if re.search("filter_unit", k) is not None:
  2405. filter_unit = True
  2406. if k.split("_")[0] == 'science':
  2407. science = v
  2408. # print("金额:{0} ,单位:{1}, 前文:{2}, filter: {3}, filter_unit: {4}".format(entity_text,unit,text_beforeMoney,filter,filter_unit))
  2409. # if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()): # 2021/7/19 修正OCR识别小数点为逗号
  2410. # if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0] - 2):_match.span()[0]]):
  2411. # entity_text = re.sub('\d+,', '', entity_text)
  2412. # else:
  2413. # entity_text = entity_text.replace(',', '.')
  2414. # # print(' 修正OCR识别小数点为逗号')
  2415. if filter != "":
  2416. continue
  2417. if len(entity_text)>30 or len(re.sub('[E-]', '', science))>2: # 限制数字长度,避免类似265339018附件金额错误,数值超大报错 decimal.InvalidOperation
  2418. continue
  2419. start_index, end_index = _match.span()
  2420. start_index += len(text_beforeMoney)
  2421. '''过滤掉手机号码作为金额'''
  2422. if re.search('电话|手机|联系|方式|编号|编码|日期|数字|时间', text_beforeMoney):
  2423. # print('过滤掉手机号码作为金额')
  2424. continue
  2425. elif re.search('^1[3-9]\d{9}$', entity_text) and re.search(':\w{1,3}$', text_beforeMoney): # 过滤掉类似 '13863441880', '金额(万元):季勇13863441880'
  2426. # print('过滤掉手机号码作为金额')
  2427. continue
  2428. if unit == "": # 2021/7/21 有明显金额特征的补充单位,避免被过滤
  2429. if (re.search('(¥|¥|RMB|CNY)[::]?$', text_beforeMoney) or re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,}', entity_text)):
  2430. if entity_text.endswith('万元'):
  2431. unit = '万元'
  2432. entity_text = entity_text[:-2]
  2433. else:
  2434. unit = '元'
  2435. # print('1明显金额特征补充单位 元')
  2436. elif re.search('USD[::]?$', text_beforeMoney):
  2437. unit = '美元'
  2438. elif re.search('EUR[::]?$', text_beforeMoney):
  2439. unit = '欧元'
  2440. elif re.search('JPY[::]?$', text_beforeMoney):
  2441. unit = '日元'
  2442. elif re.search('^[-—]+[\d,.]+万元', sentence_text[end_index:]):
  2443. # print('两个金额连接后面的有单位,用后面单位')
  2444. unit = '万元'
  2445. elif re.search('([单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资))?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费)[::为]*-?$', text_beforeMoney.strip()) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}', entity_text) == None:
  2446. if re.search('^[\d,,.]+$', entity_text) and float(re.sub('[,,]', '', entity_text))<500 and re.search('万元', sentence_text):
  2447. unit = '万元'
  2448. # print('金额较小且句子中有万元的,补充单位为万元')
  2449. elif re.search('^\d{1,3}\.\d{4,6}$', entity_text) and re.search('0000$', entity_text) == None:
  2450. unit = '万元'
  2451. else:
  2452. unit = '元'
  2453. # print('金额前面紧接关键词的补充单位 元')
  2454. elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7},?)$)|(^\d{,3}(,\d{3})+,?$)', entity_text):
  2455. unit = '元'
  2456. # print('3明显金额特征补充单位 元')
  2457. else:
  2458. # print('过滤掉没单位金额: ',entity_text)
  2459. continue
  2460. elif unit == '万元':
  2461. if end_index < len(sentence_text) and sentence_text[end_index] == '元' and re.search('\d$', entity_text):
  2462. unit = '元'
  2463. elif re.search('^[5-9]\d{6,}\.\d{2}$', entity_text): # 五百亿以上的万元改为元
  2464. unit = '元'
  2465. if unit.find("万") >= 0 and entity_text.find("万") >= 0: # 2021/7/19修改为金额文本有万,不计算单位
  2466. # print('修正金额及单位都有万, 金额:',entity_text, '单位:',unit)
  2467. unit = "元"
  2468. if re.search('.*万元万元', entity_text): # 2021/7/19 修正两个万元
  2469. # print(' 修正两个万元',entity_text)
  2470. entity_text = entity_text.replace('万元万元', '万元')
  2471. else:
  2472. if filter_unit:
  2473. continue
  2474. # symbol = '-' if entity_text.startswith('-') and not entity_text.startswith('--') and re.search('\d+$', sentence_text[:begin_index_temp]) == None else '' # 负值金额前面保留负号 ,后面这些不作为负金额 起拍价:105.29-200.46万元 预 算 --- 350000.0 2023/04/14 取消符号
  2475. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", entity_text)
  2476. # print('转换前金额:', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
  2477. if re.search('总投资|投资总额|总预算|总概算|投资规模|批复概算|投资额',
  2478. sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]): # 2021/8/5过滤掉总投资金额
  2479. # print('总投资金额: ', _match.group(0))
  2480. notes = '总投资'
  2481. elif re.search('投资|概算|建安费|其他费用|基本预备费',
  2482. sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额
  2483. notes = '投资'
  2484. elif re.search('工程造价',
  2485. sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额
  2486. notes = '工程造价'
  2487. elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
  2488. or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\((]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\))]*[::为]',
  2489. sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
  2490. or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
  2491. sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
  2492. notes = '保证金'
  2493. # print('保证金信息:', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
  2494. elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
  2495. sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
  2496. notes = '成本警戒线'
  2497. elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为:]', sentence_text[_match.span()[0]:_match.span()[1]]):
  2498. cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
  2499. notes = cost_re.group(1)
  2500. elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
  2501. notes = '单价'
  2502. elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
  2503. notes = '大写'
  2504. if entity_text[0] == "拾": # 2021/12/16 修正大写金额省略了数字转换错误问题
  2505. entity_text = "壹" + entity_text
  2506. # print("补充备注:notes = 大写")
  2507. if len(unit) > 0:
  2508. if unit.find('万') >= 0 and len(entity_text.split('.')[0]) >= 8: # 2021/7/19 修正万元金额过大的情况
  2509. # print('修正单位万元金额过大的情况 金额:', entity_text, '单位:', unit)
  2510. entity_text = str(
  2511. getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]) / 10000)
  2512. unit = '元' # 修正金额后单位 重置为元
  2513. else:
  2514. # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
  2515. entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]))
  2516. else:
  2517. if entity_text.find('万') >= 0 and entity_text.split('.')[0].isdigit() and len(
  2518. entity_text.split('.')[0]) >= 8:
  2519. entity_text = str(getUnifyMoney(entity_text) / 10000)
  2520. # print('修正金额字段含万 过大的情况')
  2521. else:
  2522. entity_text = str(getUnifyMoney(entity_text))
  2523. if science and re.search('^E-?\d+$', science): # 科学计数
  2524. entity_text = str(Decimal(entity_text + science)) if Decimal(entity_text + science) > 100 and Decimal(
  2525. entity_text + science) < 10000000000 else entity_text # 结果大于100及小于100万才使用科学计算
  2526. if float(entity_text) > 100000000000: # float(entity_text)<100 or 2022/3/4 取消最小金额限制
  2527. # print('过滤掉金额:float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
  2528. continue
  2529. if notSure and unit == "" and float(entity_text) > 100 * 10000:
  2530. # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000:', entity_text, unit)
  2531. continue
  2532. # print("金额:{0} ,单位:{1}, 前文:{2}, filter: {3}, filter_unit: {4}".format(entity_text, unit, text_beforeMoney,
  2533. # filter, filter_unit))
  2534. if re.search('[%%‰折]|费率|下浮率', text_beforeMoney) and float(entity_text)<1000: # 过滤掉可能是费率的金额
  2535. # print('过滤掉可能是费率的金额')
  2536. continue
  2537. money_list.append((entity_text, start_index, end_index, unit, notes))
  2538. return money_list, found_yeji
  2539. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  2540. '''
  2541. :param list_sentences:分局情况
  2542. :param cost_time:
  2543. :return: list_entitys
  2544. '''
  2545. list_entitys = []
  2546. not_extract_roles = ['黄埔军校', '国有资产管理处'] # 需要过滤掉的企业单位
  2547. for list_sentence in list_sentences:
  2548. sentences = []
  2549. list_entitys_temp = []
  2550. for _sentence in list_sentence:
  2551. sentences.append(_sentence.sentence_text)
  2552. time1 = time.time()
  2553. '''
  2554. tokens_all = fool.cut(sentences)
  2555. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  2556. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  2557. ner_entitys_all = fool.ner(sentences)
  2558. '''
  2559. #限流执行
  2560. key_nerToken = "nerToken"
  2561. start_time = time.time()
  2562. found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
  2563. # found_pingbiao = False
  2564. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  2565. if key_nerToken not in cost_time:
  2566. cost_time[key_nerToken] = 0
  2567. cost_time[key_nerToken] += round(time.time()-start_time,2)
  2568. doctextcon_sentence_len = sum([1 for sentence in list_sentence if not sentence.in_attachment])
  2569. company_dict = set()
  2570. company_index = dict((i,set()) for i in range(len(list_sentence)))
  2571. for sentence_index in range(len(list_sentence)):
  2572. list_sentence_entitys = []
  2573. sentence_text = list_sentence[sentence_index].sentence_text
  2574. tokens = list_sentence[sentence_index].tokens
  2575. doc_id = list_sentence[sentence_index].doc_id
  2576. in_attachment = list_sentence[sentence_index].in_attachment
  2577. list_tokenbegin = []
  2578. begin = 0
  2579. for i in range(0,len(tokens)):
  2580. list_tokenbegin.append(begin)
  2581. begin += len(str(tokens[i]))
  2582. list_tokenbegin.append(begin+1)
  2583. #pos_tag = pos_all[sentence_index]
  2584. pos_tag = ""
  2585. ner_entitys = ner_entitys_all[sentence_index]
  2586. '''正则识别角色实体 经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
  2587. for it in re.finditer(
  2588. '(?P<text_key_word>(((单一来源|中标|中选|中价|成交)(供应商|供货商|服务商|候选人|单位|人))|(供应商|供货商|服务商|候选人))(名称)?[为::]+)(?P<text>([()\w]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[,。]',
  2589. sentence_text):
  2590. for k, v in it.groupdict().items():
  2591. if k == 'text_key_word':
  2592. keyword = v
  2593. if k == 'text':
  2594. entity = v
  2595. b = it.start() + len(keyword)
  2596. e = it.end() - 1
  2597. if (b, e, 'location', entity) in ner_entitys:
  2598. ner_entitys.remove((b, e, 'location', entity))
  2599. ner_entitys.append((b, e, 'company', entity))
  2600. elif (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  2601. ner_entitys.append((b, e, 'company', entity))
  2602. for it in re.finditer(
  2603. '(?P<text_key_word>((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为::]+)(?P<text>\w{2,4}[省市县区镇]([()\w]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[,。]',
  2604. sentence_text):
  2605. for k, v in it.groupdict().items():
  2606. if k == 'text_key_word':
  2607. keyword = v
  2608. if k == 'text':
  2609. entity = v
  2610. b = it.start() + len(keyword)
  2611. e = it.end() - 1
  2612. if (b, e, 'location', entity) in ner_entitys:
  2613. ner_entitys.remove((b, e, 'location', entity))
  2614. ner_entitys.append((b, e, 'org', entity))
  2615. if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  2616. ner_entitys.append((b, e, 'org', entity))
  2617. for ner_entity in ner_entitys:
  2618. if ner_entity[2] in ['company','org']:
  2619. company_dict.add((ner_entity[2],ner_entity[3]))
  2620. company_index[sentence_index].add((ner_entity[0],ner_entity[1]))
  2621. #识别package
  2622. ner_time_list = []
  2623. #识别实体
  2624. for ner_entity in ner_entitys:
  2625. begin_index_temp = ner_entity[0]
  2626. end_index_temp = ner_entity[1]
  2627. entity_type = ner_entity[2]
  2628. entity_text = ner_entity[3]
  2629. if entity_type=='time':
  2630. ner_time_list.append((begin_index_temp,end_index_temp))
  2631. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  2632. continue
  2633. # 实体长度限制
  2634. if entity_type in ["org","company"] and len(entity_text)>30:
  2635. continue
  2636. if entity_type == "person" and len(entity_text) > 20:
  2637. continue
  2638. elif entity_type=="person" and len(entity_text)>10 and len(re.findall("[\u4e00-\u9fa5]",entity_text))<len(entity_text)/2:
  2639. continue
  2640. # 识别不完整的组织机构补充
  2641. if entity_type in ["org"]:
  2642. end_words = re.search("^[\u4e00-\u9fa5]{,5}(?:办公室|部|中心|处|会)",sentence_text[end_index_temp:end_index_temp+10])
  2643. if end_words:
  2644. entity_text = entity_text + end_words.group()
  2645. for j in range(len(list_tokenbegin)):
  2646. if list_tokenbegin[j]==begin_index_temp:
  2647. begin_index = j
  2648. break
  2649. elif list_tokenbegin[j]>begin_index_temp:
  2650. begin_index = j-1
  2651. break
  2652. begin_index_temp += len(str(entity_text))
  2653. for j in range(begin_index,len(list_tokenbegin)):
  2654. if list_tokenbegin[j]>=begin_index_temp:
  2655. end_index = j-1
  2656. break
  2657. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2658. #去掉标点符号
  2659. if entity_type!='time':
  2660. entity_text = re.sub("[,,。:!&@$\*\s]","",entity_text)
  2661. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  2662. # 组织机构实体名称补充
  2663. if entity_type in ["org", "company"]:
  2664. if entity_text in not_extract_roles: # 过滤掉名称在 需要过滤企业单位列表里的
  2665. continue
  2666. if not re.search("有限责任公司|有限公司",entity_text):
  2667. fix_name = re.search("(有限)([责贵]?任?)(公?司?)",entity_text)
  2668. if fix_name:
  2669. if len(fix_name.group(2))>0:
  2670. _text = fix_name.group()
  2671. if '司' in _text:
  2672. entity_text = entity_text.replace(_text, "有限责任公司")
  2673. else:
  2674. _text = re.search(_text + "[^司]{0,5}司", entity_text)
  2675. if _text:
  2676. _text = _text.group()
  2677. entity_text = entity_text.replace(_text, "有限责任公司")
  2678. else:
  2679. entity_text = entity_text.replace(entity_text[fix_name.start():], "有限责任公司")
  2680. elif len(fix_name.group(3))>0:
  2681. _text = fix_name.group()
  2682. if '司' in _text:
  2683. entity_text = entity_text.replace(_text, "有限公司")
  2684. else:
  2685. _text = re.search(_text + "[^司]{0,3}司", entity_text)
  2686. if _text:
  2687. _text = _text.group()
  2688. entity_text = entity_text.replace(_text, "有限公司")
  2689. else:
  2690. entity_text = entity_text.replace(entity_text[fix_name.start():], "有限公司")
  2691. elif re.search("有限$", entity_text):
  2692. entity_text = re.sub("有限$","有限公司",entity_text)
  2693. entity_text = entity_text.replace("有公司","有限公司")
  2694. '''下面对公司实体进行清洗'''
  2695. entity_text = re.sub('\s', '', entity_text)
  2696. if re.search('^(\d{4}年)?[\-\d月日份]*\w{2,3}分公司$', entity_text): # 删除
  2697. # print('公司实体不符合规范:', entity_text)
  2698. continue
  2699. elif re.match('xx|XX', entity_text): # 删除
  2700. # print('公司实体不符合规范:', entity_text)
  2701. continue
  2702. elif re.match('\.?(rar|zip|pdf|df|doc|docx|xls|xlsx|jpg|png)', entity_text):
  2703. entity_text = re.sub('\.?(rar|zip|pdf|df|doc|docx|xls|xlsx|jpg|png)', '', entity_text)
  2704. elif re.match(
  2705. '((\d{4}[年-])[\-\d:\s元月日份]*|\d{1,2}月[\d日.-]*(日?常?计划)?|\d{1,2}[.-]?|[A-Za-z](包|标段?)?|[a-zA-Z0-9]+-[a-zA-Z0-9-]*|[a-zA-Z]{1,2}|[①②③④⑤⑥⑦⑧⑨⑩]|\s|title\=|【[a-zA-Z0-9]+】|[^\w])[\u4e00-\u9fa5]+',
  2706. entity_text):
  2707. filter = re.match(
  2708. '((\d{4}[年-])[\-\d:\s元月日份]*|\d{1,2}月[\d日.-]*(日?常?计划)?|\d{1,2}[.-]?|[A-Za-z](包|标段?)?|[a-zA-Z0-9]+-[a-zA-Z0-9-]*|[a-zA-Z]{1,2}|[①②③④⑤⑥⑦⑧⑨⑩]|\s|title\=|【[a-zA-Z0-9]+】|[^\w])[\u4e00-\u9fa5]+',
  2709. entity_text).group(1)
  2710. entity_text = entity_text.replace(filter, '')
  2711. elif re.search('\]|\[|\]|[【】{}「?:∶〔·.\'#~_ΓΙεⅠ]', entity_text):
  2712. entity_text = re.sub('\]|\[|\]|[【】「?:∶〔·.\'#~_ΓΙεⅠ]', '', entity_text)
  2713. if len(re.sub('(项目|分|有限)?公司|集团|制造部|中心|医院|学校|大学|中学|小学|幼儿园', '', entity_text))<2:
  2714. # print('公司实体不符合规范:', entity_text)
  2715. continue
  2716. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
  2717. # 标记文章末尾的"发布人”、“发布时间”实体
  2718. if sentence_index==len(list_sentence)-1 or sentence_index==doctextcon_sentence_len-1:
  2719. if len(list_sentence_entitys[-2:])==2:
  2720. second2last = list_sentence_entitys[-2]
  2721. last = list_sentence_entitys[-1]
  2722. if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
  2723. second2last.entity_type=="time" and last.entity_type in ["company",'org']):
  2724. if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
  2725. last.is_tail = True
  2726. second2last.is_tail = True
  2727. #使用正则识别金额
  2728. money_list, found_yeji = get_money_entity(sentence_text, found_yeji, in_attachment)
  2729. entity_type = "money"
  2730. for money in money_list:
  2731. # print('money: ', money)
  2732. entity_text, begin_index, end_index, unit, notes = money
  2733. end_index = end_index - 1 if entity_text.endswith(',') else end_index
  2734. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2735. _exists = False
  2736. for item in list_sentence_entitys:
  2737. if item.entity_id==entity_id and item.entity_type==entity_type:
  2738. _exists = True
  2739. if (begin_index >=item.wordOffset_begin and begin_index<item.wordOffset_end) or (end_index>item.wordOffset_begin and end_index<=item.wordOffset_end):
  2740. _exists = True
  2741. # print('_exists: ',begin_index, end_index, item.wordOffset_begin, item.wordOffset_end, item.entity_text, item.entity_type)
  2742. if not _exists:
  2743. if float(entity_text)>1:
  2744. # if symbol == '-': # 负值金额保留负号
  2745. # entity_text = '-'+entity_text # 20230414 取消符号
  2746. begin_words = changeIndexFromWordToWords(tokens, begin_index)
  2747. end_words = changeIndexFromWordToWords(tokens, end_index)
  2748. # print('金额位置: ', begin_index, begin_words,end_index, end_words)
  2749. # print('金额召回: ', entity_text, sentence_text[begin_index:end_index], tokens[begin_words:end_words])
  2750. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_words,end_words,begin_index,end_index,in_attachment=in_attachment))
  2751. list_sentence_entitys[-1].notes = notes # 2021/7/20 新增金额备注
  2752. list_sentence_entitys[-1].money_unit = unit # 2021/7/20 新增金额备注
  2753. # print('预处理中的 金额:%s, 单位:%s'%(entity_text,unit))
  2754. # print(entity_text,unit,notes)
  2755. # "联系人"正则补充提取 2021/11/15 新增
  2756. list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
  2757. error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
  2758. '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理',
  2759. '代理人','采购','附件','注意','登录','报名','踏勘',"测试",'交货']
  2760. list_person_text = set(list_person_text + error_text)
  2761. re_person = re.compile("联系人[::]([\u4e00-\u9fa5]工)|"
  2762. "联系人[::]([\u4e00-\u9fa5]{2,3})(?=,?联系)|"
  2763. "联系人[::]([\u4e00-\u9fa5]{2,3})(?=[,。;、])"
  2764. )
  2765. list_person = []
  2766. if not in_attachment:
  2767. for match_result in re_person.finditer(sentence_text):
  2768. match_text = match_result.group()
  2769. entity_text = match_text[4:]
  2770. wordOffset_begin = match_result.start() + 4
  2771. wordOffset_end = match_result.end()
  2772. # print(text[wordOffset_begin:wordOffset_end])
  2773. # 排除一些不为人名的实体
  2774. if re.search("^[\u4e00-\u9fa5]{7,}([,。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
  2775. continue
  2776. if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
  2777. _person = dict()
  2778. _person['body'] = entity_text
  2779. _person['begin_index'] = wordOffset_begin
  2780. _person['end_index'] = wordOffset_end
  2781. list_person.append(_person)
  2782. entity_type = "person"
  2783. for person in list_person:
  2784. begin_index_temp = person['begin_index']
  2785. for j in range(len(list_tokenbegin)):
  2786. if list_tokenbegin[j] == begin_index_temp:
  2787. begin_index = j
  2788. break
  2789. elif list_tokenbegin[j] > begin_index_temp:
  2790. begin_index = j - 1
  2791. break
  2792. index = person['end_index']
  2793. end_index_temp = index
  2794. for j in range(begin_index, len(list_tokenbegin)):
  2795. if list_tokenbegin[j] >= index:
  2796. end_index = j - 1
  2797. break
  2798. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2799. entity_text = person['body']
  2800. list_sentence_entitys.append(
  2801. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2802. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2803. # 时间实体格式补充
  2804. re_time_new = re.compile("20\d{2}-\d{1,2}-\d{1,2}|20\d{2}/\d{1,2}/\d{1,2}|20\d{2}\.\d{1,2}\.\d{1,2}|20\d{2}(?:0[1-9]|1[0-2])(?:0[1-9]|[1-2][0-9]|3[0-1])")
  2805. entity_type = "time"
  2806. for _time in re.finditer(re_time_new,sentence_text):
  2807. entity_text = _time.group()
  2808. begin_index_temp = _time.start()
  2809. end_index_temp = _time.end()
  2810. is_same = False
  2811. for t_index in ner_time_list:
  2812. if begin_index_temp>=t_index[0] and end_index_temp<=t_index[1]:
  2813. is_same = True
  2814. break
  2815. if is_same:
  2816. continue
  2817. if _time.start()!=0 and re.search("\d",sentence_text[_time.start()-1:_time.start()]):
  2818. continue
  2819. # 纯数字格式,例:20190509
  2820. if re.search("^\d{8}$",entity_text):
  2821. if _time.end()!=len(sentence_text) and re.search("[\da-zA-z]",sentence_text[_time.end():_time.end()+1]):
  2822. continue
  2823. entity_text = entity_text[:4] + "-" + entity_text[4:6] + "-" + entity_text[6:8]
  2824. if not timeFormat(entity_text):
  2825. continue
  2826. for j in range(len(list_tokenbegin)):
  2827. if list_tokenbegin[j] == begin_index_temp:
  2828. begin_index = j
  2829. break
  2830. elif list_tokenbegin[j] > begin_index_temp:
  2831. begin_index = j - 1
  2832. break
  2833. for j in range(begin_index, len(list_tokenbegin)):
  2834. if list_tokenbegin[j] >= end_index_temp:
  2835. end_index = j - 1
  2836. break
  2837. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2838. list_sentence_entitys.append(
  2839. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2840. begin_index_temp, end_index_temp, in_attachment=in_attachment))
  2841. # 资金来源提取 2020/12/30 新增
  2842. list_moneySource = extract_moneySource(sentence_text)
  2843. entity_type = "moneysource"
  2844. for moneySource in list_moneySource:
  2845. entity_text = moneySource['body']
  2846. if len(entity_text)>50:
  2847. continue
  2848. begin_index_temp = moneySource['begin_index']
  2849. for j in range(len(list_tokenbegin)):
  2850. if list_tokenbegin[j] == begin_index_temp:
  2851. begin_index = j
  2852. break
  2853. elif list_tokenbegin[j] > begin_index_temp:
  2854. begin_index = j - 1
  2855. break
  2856. index = moneySource['end_index']
  2857. end_index_temp = index
  2858. for j in range(begin_index, len(list_tokenbegin)):
  2859. if list_tokenbegin[j] >= index:
  2860. end_index = j - 1
  2861. break
  2862. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2863. list_sentence_entitys.append(
  2864. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2865. begin_index_temp, end_index_temp,in_attachment=in_attachment,prob=moneySource['prob']))
  2866. # 电子邮箱提取 2021/11/04 新增
  2867. list_email = extract_email(sentence_text)
  2868. entity_type = "email" # 电子邮箱
  2869. for email in list_email:
  2870. begin_index_temp = email['begin_index']
  2871. for j in range(len(list_tokenbegin)):
  2872. if list_tokenbegin[j] == begin_index_temp:
  2873. begin_index = j
  2874. break
  2875. elif list_tokenbegin[j] > begin_index_temp:
  2876. begin_index = j - 1
  2877. break
  2878. index = email['end_index']
  2879. end_index_temp = index
  2880. for j in range(begin_index, len(list_tokenbegin)):
  2881. if list_tokenbegin[j] >= index:
  2882. end_index = j - 1
  2883. break
  2884. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2885. entity_text = email['body']
  2886. list_sentence_entitys.append(
  2887. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2888. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2889. # 服务期限提取 2020/12/30 新增
  2890. list_servicetime = extract_servicetime(sentence_text)
  2891. entity_type = "serviceTime"
  2892. for servicetime in list_servicetime:
  2893. entity_text = servicetime['body']
  2894. begin_index_temp = servicetime['begin_index']
  2895. for j in range(len(list_tokenbegin)):
  2896. if list_tokenbegin[j] == begin_index_temp:
  2897. begin_index = j
  2898. break
  2899. elif list_tokenbegin[j] > begin_index_temp:
  2900. begin_index = j - 1
  2901. break
  2902. index = servicetime['end_index']
  2903. end_index_temp = index
  2904. for j in range(begin_index, len(list_tokenbegin)):
  2905. if list_tokenbegin[j] >= index:
  2906. end_index = j - 1
  2907. break
  2908. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2909. list_sentence_entitys.append(
  2910. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2911. begin_index_temp, end_index_temp,in_attachment=in_attachment, prob=servicetime["prob"]))
  2912. # 2021/12/29 新增比率提取
  2913. list_ratio = extract_ratio(sentence_text)
  2914. entity_type = "ratio"
  2915. for ratio in list_ratio:
  2916. # print("ratio", ratio)
  2917. begin_index_temp = ratio['begin_index']
  2918. for j in range(len(list_tokenbegin)):
  2919. if list_tokenbegin[j] == begin_index_temp:
  2920. begin_index = j
  2921. break
  2922. elif list_tokenbegin[j] > begin_index_temp:
  2923. begin_index = j - 1
  2924. break
  2925. index = ratio['end_index']
  2926. end_index_temp = index
  2927. for j in range(begin_index, len(list_tokenbegin)):
  2928. if list_tokenbegin[j] >= index:
  2929. end_index = j - 1
  2930. break
  2931. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2932. entity_text = ratio['body']
  2933. ratio_value = (ratio['value'],ratio['type'])
  2934. _entity = Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2935. begin_index_temp, end_index_temp,in_attachment=in_attachment)
  2936. _entity.ratio_value = ratio_value
  2937. list_sentence_entitys.append(_entity)
  2938. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  2939. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  2940. # 补充ner模型未识别全的company/org实体
  2941. for sentence_index in range(len(list_sentence)):
  2942. sentence_text = list_sentence[sentence_index].sentence_text
  2943. tokens = list_sentence[sentence_index].tokens
  2944. doc_id = list_sentence[sentence_index].doc_id
  2945. in_attachment = list_sentence[sentence_index].in_attachment
  2946. list_tokenbegin = []
  2947. begin = 0
  2948. for i in range(0, len(tokens)):
  2949. list_tokenbegin.append(begin)
  2950. begin += len(str(tokens[i]))
  2951. list_tokenbegin.append(begin + 1)
  2952. add_sentence_entitys = []
  2953. company_dict = sorted(list(company_dict),key=lambda x:len(x[1]),reverse=True)
  2954. for company_type,company_text in company_dict:
  2955. begin_index_list = findAllIndex(company_text,sentence_text)
  2956. for begin_index in begin_index_list:
  2957. is_continue = False
  2958. for t_begin,t_end in list(company_index[sentence_index]):
  2959. if begin_index>=t_begin and begin_index+len(company_text)<=t_end:
  2960. is_continue = True
  2961. break
  2962. if not is_continue:
  2963. add_sentence_entitys.append((begin_index,begin_index+len(company_text),company_type,company_text))
  2964. company_index[sentence_index].add((begin_index,begin_index+len(company_text)))
  2965. else:
  2966. continue
  2967. for ner_entity in add_sentence_entitys:
  2968. begin_index_temp = ner_entity[0]
  2969. end_index_temp = ner_entity[1]
  2970. entity_type = ner_entity[2]
  2971. entity_text = ner_entity[3]
  2972. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  2973. continue
  2974. for j in range(len(list_tokenbegin)):
  2975. if list_tokenbegin[j]==begin_index_temp:
  2976. begin_index = j
  2977. break
  2978. elif list_tokenbegin[j]>begin_index_temp:
  2979. begin_index = j-1
  2980. break
  2981. begin_index_temp += len(str(entity_text))
  2982. for j in range(begin_index,len(list_tokenbegin)):
  2983. if list_tokenbegin[j]>=begin_index_temp:
  2984. end_index = j-1
  2985. break
  2986. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2987. #去掉标点符号
  2988. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  2989. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  2990. list_entitys_temp.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
  2991. list_entitys_temp.sort(key=lambda x:(x.sentence_index,x.begin_index))
  2992. list_entitys.append(list_entitys_temp)
  2993. return list_entitys
  2994. def union_result(codeName,prem):
  2995. '''
  2996. @summary:模型的结果拼成字典
  2997. @param:
  2998. codeName:编号名称模型的结果字典
  2999. prem:拿到属性的角色的字典
  3000. @return:拼接起来的字典
  3001. '''
  3002. result = []
  3003. assert len(codeName)==len(prem)
  3004. for item_code,item_prem in zip(codeName,prem):
  3005. result.append(dict(item_code,**item_prem))
  3006. return result
  3007. def persistenceData(data):
  3008. '''
  3009. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  3010. '''
  3011. import psycopg2
  3012. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  3013. cursor = conn.cursor()
  3014. for item_index in range(len(data)):
  3015. item = data[item_index]
  3016. doc_id = item[0]
  3017. dic = item[1]
  3018. code = dic['code']
  3019. name = dic['name']
  3020. prem = dic['prem']
  3021. if len(code)==0:
  3022. code_insert = ""
  3023. else:
  3024. code_insert = ";".join(code)
  3025. prem_insert = ""
  3026. for item in prem:
  3027. for x in item:
  3028. if isinstance(x, list):
  3029. if len(x)>0:
  3030. for x1 in x:
  3031. prem_insert+="/".join(x1)+","
  3032. prem_insert+="$"
  3033. else:
  3034. prem_insert+=str(x)+"$"
  3035. prem_insert+=";"
  3036. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  3037. cursor.execute(sql)
  3038. conn.commit()
  3039. conn.close()
  3040. def persistenceData1(list_entitys,list_sentences):
  3041. '''
  3042. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  3043. '''
  3044. import psycopg2
  3045. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  3046. cursor = conn.cursor()
  3047. for list_entity in list_entitys:
  3048. for entity in list_entity:
  3049. if entity.values is not None:
  3050. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  3051. else:
  3052. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  3053. cursor.execute(sql)
  3054. for list_sentence in list_sentences:
  3055. for sentence in list_sentence:
  3056. str_tokens = "["
  3057. for item in sentence.tokens:
  3058. str_tokens += "'"
  3059. if item=="'":
  3060. str_tokens += "''"
  3061. else:
  3062. str_tokens += item
  3063. str_tokens += "',"
  3064. str_tokens = str_tokens[:-1]+"]"
  3065. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  3066. cursor.execute(sql)
  3067. conn.commit()
  3068. conn.close()
  3069. def _handle(item,result_queue):
  3070. dochtml = item["dochtml"]
  3071. docid = item["docid"]
  3072. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  3073. flag = False
  3074. if list_innerTable:
  3075. flag = True
  3076. for table in list_innerTable:
  3077. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  3078. def getPredictTable():
  3079. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  3080. import pandas as pd
  3081. import json
  3082. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  3083. df = pd.read_csv(filename)
  3084. df_data = {"json_table":[],"docid":[]}
  3085. _count = 0
  3086. _sum = len(df["docid"])
  3087. task_queue = Queue()
  3088. result_queue = Queue()
  3089. _index = 0
  3090. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  3091. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  3092. _index += 1
  3093. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  3094. mh.run()
  3095. while True:
  3096. try:
  3097. item = result_queue.get(block=True,timeout=1)
  3098. df_data["docid"].append(item["docid"])
  3099. df_data["json_table"].append(item["json_table"])
  3100. except Exception as e:
  3101. print(e)
  3102. break
  3103. df_1 = pd.DataFrame(df_data)
  3104. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  3105. if __name__=="__main__":
  3106. '''
  3107. import glob
  3108. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  3109. file_txt = str(file).replace("html","txt")
  3110. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  3111. f.write("\n================\n")
  3112. content = codecs.open(file,"r",encoding="utf8").read()
  3113. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  3114. '''
  3115. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  3116. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  3117. # getPredictTable()
  3118. with open('D:/138786703.html', 'r', encoding='utf-8') as f:
  3119. sourceContent = f.read()
  3120. # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
  3121. # print(article_processed)
  3122. list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
  3123. for entity in list_entitys[0]:
  3124. print(entity.entity_type, entity.entity_text)