Preprocessing.py 140 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851
  1. # -*- coding: utf-8 -*-
  2. from bs4 import BeautifulSoup, Comment
  3. import copy
  4. import sys
  5. import os
  6. import time
  7. import codecs
  8. from BiddingKG.dl.ratio.re_ratio import extract_ratio
  9. from BiddingKG.dl.table_head.predict import predict
  10. sys.setrecursionlimit(1000000)
  11. sys.path.append(os.path.abspath("../.."))
  12. sys.path.append(os.path.abspath(".."))
  13. from BiddingKG.dl.common.Utils import *
  14. from BiddingKG.dl.interface.Entitys import *
  15. from BiddingKG.dl.interface.predictor import getPredictor
  16. from BiddingKG.dl.common.nerUtils import *
  17. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  18. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  19. from BiddingKG.dl.relation_extraction.re_email import extract_email
  20. from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
  21. from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
  22. from BiddingKG.dl.entityLink.entityLink import *
  23. #
  24. def tableToText(soup):
  25. '''
  26. @param:
  27. soup:网页html的soup
  28. @return:处理完表格信息的网页text
  29. '''
  30. def getTrs(tbody):
  31. #获取所有的tr
  32. trs = []
  33. objs = tbody.find_all(recursive=False)
  34. for obj in objs:
  35. if obj.name=="tr":
  36. trs.append(obj)
  37. if obj.name=="tbody":
  38. for tr in obj.find_all("tr",recursive=False):
  39. trs.append(tr)
  40. return trs
  41. def fixSpan(tbody):
  42. # 处理colspan, rowspan信息补全问题
  43. #trs = tbody.findChildren('tr', recursive=False)
  44. trs = getTrs(tbody)
  45. ths_len = 0
  46. ths = list()
  47. trs_set = set()
  48. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  49. # 遍历每一个tr
  50. for indtr, tr in enumerate(trs):
  51. ths_tmp = tr.findChildren('th', recursive=False)
  52. #不补全含有表格的tr
  53. if len(tr.findChildren('table'))>0:
  54. continue
  55. if len(ths_tmp) > 0:
  56. ths_len = ths_len + len(ths_tmp)
  57. for th in ths_tmp:
  58. ths.append(th)
  59. trs_set.add(tr)
  60. # 遍历每行中的element
  61. tds = tr.findChildren(recursive=False)
  62. for indtd, td in enumerate(tds):
  63. # 若有colspan 则补全同一行下一个位置
  64. if 'colspan' in td.attrs:
  65. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  66. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  67. if col<100 and len(td.get_text())<1000:
  68. td['colspan'] = 1
  69. for i in range(1, col, 1):
  70. td.insert_after(copy.copy(td))
  71. for indtr, tr in enumerate(trs):
  72. ths_tmp = tr.findChildren('th', recursive=False)
  73. #不补全含有表格的tr
  74. if len(tr.findChildren('table'))>0:
  75. continue
  76. if len(ths_tmp) > 0:
  77. ths_len = ths_len + len(ths_tmp)
  78. for th in ths_tmp:
  79. ths.append(th)
  80. trs_set.add(tr)
  81. # 遍历每行中的element
  82. tds = tr.findChildren(recursive=False)
  83. for indtd, td in enumerate(tds):
  84. # 若有rowspan 则补全下一行同样位置
  85. if 'rowspan' in td.attrs:
  86. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  87. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  88. td['rowspan'] = 1
  89. for i in range(1, row, 1):
  90. # 获取下一行的所有td, 在对应的位置插入
  91. if indtr+i<len(trs):
  92. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  93. if len(tds1) >= (indtd) and len(tds1)>0:
  94. if indtd > 0:
  95. tds1[indtd - 1].insert_after(copy.copy(td))
  96. else:
  97. tds1[0].insert_before(copy.copy(td))
  98. elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全
  99. tds1[indtd-2].insert_after(copy.copy(td))
  100. def getTable(tbody):
  101. #trs = tbody.findChildren('tr', recursive=False)
  102. trs = getTrs(tbody)
  103. inner_table = []
  104. for tr in trs:
  105. tr_line = []
  106. tds = tr.findChildren(['td','th'], recursive=False)
  107. if len(tds)==0:
  108. tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
  109. for td in tds:
  110. # print("td", td)
  111. tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
  112. # print("segment td", segment(td,final=False))
  113. #tr_line.append([td.get_text(),0])
  114. inner_table.append(tr_line)
  115. return inner_table
  116. #处理表格不对齐的问题
  117. def fixTable(inner_table,fix_value="~~"):
  118. maxWidth = 0
  119. for item in inner_table:
  120. if len(item)>maxWidth:
  121. maxWidth = len(item)
  122. for i in range(len(inner_table)):
  123. if len(inner_table[i])<maxWidth:
  124. for j in range(maxWidth-len(inner_table[i])):
  125. inner_table[i].append([fix_value,0])
  126. return inner_table
  127. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  128. height = len(inner_table)
  129. width = len(inner_table[0])
  130. for i in range(height):
  131. point = ""
  132. for j in range(width):
  133. if inner_table[i][j][0]==point and point!="":
  134. inner_table[i][j][0] = pad_row
  135. else:
  136. if inner_table[i][j][0] not in [pad_row,pad_col]:
  137. point = inner_table[i][j][0]
  138. for j in range(width):
  139. point = ""
  140. for i in range(height):
  141. if inner_table[i][j][0]==point and point!="":
  142. inner_table[i][j][0] = pad_col
  143. else:
  144. if inner_table[i][j][0] not in [pad_row,pad_col]:
  145. point = inner_table[i][j][0]
  146. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  147. height = len(inner_table)
  148. width = len(inner_table[0])
  149. for i in range(height):
  150. for j in range(width):
  151. if inner_table[i][j][0]==pad_row:
  152. inner_table[i][j][0] = inner_table[i][j-1][0]
  153. inner_table[i][j][1] = inner_table[i][j-1][1]
  154. if inner_table[i][j][0]==pad_col:
  155. inner_table[i][j][0] = inner_table[i-1][j][0]
  156. inner_table[i][j][1] = inner_table[i-1][j][1]
  157. def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
  158. '''
  159. @summary: 修复表头识别,将明显错误的进行修正
  160. '''
  161. def repairNeeded(line):
  162. first_1 = -1
  163. last_1 = -1
  164. first_0 = -1
  165. last_0 = -1
  166. count_1 = 0
  167. count_0 = 0
  168. for i in range(len(line)):
  169. if line[i][0]==fix_value:
  170. continue
  171. if line[i][1]==1:
  172. if first_1==-1:
  173. first_1 = i
  174. last_1 = i
  175. count_1 += 1
  176. if line[i][1]==0:
  177. if first_0 == -1:
  178. first_0 = i
  179. last_0 = i
  180. count_0 += 1
  181. if first_1 ==-1 or last_0 == -1:
  182. return False
  183. #异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  184. if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
  185. return True
  186. return False
  187. def getsimilarity(line,line1):
  188. same_count = 0
  189. for item,item1 in zip(line,line1):
  190. if item[1]==item1[1]:
  191. same_count += 1
  192. return same_count/len(line)
  193. def selfrepair(inner_table,index,dye_set,key_set):
  194. '''
  195. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  196. '''
  197. #print("B",inner_table[index])
  198. min_presure = 3
  199. list_dye = []
  200. first = None
  201. count = 0
  202. temp_set = set()
  203. _index = 0
  204. for item in inner_table[index]:
  205. if first is None:
  206. first = item[1]
  207. if item[0] not in temp_set:
  208. count += 1
  209. temp_set.add(item[0])
  210. else:
  211. if first == item[1]:
  212. if item[0] not in temp_set:
  213. temp_set.add(item[0])
  214. count += 1
  215. else:
  216. list_dye.append([first,count,_index])
  217. first = item[1]
  218. temp_set.add(item[0])
  219. count = 1
  220. _index += 1
  221. list_dye.append([first,count,_index])
  222. if len(list_dye)>1:
  223. begin = 0
  224. end = 0
  225. for i in range(len(list_dye)):
  226. end = list_dye[i][2]
  227. dye_flag = False
  228. #首尾要求压力减一
  229. if i==0:
  230. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  231. dye_flag = True
  232. dye_type = list_dye[i+1][0]
  233. elif i==len(list_dye)-1:
  234. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  235. dye_flag = True
  236. dye_type = list_dye[i-1][0]
  237. else:
  238. if list_dye[i][1]>1:
  239. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  240. dye_flag = True
  241. dye_type = list_dye[i+1][0]
  242. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  243. dye_flag = True
  244. dye_type = list_dye[i-1][0]
  245. else:
  246. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  247. dye_flag = True
  248. dye_type = list_dye[i+1][0]
  249. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  250. dye_flag = True
  251. dye_type = list_dye[i-1][0]
  252. if dye_flag:
  253. for h in range(begin,end):
  254. inner_table[index][h][1] = dye_type
  255. dye_set.add((inner_table[index][h][0],dye_type))
  256. key_set.add(inner_table[index][h][0])
  257. begin = end
  258. #print("E",inner_table[index])
  259. def otherrepair(inner_table,index,dye_set,key_set):
  260. list_provide_repair = []
  261. if index==0 and len(inner_table)>1:
  262. list_provide_repair.append(index+1)
  263. elif index==len(inner_table)-1:
  264. list_provide_repair.append(index-1)
  265. else:
  266. list_provide_repair.append(index+1)
  267. list_provide_repair.append(index-1)
  268. for provide_index in list_provide_repair:
  269. if not repairNeeded(inner_table[provide_index]):
  270. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  271. if same_prob>=0.8:
  272. for i in range(len(inner_table[provide_index])):
  273. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  274. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  275. key_set.add(inner_table[index][i][0])
  276. inner_table[index][i][1] = inner_table[provide_index][i][1]
  277. elif same_prob<=0.2:
  278. for i in range(len(inner_table[provide_index])):
  279. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  280. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  281. key_set.add(inner_table[index][i][0])
  282. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  283. len_dye_set = len(dye_set)
  284. height = len(inner_table)
  285. for i in range(height):
  286. if repairNeeded(inner_table[i]):
  287. selfrepair(inner_table,i,dye_set,key_set)
  288. #otherrepair(inner_table,i,dye_set,key_set)
  289. for h in range(len(inner_table)):
  290. for w in range(len(inner_table[0])):
  291. if inner_table[h][w][0] in key_set:
  292. for item in dye_set:
  293. if inner_table[h][w][0]==item[0]:
  294. inner_table[h][w][1] = item[1]
  295. #如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  296. if len(dye_set)!=len(key_set):
  297. for i in range(height):
  298. if repairNeeded(inner_table[i]):
  299. selfrepair(inner_table,i,dye_set,key_set)
  300. #otherrepair(inner_table,i,dye_set,key_set)
  301. return
  302. if len(dye_set)==len_dye_set:
  303. '''
  304. for i in range(height):
  305. if repairNeeded(inner_table[i]):
  306. otherrepair(inner_table,i,dye_set,key_set)
  307. '''
  308. return
  309. repairTable(inner_table, dye_set, key_set)
  310. def sliceTable(inner_table,fix_value="~~"):
  311. #进行分块
  312. height = len(inner_table)
  313. width = len(inner_table[0])
  314. head_list = []
  315. head_list.append(0)
  316. last_head = None
  317. last_is_same_value = False
  318. for h in range(height):
  319. is_all_key = True#是否是全表头行
  320. is_all_value = True#是否是全属性值
  321. is_same_with_lastHead = True#和上一行的结构是否相同
  322. is_same_value=True#一行的item都一样
  323. #is_same_first_item = True#与上一行的第一项是否相同
  324. same_value = inner_table[h][0][0]
  325. for w in range(width):
  326. if last_head is not None:
  327. if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
  328. is_all_key = False
  329. if inner_table[h][w][0]==1:
  330. is_all_value = False
  331. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  332. is_same_with_lastHead = False
  333. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  334. is_same_value = False
  335. else:
  336. if re.search("\d+",same_value) is not None:
  337. is_same_value = False
  338. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  339. is_same_first_item = False
  340. last_head = h
  341. if last_is_same_value:
  342. last_is_same_value = is_same_value
  343. continue
  344. if is_same_value:
  345. head_list.append(h)
  346. last_is_same_value = is_same_value
  347. continue
  348. if not is_all_key:
  349. if not is_same_with_lastHead:
  350. head_list.append(h)
  351. head_list.append(height)
  352. return head_list
  353. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  354. set_item = set()
  355. height = len(inner_table)
  356. width = len(inner_table[0])
  357. empty_set = set()
  358. for i in range(height):
  359. for j in range(width):
  360. item = inner_table[i][j][0]
  361. if item.strip()=="":
  362. empty_set.add(item)
  363. else:
  364. set_item.add(item)
  365. list_item = list(set_item)
  366. if list_item:
  367. x = []
  368. for item in list_item:
  369. x.append(getPredictor("form").encode(item))
  370. predict_y = getPredictor("form").predict(np.array(x),type="item")
  371. _dict = dict()
  372. for item,values in zip(list_item,list(predict_y)):
  373. _dict[item] = values[1]
  374. # print("##",item,values)
  375. #print(_dict)
  376. for i in range(height):
  377. for j in range(width):
  378. item = inner_table[i][j][0]
  379. if item not in empty_set:
  380. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  381. # print("=====")
  382. # for item in inner_table:
  383. # print(item)
  384. # print("======")
  385. repairTable(inner_table)
  386. head_list = sliceTable(inner_table)
  387. return inner_table,head_list
  388. def set_head_model(inner_table):
  389. for i in range(len(inner_table)):
  390. for j in range(len(inner_table[i])):
  391. # 删掉单格前后符号,以免影响表头预测
  392. col = inner_table[i][j][0]
  393. col = re.sub("^[^\u4e00-\u9fa5a-zA-Z0-9]+", "", col)
  394. col = re.sub("[^\u4e00-\u9fa5a-zA-Z0-9]+$", "", col)
  395. inner_table[i][j] = col
  396. # 模型预测表头
  397. predict_list = predict(inner_table)
  398. with open(r"C:\Users\Administrator\Desktop\table_head_test.txt", "a") as f:
  399. for i in range(len(predict_list)):
  400. f.write(str(i) + " " + str(inner_table[i]) + "\n")
  401. f.write(str(i) + " " + str(predict_list[i]) + "\n")
  402. f.write("\n")
  403. # print("table_list", inner_table)
  404. # print("predict_list", predict_list)
  405. for i in range(len(inner_table)):
  406. for j in range(len(inner_table[i])):
  407. inner_table[i][j] = [inner_table[i][j], int(predict_list[i][j])]
  408. head_list = sliceTable(inner_table)
  409. return inner_table, head_list
  410. def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  411. data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
  412. predict_y = getPredictor("form").getModel("context").predict(data_x)
  413. for _position,_y in zip(data_position,predict_y):
  414. _w = _position[0]
  415. _h = _position[1]
  416. if _y[1]>prob_min:
  417. inner_table[_h][_w][1] = 1
  418. else:
  419. inner_table[_h][_w][1] = 0
  420. _item = inner_table[_h][_w][0]
  421. if re.search(pat_head,_item) is not None and len(_item)<8:
  422. inner_table[_h][_w][1] = 1
  423. # print("=====")
  424. # for item in inner_table:
  425. # print(item)
  426. # print("======")
  427. height = len(inner_table)
  428. width = len(inner_table[0])
  429. for i in range(height):
  430. for j in range(width):
  431. if re.search("[::]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
  432. inner_table[i][j][1] = 1
  433. repairTable(inner_table)
  434. head_list = sliceTable(inner_table)
  435. # print("inner_table:",inner_table)
  436. return inner_table,head_list
  437. #设置表头
  438. def setHead_inline(inner_table,prob_min=0.64):
  439. pad_row = "@@"
  440. pad_col = "##"
  441. removePadding(inner_table, pad_row, pad_col)
  442. pad_pattern = re.compile(pad_row+"|"+pad_col)
  443. height = len(inner_table)
  444. width = len(inner_table[0])
  445. head_list = []
  446. head_list.append(0)
  447. #行表头
  448. is_head_last = False
  449. for i in range(height):
  450. is_head = False
  451. is_long_value = False
  452. #判断是否是全padding值
  453. is_same_value = True
  454. same_value = inner_table[i][0][0]
  455. for j in range(width):
  456. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  457. is_same_value = False
  458. break
  459. #predict is head or not with model
  460. temp_item = ""
  461. for j in range(width):
  462. temp_item += inner_table[i][j][0]+"|"
  463. temp_item = re.sub(pad_pattern,"",temp_item)
  464. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  465. if form_prob is not None:
  466. if form_prob[0][1]>prob_min:
  467. is_head = True
  468. else:
  469. is_head = False
  470. #print(temp_item,form_prob)
  471. if len(inner_table[i][0][0])>40:
  472. is_long_value = True
  473. if is_head or is_long_value or is_same_value:
  474. #不把连续表头分开
  475. if not is_head_last:
  476. head_list.append(i)
  477. if is_long_value or is_same_value:
  478. head_list.append(i+1)
  479. if is_head:
  480. for j in range(width):
  481. inner_table[i][j][1] = 1
  482. is_head_last = is_head
  483. head_list.append(height)
  484. #列表头
  485. for i in range(len(head_list)-1):
  486. head_begin = head_list[i]
  487. head_end = head_list[i+1]
  488. #最后一列不设置为列表头
  489. for i in range(width-1):
  490. is_head = False
  491. #predict is head or not with model
  492. temp_item = ""
  493. for j in range(head_begin,head_end):
  494. temp_item += inner_table[j][i][0]+"|"
  495. temp_item = re.sub(pad_pattern,"",temp_item)
  496. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  497. if form_prob is not None:
  498. if form_prob[0][1]>prob_min:
  499. is_head = True
  500. else:
  501. is_head = False
  502. if is_head:
  503. for j in range(head_begin,head_end):
  504. inner_table[j][i][1] = 2
  505. addPadding(inner_table, pad_row, pad_col)
  506. return inner_table,head_list
  507. #设置表头
  508. def setHead_withRule(inner_table,pattern,pat_value,count):
  509. height = len(inner_table)
  510. width = len(inner_table[0])
  511. head_list = []
  512. head_list.append(0)
  513. #行表头
  514. is_head_last = False
  515. for i in range(height):
  516. set_match = set()
  517. is_head = False
  518. is_long_value = False
  519. is_same_value = True
  520. same_value = inner_table[i][0][0]
  521. for j in range(width):
  522. if inner_table[i][j][0]!=same_value:
  523. is_same_value = False
  524. break
  525. for j in range(width):
  526. if re.search(pat_value,inner_table[i][j][0]) is not None:
  527. is_head = False
  528. break
  529. str_find = re.findall(pattern,inner_table[i][j][0])
  530. if len(str_find)>0:
  531. set_match.add(inner_table[i][j][0])
  532. if len(set_match)>=count:
  533. is_head = True
  534. if len(inner_table[i][0][0])>40:
  535. is_long_value = True
  536. if is_head or is_long_value or is_same_value:
  537. if not is_head_last:
  538. head_list.append(i)
  539. if is_head:
  540. for j in range(width):
  541. inner_table[i][j][1] = 1
  542. is_head_last = is_head
  543. head_list.append(height)
  544. #列表头
  545. for i in range(len(head_list)-1):
  546. head_begin = head_list[i]
  547. head_end = head_list[i+1]
  548. #最后一列不设置为列表头
  549. for i in range(width-1):
  550. set_match = set()
  551. is_head = False
  552. for j in range(head_begin,head_end):
  553. if re.search(pat_value,inner_table[j][i][0]) is not None:
  554. is_head = False
  555. break
  556. str_find = re.findall(pattern,inner_table[j][i][0])
  557. if len(str_find)>0:
  558. set_match.add(inner_table[j][i][0])
  559. if len(set_match)>=count:
  560. is_head = True
  561. if is_head:
  562. for j in range(head_begin,head_end):
  563. inner_table[j][i][1] = 2
  564. return inner_table,head_list
  565. #取得表格的处理方向
  566. def getDirect(inner_table,begin,end):
  567. '''
  568. column_head = set()
  569. row_head = set()
  570. widths = len(inner_table[0])
  571. for height in range(begin,end):
  572. for width in range(widths):
  573. if inner_table[height][width][1] ==1:
  574. row_head.add(height)
  575. if inner_table[height][width][1] ==2:
  576. column_head.add(width)
  577. company_pattern = re.compile("公司")
  578. if 0 in column_head and begin not in row_head:
  579. return "column"
  580. if 0 in column_head and begin in row_head:
  581. for height in range(begin,end):
  582. count = 0
  583. count_flag = True
  584. for width_index in range(width):
  585. if inner_table[height][width_index][1]==0:
  586. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  587. count += 1
  588. else:
  589. count_flag = False
  590. if count_flag and count>=2:
  591. return "column"
  592. return "row"
  593. '''
  594. count_row_keys = 0
  595. count_column_keys = 0
  596. width = len(inner_table[0])
  597. if begin<end:
  598. for w in range(len(inner_table[begin])):
  599. if inner_table[begin][w][1]!=0:
  600. count_row_keys += 1
  601. for h in range(begin,end):
  602. if inner_table[h][0][1]!=0:
  603. count_column_keys += 1
  604. company_pattern = re.compile("有限(责任)?公司")
  605. for height in range(begin,end):
  606. count_set = set()
  607. count_flag = True
  608. for width_index in range(width):
  609. if inner_table[height][width_index][1]==0:
  610. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  611. count_set.add(inner_table[height][width_index][0])
  612. else:
  613. count_flag = False
  614. if count_flag and len(count_set)>=2:
  615. return "column"
  616. # if count_column_keys>count_row_keys: #2022/2/15 此项不够严谨,造成很多错误,故取消
  617. # return "column"
  618. return "row"
  619. #根据表格处理方向生成句子,
  620. def getTableText(inner_table,head_list,key_direct=False):
  621. # packPattern = "(标包|[标包][号段名])"
  622. packPattern = "(标包|标的|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  623. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)" # 2020/11/23 大网站规则,添加序号为排序
  624. entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))"
  625. moneyPattern = "([中投]标|报价)(金额|价)"
  626. height = len(inner_table)
  627. width = len(inner_table[0])
  628. text = ""
  629. for head_i in range(len(head_list)-1):
  630. head_begin = head_list[head_i]
  631. head_end = head_list[head_i+1]
  632. direct = getDirect(inner_table, head_begin, head_end)
  633. #若只有一行,则直接按行读取
  634. if head_end-head_begin==1:
  635. text_line = ""
  636. for i in range(head_begin,head_end):
  637. for w in range(len(inner_table[i])):
  638. if inner_table[i][w][1]==1:
  639. _punctuation = ":"
  640. else:
  641. _punctuation = "," #2021/12/15 统一为中文标点,避免 206893924 国际F座1108,1,009,197.49元
  642. if w>0:
  643. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  644. text_line += inner_table[i][w][0]+_punctuation
  645. else:
  646. text_line += inner_table[i][w][0]+_punctuation
  647. text_line = text_line+"。" if text_line!="" else text_line
  648. text += text_line
  649. else:
  650. #构建一个共现矩阵
  651. table_occurence = []
  652. for i in range(head_begin,head_end):
  653. line_oc = []
  654. for j in range(width):
  655. cell = inner_table[i][j]
  656. line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
  657. table_occurence.append(line_oc)
  658. occu_height = len(table_occurence)
  659. occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
  660. #为每个属性值寻找表头
  661. for i in range(occu_height):
  662. for j in range(occu_width):
  663. cell = table_occurence[i][j]
  664. #是属性值
  665. if cell["type"]==0 and cell["text"]!="":
  666. left_head = ""
  667. top_head = ""
  668. find_flag = False
  669. temp_head = ""
  670. for loop_i in range(1,i+1):
  671. if not key_direct:
  672. key_values = [1,2]
  673. else:
  674. key_values = [1]
  675. if table_occurence[i-loop_i][j]["type"] in key_values:
  676. if find_flag:
  677. if table_occurence[i-loop_i][j]["text"]!=temp_head:
  678. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  679. else:
  680. top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
  681. find_flag = True
  682. temp_head = table_occurence[i-loop_i][j]["text"]
  683. table_occurence[i-loop_i][j]["occu_count"] += 1
  684. else:
  685. #找到表头后遇到属性值就返回
  686. if find_flag:
  687. break
  688. cell["top_head"] += top_head
  689. find_flag = False
  690. temp_head = ""
  691. for loop_j in range(1,j+1):
  692. if not key_direct:
  693. key_values = [1,2]
  694. else:
  695. key_values = [2]
  696. if table_occurence[i][j-loop_j]["type"] in key_values:
  697. if find_flag:
  698. if table_occurence[i][j-loop_j]["text"]!=temp_head:
  699. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  700. else:
  701. left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
  702. find_flag = True
  703. temp_head = table_occurence[i][j-loop_j]["text"]
  704. table_occurence[i][j-loop_j]["occu_count"] += 1
  705. else:
  706. if find_flag:
  707. break
  708. cell["left_head"] += left_head
  709. if direct=="row":
  710. for i in range(occu_height):
  711. pack_text = ""
  712. rank_text = ""
  713. entity_text = ""
  714. text_line = ""
  715. money_text = ""
  716. #在同一句话中重复的可以去掉
  717. text_set = set()
  718. for j in range(width):
  719. cell = table_occurence[i][j]
  720. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  721. cell = table_occurence[i][j]
  722. head = (cell["top_head"]+":") if len(cell["top_head"])>0 else ""
  723. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  724. head = cell["left_head"] + head
  725. else:
  726. head += cell["left_head"]
  727. if str(head+cell["text"]) in text_set:
  728. continue
  729. if re.search(packPattern,head) is not None:
  730. pack_text += head+cell["text"]+","
  731. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  732. #排名替换为同一种表达
  733. rank_text += head+cell["text"]+","
  734. #print(rank_text)
  735. elif re.search(entityPattern,head) is not None:
  736. entity_text += head+cell["text"]+","
  737. #print(entity_text)
  738. else:
  739. if re.search(moneyPattern,head) is not None and entity_text!="":
  740. money_text += head+cell["text"]+","
  741. else:
  742. text_line += head+cell["text"]+","
  743. text_set.add(str(head+cell["text"]))
  744. text += pack_text+rank_text+entity_text+money_text+text_line
  745. text = text[:-1]+"。" if len(text)>0 else text
  746. else:
  747. for j in range(occu_width):
  748. pack_text = ""
  749. rank_text = ""
  750. entity_text = ""
  751. text_line = ""
  752. text_set = set()
  753. for i in range(occu_height):
  754. cell = table_occurence[i][j]
  755. if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):
  756. cell = table_occurence[i][j]
  757. head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
  758. if re.search("单报标限总]价|金额|成交报?价|报价", head):
  759. head = cell["top_head"] + head
  760. else:
  761. head += cell["top_head"]
  762. if str(head+cell["text"]) in text_set:
  763. continue
  764. if re.search(packPattern,head) is not None:
  765. pack_text += head+cell["text"]+","
  766. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  767. #排名替换为同一种表达
  768. rank_text += head+cell["text"]+","
  769. #print(rank_text)
  770. elif re.search(entityPattern,head) is not None and \
  771. re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
  772. entity_text += head+cell["text"]+","
  773. #print(entity_text)
  774. else:
  775. text_line += head+cell["text"]+","
  776. text_set.add(str(head+cell["text"]))
  777. text += pack_text+rank_text+entity_text+text_line
  778. text = text[:-1]+"。" if len(text)>0 else text
  779. # if direct=="row":
  780. # for i in range(head_begin,head_end):
  781. # pack_text = ""
  782. # rank_text = ""
  783. # entity_text = ""
  784. # text_line = ""
  785. # #在同一句话中重复的可以去掉
  786. # text_set = set()
  787. # for j in range(width):
  788. # cell = inner_table[i][j]
  789. # #是属性值
  790. # if cell[1]==0 and cell[0]!="":
  791. # head = ""
  792. #
  793. # find_flag = False
  794. # temp_head = ""
  795. # for loop_i in range(0,i+1-head_begin):
  796. # if not key_direct:
  797. # key_values = [1,2]
  798. # else:
  799. # key_values = [1]
  800. # if inner_table[i-loop_i][j][1] in key_values:
  801. # if find_flag:
  802. # if inner_table[i-loop_i][j][0]!=temp_head:
  803. # head = inner_table[i-loop_i][j][0]+":"+head
  804. # else:
  805. # head = inner_table[i-loop_i][j][0]+":"+head
  806. # find_flag = True
  807. # temp_head = inner_table[i-loop_i][j][0]
  808. # else:
  809. # #找到表头后遇到属性值就返回
  810. # if find_flag:
  811. # break
  812. #
  813. # find_flag = False
  814. # temp_head = ""
  815. #
  816. #
  817. #
  818. # for loop_j in range(1,j+1):
  819. # if not key_direct:
  820. # key_values = [1,2]
  821. # else:
  822. # key_values = [2]
  823. # if inner_table[i][j-loop_j][1] in key_values:
  824. # if find_flag:
  825. # if inner_table[i][j-loop_j][0]!=temp_head:
  826. # head = inner_table[i][j-loop_j][0]+":"+head
  827. # else:
  828. # head = inner_table[i][j-loop_j][0]+":"+head
  829. # find_flag = True
  830. # temp_head = inner_table[i][j-loop_j][0]
  831. # else:
  832. # if find_flag:
  833. # break
  834. #
  835. # if str(head+inner_table[i][j][0]) in text_set:
  836. # continue
  837. # if re.search(packPattern,head) is not None:
  838. # pack_text += head+inner_table[i][j][0]+","
  839. # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  840. # #排名替换为同一种表达
  841. # rank_text += head+inner_table[i][j][0]+","
  842. # #print(rank_text)
  843. # elif re.search(entityPattern,head) is not None:
  844. # entity_text += head+inner_table[i][j][0]+","
  845. # #print(entity_text)
  846. # else:
  847. # text_line += head+inner_table[i][j][0]+","
  848. # text_set.add(str(head+inner_table[i][j][0]))
  849. # text += pack_text+rank_text+entity_text+text_line
  850. # text = text[:-1]+"。" if len(text)>0 else text
  851. # else:
  852. # for j in range(width):
  853. #
  854. # rank_text = ""
  855. # entity_text = ""
  856. # text_line = ""
  857. # text_set = set()
  858. # for i in range(head_begin,head_end):
  859. # cell = inner_table[i][j]
  860. # #是属性值
  861. # if cell[1]==0 and cell[0]!="":
  862. # find_flag = False
  863. # head = ""
  864. # temp_head = ""
  865. #
  866. # for loop_j in range(1,j+1):
  867. # if not key_direct:
  868. # key_values = [1,2]
  869. # else:
  870. # key_values = [2]
  871. # if inner_table[i][j-loop_j][1] in key_values:
  872. # if find_flag:
  873. # if inner_table[i][j-loop_j][0]!=temp_head:
  874. # head = inner_table[i][j-loop_j][0]+":"+head
  875. # else:
  876. # head = inner_table[i][j-loop_j][0]+":"+head
  877. # find_flag = True
  878. # temp_head = inner_table[i][j-loop_j][0]
  879. # else:
  880. # if find_flag:
  881. # break
  882. # find_flag = False
  883. # temp_head = ""
  884. # for loop_i in range(0,i+1-head_begin):
  885. # if not key_direct:
  886. # key_values = [1,2]
  887. # else:
  888. # key_values = [1]
  889. # if inner_table[i-loop_i][j][1] in key_values:
  890. # if find_flag:
  891. # if inner_table[i-loop_i][j][0]!=temp_head:
  892. # head = inner_table[i-loop_i][j][0]+":"+head
  893. # else:
  894. # head = inner_table[i-loop_i][j][0]+":"+head
  895. # find_flag = True
  896. # temp_head = inner_table[i-loop_i][j][0]
  897. # else:
  898. # if find_flag:
  899. # break
  900. # if str(head+inner_table[i][j][0]) in text_set:
  901. # continue
  902. # if re.search(rankPattern,head) is not None:
  903. # rank_text += head+inner_table[i][j][0]+","
  904. # #print(rank_text)
  905. # elif re.search(entityPattern,head) is not None:
  906. # entity_text += head+inner_table[i][j][0]+","
  907. # #print(entity_text)
  908. # else:
  909. # text_line += head+inner_table[i][j][0]+","
  910. # text_set.add(str(head+inner_table[i][j][0]))
  911. # text += rank_text+entity_text+text_line
  912. # text = text[:-1]+"。" if len(text)>0 else text
  913. return text
  914. def removeFix(inner_table,fix_value="~~"):
  915. height = len(inner_table)
  916. width = len(inner_table[0])
  917. for h in range(height):
  918. for w in range(width):
  919. if inner_table[h][w][0]==fix_value:
  920. inner_table[h][w][0] = ""
  921. def trunTable(tbody,in_attachment):
  922. # print("tbody", tbody.find('tbody'))
  923. # 附件中的表格,排除异常错乱的表格
  924. if in_attachment:
  925. if tbody.name=='table':
  926. _tbody = tbody.find('tbody')
  927. if _tbody is None:
  928. _tbody = tbody
  929. else:
  930. _tbody = tbody
  931. _td_len_list = []
  932. for _tr in _tbody.find_all(recursive=False):
  933. len_td = len(_tr.find_all(recursive=False))
  934. _td_len_list.append(len_td)
  935. if len(list(set(_td_len_list)))>8:
  936. return None
  937. fixSpan(tbody)
  938. inner_table = getTable(tbody)
  939. inner_table = fixTable(inner_table)
  940. if len(inner_table)>0 and len(inner_table[0])>0:
  941. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  942. #inner_table,head_list = setHead_inline(inner_table)
  943. # inner_table, head_list = setHead_initem(inner_table,pat_head)
  944. inner_table, head_list = set_head_model(inner_table)
  945. # inner_table,head_list = setHead_incontext(inner_table,pat_head)
  946. # print("table_head", inner_table)
  947. # for begin in range(len(head_list[:-1])):
  948. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  949. # print(item)
  950. # print("====")
  951. removeFix(inner_table)
  952. # print("----")
  953. # print(head_list)
  954. # for item in inner_table:
  955. # print(item)
  956. tbody.string = getTableText(inner_table,head_list)
  957. #print(tbody.string)
  958. tbody.name = "turntable"
  959. return inner_table
  960. return None
  961. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标|(需求|服务|项目|施工|采购|招租|出租|转让|出让|业主|询价|委托|权属|招标|竞得|抽取|承建)(人|方|单位)(名称)?|(供应商|供货商|服务商)(名称)?)$')
  962. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  963. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  964. list_innerTable = []
  965. # 2022/2/9 删除干扰标签
  966. for tag in soup.find_all('option'): #例子: 216661412
  967. if 'selected' not in tag.attrs:
  968. tag.extract()
  969. for ul in soup.find_all('ul'): #例子 156439663 多个不同channel 类别的标题
  970. if ul.find_all('li') == ul.findChildren(recursive=False) and len(set(re.findall(
  971. '招标公告|中标结果公示|中标候选人公示|招标答疑|开标评标|合同履?约?公示|开标评标|资格评审',
  972. ul.get_text(), re.S)))>3:
  973. ul.extract()
  974. # tbodies = soup.find_all('table')
  975. # 遍历表格中的每个tbody
  976. tbodies = []
  977. in_attachment = False
  978. for _part in soup.find_all():
  979. if _part.name=='table':
  980. tbodies.append((_part,in_attachment))
  981. elif _part.name=='div':
  982. if 'class' in _part.attrs and "richTextFetch" in _part['class']:
  983. in_attachment = True
  984. #逆序处理嵌套表格
  985. for tbody_index in range(1,len(tbodies)+1):
  986. tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
  987. inner_table = trunTable(tbody,_in_attachment)
  988. list_innerTable.append(inner_table)
  989. # tbodies = soup.find_all('tbody')
  990. # 遍历表格中的每个tbody
  991. tbodies = []
  992. in_attachment = False
  993. for _part in soup.find_all():
  994. if _part.name == 'tbody':
  995. tbodies.append((_part, in_attachment))
  996. elif _part.name == 'div':
  997. if 'class' in _part.attrs and "richTextFetch" in _part['class']:
  998. in_attachment = True
  999. #逆序处理嵌套表格
  1000. for tbody_index in range(1,len(tbodies)+1):
  1001. tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
  1002. inner_table = trunTable(tbody,_in_attachment)
  1003. list_innerTable.append(inner_table)
  1004. return soup
  1005. # return list_innerTable
  1006. re_num = re.compile("[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十]")
  1007. num_dict = {
  1008. "一": 1, "二": 2,
  1009. "三": 3, "四": 4,
  1010. "五": 5, "六": 6,
  1011. "七": 7, "八": 8,
  1012. "九": 9, "十": 10}
  1013. # 一百以内的中文大写转换为数字
  1014. def change2num(text):
  1015. result_num = -1
  1016. # text = text[:6]
  1017. match = re_num.search(text)
  1018. if match:
  1019. _num = match.group()
  1020. if num_dict.get(_num):
  1021. return num_dict.get(_num)
  1022. else:
  1023. tenths = 1
  1024. the_unit = 0
  1025. num_split = _num.split("十")
  1026. if num_dict.get(num_split[0]):
  1027. tenths = num_dict.get(num_split[0])
  1028. if num_dict.get(num_split[1]):
  1029. the_unit = num_dict.get(num_split[1])
  1030. result_num = tenths * 10 + the_unit
  1031. elif re.search("\d{1,2}",text):
  1032. _num = re.search("\d{1,2}",text).group()
  1033. result_num = int(_num)
  1034. return result_num
  1035. #大纲分段处理
  1036. def get_preprocessed_outline(soup):
  1037. pattern_0 = re.compile("^(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[、.\.]")
  1038. pattern_1 = re.compile("^[\((]?(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[\))]")
  1039. pattern_2 = re.compile("^\d{1,2}[、.\.](?=[^\d]{1,2}|$)")
  1040. pattern_3 = re.compile("^[\((]?\d{1,2}[\))]")
  1041. pattern_list = [pattern_0, pattern_1, pattern_2, pattern_3]
  1042. body = soup.find("body")
  1043. body_child = body.find_all(recursive=False)
  1044. deal_part = body
  1045. # print(body_child[0]['id'])
  1046. if 'id' in body_child[0].attrs:
  1047. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1048. deal_part = body_child[0]
  1049. if len(deal_part.find_all(recursive=False))>2:
  1050. deal_part = deal_part.parent
  1051. skip_tag = ['turntable', 'tbody', 'th', 'tr', 'td', 'table','thead','tfoot']
  1052. for part in deal_part.find_all(recursive=False):
  1053. # 查找解析文本的主干部分
  1054. is_main_text = False
  1055. through_text_num = 0
  1056. while (not is_main_text and part.find_all(recursive=False)):
  1057. while len(part.find_all(recursive=False)) == 1 and part.get_text(strip=True) == \
  1058. part.find_all(recursive=False)[0].get_text(strip=True):
  1059. part = part.find_all(recursive=False)[0]
  1060. max_len = len(part.get_text(strip=True))
  1061. is_main_text = True
  1062. for t_part in part.find_all(recursive=False):
  1063. if t_part.name not in skip_tag and t_part.get_text(strip=True)!="":
  1064. through_text_num += 1
  1065. if t_part.get_text(strip=True)!="" and len(t_part.get_text(strip=True))/max_len>=0.65:
  1066. if t_part.name not in skip_tag:
  1067. is_main_text = False
  1068. part = t_part
  1069. break
  1070. else:
  1071. while len(t_part.find_all(recursive=False)) == 1 and t_part.get_text(strip=True) == \
  1072. t_part.find_all(recursive=False)[0].get_text(strip=True):
  1073. t_part = t_part.find_all(recursive=False)[0]
  1074. if through_text_num>2:
  1075. is_table = True
  1076. for _t_part in t_part.find_all(recursive=False):
  1077. if _t_part.name not in skip_tag:
  1078. is_table = False
  1079. break
  1080. if not is_table:
  1081. is_main_text = False
  1082. part = t_part
  1083. break
  1084. else:
  1085. is_main_text = False
  1086. part = t_part
  1087. break
  1088. is_find = False
  1089. for _pattern in pattern_list:
  1090. last_index = 0
  1091. handle_list = []
  1092. for _part in part.find_all(recursive=False):
  1093. if _part.name not in skip_tag and _part.get_text(strip=True) != "":
  1094. # print('text:', _part.get_text(strip=True))
  1095. re_match = re.search(_pattern, _part.get_text(strip=True))
  1096. if re_match:
  1097. outline_index = change2num(re_match.group())
  1098. if last_index < outline_index:
  1099. # _part.insert_before("##split##")
  1100. handle_list.append(_part)
  1101. last_index = outline_index
  1102. if len(handle_list)>1:
  1103. is_find = True
  1104. for _part in handle_list:
  1105. _part.insert_before("##split##")
  1106. if is_find:
  1107. break
  1108. # print(soup)
  1109. return soup
  1110. #数据清洗
  1111. def segment(soup,final=True):
  1112. # print("==")
  1113. # print(soup)
  1114. # print("====")
  1115. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  1116. subspaceList = ["td",'a',"span","p"]
  1117. if soup.name in subspaceList:
  1118. #判断有值叶子节点数
  1119. _count = 0
  1120. for child in soup.find_all(recursive=True):
  1121. if child.get_text().strip()!="" and len(child.find_all())==0:
  1122. _count += 1
  1123. if _count<=1:
  1124. text = soup.get_text()
  1125. # 2020/11/24 大网站规则添加
  1126. if 'title' in soup.attrs:
  1127. if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']:
  1128. text = soup.attrs['title']
  1129. _list = []
  1130. for x in re.split("\s+",text):
  1131. if x.strip()!="":
  1132. _list.append(len(x))
  1133. if len(_list)>0:
  1134. _minLength = min(_list)
  1135. if _minLength>2:
  1136. _substr = ","
  1137. else:
  1138. _substr = ""
  1139. else:
  1140. _substr = ""
  1141. text = text.replace("\r\n",",").replace("\n",",")
  1142. text = re.sub("\s+",_substr,text)
  1143. # text = re.sub("\s+","##space##",text)
  1144. return text
  1145. segList = ["title"]
  1146. commaList = ["div","br","td","p","li"]
  1147. #commaList = []
  1148. spaceList = ["span"]
  1149. tbodies = soup.find_all('tbody')
  1150. if len(tbodies) == 0:
  1151. tbodies = soup.find_all('table')
  1152. # 递归遍历所有节点,插入符号
  1153. for child in soup.find_all(recursive=True):
  1154. # print(child.name,child.get_text())
  1155. if child.name in segList:
  1156. child.insert_after("。")
  1157. if child.name in commaList:
  1158. child.insert_after(",")
  1159. # if child.name == 'div' and 'class' in child.attrs:
  1160. # # 添加附件"attachment"标识
  1161. # if "richTextFetch" in child['class']:
  1162. # child.insert_before("##attachment##")
  1163. # print(child.parent)
  1164. # if child.name in subspaceList:
  1165. # child.insert_before("#subs"+str(child.name)+"#")
  1166. # child.insert_after("#sube"+str(child.name)+"#")
  1167. # if child.name in spaceList:
  1168. # child.insert_after(" ")
  1169. text = str(soup.get_text())
  1170. #替换英文冒号为中文冒号
  1171. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1172. #替换为中文逗号
  1173. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1174. #替换为中文分号
  1175. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1176. # 感叹号替换为中文句号
  1177. text = re.sub("(?<=[\u4e00-\u9fa5])[!!]|[!!](?=[\u4e00-\u9fa5])","。",text)
  1178. #替换格式未识别的问号为" " ,update:2021/7/20
  1179. text = re.sub("[?\?]{2,}"," ",text)
  1180. #替换"""为"“",否则导入deepdive出错
  1181. # text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1182. text = text.replace('"',"“").replace("\r","").replace("\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
  1183. # print('==1',text)
  1184. # text = re.sub("\s{4,}",",",text)
  1185. # 解决公告中的" "空格替换问题
  1186. if re.search("\s{4,}",text):
  1187. _text = ""
  1188. for _sent in re.split("。+",text):
  1189. for _sent2 in re.split(',+',_sent):
  1190. for _sent3 in re.split(":+",_sent2):
  1191. for _t in re.split("\s{4,}",_sent3):
  1192. if len(_t)<3:
  1193. _text += _t
  1194. else:
  1195. _text += ","+_t
  1196. _text += ":"
  1197. _text = _text[:-1]
  1198. _text += ","
  1199. _text = _text[:-1]
  1200. _text += "。"
  1201. _text = _text[:-1]
  1202. text = _text
  1203. # print('==2',text)
  1204. #替换标点
  1205. #替换连续的标点
  1206. if final:
  1207. text = re.sub("##space##"," ",text)
  1208. punc_pattern = "(?P<del>[。,;::,\s]+)"
  1209. list_punc = re.findall(punc_pattern,text)
  1210. list_punc.sort(key=lambda x:len(x),reverse=True)
  1211. for punc_del in list_punc:
  1212. if len(punc_del)>1:
  1213. if len(punc_del.strip())>0:
  1214. if ":" in punc_del.strip():
  1215. if "。" in punc_del.strip():
  1216. text = re.sub(punc_del, ":。", text)
  1217. else:
  1218. text = re.sub(punc_del,":",text)
  1219. else:
  1220. text = re.sub(punc_del,punc_del.strip()[0],text) #2021/12/09 修正由于某些标签后插入符号把原来符号替换
  1221. else:
  1222. text = re.sub(punc_del,"",text)
  1223. #将连续的中文句号替换为一个
  1224. text_split = text.split("。")
  1225. text_split = [x for x in text_split if len(x)>0]
  1226. text = "。".join(text_split)
  1227. # #删除标签中的所有空格
  1228. # for subs in subspaceList:
  1229. # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  1230. # while(True):
  1231. # oneMatch = re.search(re.compile(patten),text)
  1232. # if oneMatch is not None:
  1233. # _match = oneMatch.group(1)
  1234. # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
  1235. # else:
  1236. # break
  1237. # text过大报错
  1238. LOOP_LEN = 10000
  1239. LOOP_BEGIN = 0
  1240. _text = ""
  1241. if len(text)<10000000:
  1242. while(LOOP_BEGIN<len(text)):
  1243. _text += re.sub(")",")",re.sub("(","(",re.sub("\s+","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  1244. LOOP_BEGIN += LOOP_LEN
  1245. text = _text
  1246. # 附件标识前修改为句号,避免正文和附件内容混合在一起
  1247. text = re.sub("[^。](?=##attachment##)","。",text)
  1248. return text
  1249. '''
  1250. #数据清洗
  1251. def segment(soup):
  1252. segList = ["title"]
  1253. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  1254. spaceList = ["span"]
  1255. tbodies = soup.find_all('tbody')
  1256. if len(tbodies) == 0:
  1257. tbodies = soup.find_all('table')
  1258. # 递归遍历所有节点,插入符号
  1259. for child in soup.find_all(recursive=True):
  1260. if child.name == 'br':
  1261. child.insert_before(',')
  1262. child_text = re.sub('\s', '', child.get_text())
  1263. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  1264. continue
  1265. if child.name in segList:
  1266. child.insert_after("。")
  1267. if child.name in commaList:
  1268. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  1269. child.insert_after(",")
  1270. elif len(child_text) >=50:
  1271. child.insert_after("。")
  1272. #if child.name in spaceList:
  1273. #child.insert_after(" ")
  1274. text = str(soup.get_text())
  1275. text = re.sub("\s{5,}",",",text)
  1276. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  1277. #替换"""为"“",否则导入deepdive出错
  1278. text = text.replace('"',"“")
  1279. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  1280. #删除所有空格
  1281. text = re.sub("\s+","#nbsp#",text)
  1282. text_list = text.split('#nbsp#')
  1283. new_text = ''
  1284. for i in range(len(text_list)-1):
  1285. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  1286. new_text += text_list[i]
  1287. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  1288. new_text += text_list[i] + '。'
  1289. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  1290. new_text += text_list[i] + ';'
  1291. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  1292. new_text += text_list[i] + ' '
  1293. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  1294. new_text += text_list[i]
  1295. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  1296. new_text += text_list[i] + ','
  1297. else:
  1298. new_text += text_list[i]
  1299. new_text += text_list[-1]
  1300. text = new_text
  1301. #替换英文冒号为中文冒号
  1302. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  1303. #替换为中文逗号
  1304. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  1305. #替换为中文分号
  1306. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  1307. #替换标点
  1308. while(True):
  1309. #替换连续的标点
  1310. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  1311. if punc is not None:
  1312. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  1313. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  1314. if punc is not None:
  1315. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  1316. else:
  1317. #替换标点之后的空格
  1318. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  1319. if punc is not None:
  1320. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  1321. else:
  1322. break
  1323. #将连续的中文句号替换为一个
  1324. text_split = text.split("。")
  1325. text_split = [x for x in text_split if len(x)>0]
  1326. text = "。".join(text_split)
  1327. #替换中文括号为英文括号
  1328. text = re.sub("(","(",text)
  1329. text = re.sub(")",")",text)
  1330. return text
  1331. '''
  1332. #连续实体合并(弃用)
  1333. def union_ner(list_ner):
  1334. result_list = []
  1335. union_index = []
  1336. union_index_set = set()
  1337. for i in range(len(list_ner)-1):
  1338. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  1339. if list_ner[i][1]-list_ner[i+1][0]==1:
  1340. union_index_set.add(i)
  1341. union_index_set.add(i+1)
  1342. union_index.append((i,i+1))
  1343. for i in range(len(list_ner)):
  1344. if i not in union_index_set:
  1345. result_list.append(list_ner[i])
  1346. for item in union_index:
  1347. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  1348. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  1349. return result_list
  1350. # def get_preprocessed(articles,useselffool=False):
  1351. # '''
  1352. # @summary:预处理步骤,NLP处理、实体识别
  1353. # @param:
  1354. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1355. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1356. # '''
  1357. # list_articles = []
  1358. # list_sentences = []
  1359. # list_entitys = []
  1360. # cost_time = dict()
  1361. # for article in articles:
  1362. # list_sentences_temp = []
  1363. # list_entitys_temp = []
  1364. # doc_id = article[0]
  1365. # sourceContent = article[1]
  1366. # _send_doc_id = article[3]
  1367. # _title = article[4]
  1368. # #表格处理
  1369. # key_preprocess = "tableToText"
  1370. # start_time = time.time()
  1371. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1372. #
  1373. # # log(article_processed)
  1374. #
  1375. # if key_preprocess not in cost_time:
  1376. # cost_time[key_preprocess] = 0
  1377. # cost_time[key_preprocess] += time.time()-start_time
  1378. #
  1379. # #article_processed = article[1]
  1380. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1381. # #nlp处理
  1382. # if article_processed is not None and len(article_processed)!=0:
  1383. # split_patten = "。"
  1384. # sentences = []
  1385. # _begin = 0
  1386. # for _iter in re.finditer(split_patten,article_processed):
  1387. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1388. # _begin = _iter.span()[1]
  1389. # sentences.append(article_processed[_begin:])
  1390. #
  1391. # lemmas = []
  1392. # doc_offsets = []
  1393. # dep_types = []
  1394. # dep_tokens = []
  1395. #
  1396. # time1 = time.time()
  1397. #
  1398. # '''
  1399. # tokens_all = fool.cut(sentences)
  1400. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1401. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1402. # ner_entitys_all = fool.ner(sentences)
  1403. # '''
  1404. # #限流执行
  1405. # key_nerToken = "nerToken"
  1406. # start_time = time.time()
  1407. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1408. # if key_nerToken not in cost_time:
  1409. # cost_time[key_nerToken] = 0
  1410. # cost_time[key_nerToken] += time.time()-start_time
  1411. #
  1412. #
  1413. # for sentence_index in range(len(sentences)):
  1414. #
  1415. #
  1416. #
  1417. # list_sentence_entitys = []
  1418. # sentence_text = sentences[sentence_index]
  1419. # tokens = tokens_all[sentence_index]
  1420. #
  1421. # list_tokenbegin = []
  1422. # begin = 0
  1423. # for i in range(0,len(tokens)):
  1424. # list_tokenbegin.append(begin)
  1425. # begin += len(str(tokens[i]))
  1426. # list_tokenbegin.append(begin+1)
  1427. # #pos_tag = pos_all[sentence_index]
  1428. # pos_tag = ""
  1429. #
  1430. # ner_entitys = ner_entitys_all[sentence_index]
  1431. #
  1432. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1433. #
  1434. # #识别package
  1435. #
  1436. #
  1437. # #识别实体
  1438. # for ner_entity in ner_entitys:
  1439. # begin_index_temp = ner_entity[0]
  1440. # end_index_temp = ner_entity[1]
  1441. # entity_type = ner_entity[2]
  1442. # entity_text = ner_entity[3]
  1443. #
  1444. # for j in range(len(list_tokenbegin)):
  1445. # if list_tokenbegin[j]==begin_index_temp:
  1446. # begin_index = j
  1447. # break
  1448. # elif list_tokenbegin[j]>begin_index_temp:
  1449. # begin_index = j-1
  1450. # break
  1451. # begin_index_temp += len(str(entity_text))
  1452. # for j in range(begin_index,len(list_tokenbegin)):
  1453. # if list_tokenbegin[j]>=begin_index_temp:
  1454. # end_index = j-1
  1455. # break
  1456. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1457. #
  1458. # #去掉标点符号
  1459. # entity_text = re.sub("[,,。:]","",entity_text)
  1460. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1461. #
  1462. #
  1463. # #使用正则识别金额
  1464. # entity_type = "money"
  1465. #
  1466. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1467. #
  1468. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1469. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1470. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1471. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1472. #
  1473. # set_begin = set()
  1474. # for pattern_key in list_money_pattern.keys():
  1475. # pattern = re.compile(list_money_pattern[pattern_key])
  1476. # all_match = re.findall(pattern, sentence_text)
  1477. # index = 0
  1478. # for i in range(len(all_match)):
  1479. # if len(all_match[i][0])>0:
  1480. # # print("===",all_match[i])
  1481. # #print(all_match[i][0])
  1482. # unit = ""
  1483. # entity_text = all_match[i][3]
  1484. # if pattern_key in ["key_word","front_m"]:
  1485. # unit = all_match[i][1]
  1486. # else:
  1487. # unit = all_match[i][4]
  1488. # if entity_text.find("元")>=0:
  1489. # unit = ""
  1490. #
  1491. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1492. #
  1493. # begin_index_temp = index
  1494. # for j in range(len(list_tokenbegin)):
  1495. # if list_tokenbegin[j]==index:
  1496. # begin_index = j
  1497. # break
  1498. # elif list_tokenbegin[j]>index:
  1499. # begin_index = j-1
  1500. # break
  1501. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1502. # end_index_temp = index
  1503. # #index += len(str(all_match[i][0]))
  1504. # for j in range(begin_index,len(list_tokenbegin)):
  1505. # if list_tokenbegin[j]>=index:
  1506. # end_index = j-1
  1507. # break
  1508. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1509. #
  1510. #
  1511. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1512. # if len(unit)>0:
  1513. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1514. # else:
  1515. # entity_text = str(getUnifyMoney(entity_text))
  1516. #
  1517. # _exists = False
  1518. # for item in list_sentence_entitys:
  1519. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1520. # _exists = True
  1521. # if not _exists:
  1522. # if float(entity_text)>10:
  1523. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1524. #
  1525. # else:
  1526. # index += 1
  1527. #
  1528. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1529. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1530. # list_sentences.append(list_sentences_temp)
  1531. # list_entitys.append(list_entitys_temp)
  1532. # return list_articles,list_sentences,list_entitys,cost_time
  1533. def get_preprocessed(articles, useselffool=False):
  1534. '''
  1535. @summary:预处理步骤,NLP处理、实体识别
  1536. @param:
  1537. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1538. @return:list of articles,list of each article of sentences,list of each article of entitys
  1539. '''
  1540. cost_time = dict()
  1541. list_articles = get_preprocessed_article(articles,cost_time)
  1542. list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time)
  1543. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1544. calibrateEnterprise(list_articles,list_sentences,list_entitys)
  1545. return list_articles,list_sentences,list_entitys,list_outlines,cost_time
  1546. def special_treatment(sourceContent, web_source_no):
  1547. try:
  1548. if web_source_no == 'DX000202-1':
  1549. ser = re.search('中标供应商及中标金额:【((\w{5,20}-[\d,.]+,)+)】', sourceContent)
  1550. if ser:
  1551. new = ""
  1552. l = ser.group(1).split(',')
  1553. for i in range(len(l)):
  1554. it = l[i]
  1555. if '-' in it:
  1556. role, money = it.split('-')
  1557. new += '标段%d, 中标供应商: ' % (i + 1) + role + ',中标金额:' + money + '。'
  1558. sourceContent = sourceContent.replace(ser.group(0), new, 1)
  1559. elif web_source_no == '00753-14':
  1560. body = sourceContent.find("body")
  1561. body_child = body.find_all(recursive=False)
  1562. pcontent = body
  1563. if 'id' in body_child[0].attrs:
  1564. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1565. pcontent = body_child[0]
  1566. # pcontent = sourceContent.find("div", id="pcontent")
  1567. pcontent = pcontent.find_all(recursive=False)[0]
  1568. first_table = None
  1569. for idx in range(len(pcontent.find_all(recursive=False))):
  1570. t_part = pcontent.find_all(recursive=False)[idx]
  1571. if t_part.name != "table":
  1572. break
  1573. if idx == 0:
  1574. first_table = t_part
  1575. else:
  1576. for _tr in t_part.find("tbody").find_all(recursive=False):
  1577. first_table.find("tbody").append(_tr)
  1578. t_part.clear()
  1579. elif web_source_no == 'DX008357-11':
  1580. body = sourceContent.find("body")
  1581. body_child = body.find_all(recursive=False)
  1582. pcontent = body
  1583. if 'id' in body_child[0].attrs:
  1584. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1585. pcontent = body_child[0]
  1586. # pcontent = sourceContent.find("div", id="pcontent")
  1587. pcontent = pcontent.find_all(recursive=False)[0]
  1588. error_table = []
  1589. is_error_table = False
  1590. for part in pcontent.find_all(recursive=False):
  1591. if is_error_table:
  1592. if part.name == "table":
  1593. error_table.append(part)
  1594. else:
  1595. break
  1596. if part.name == "div" and part.get_text(strip=True) == "中标候选单位:":
  1597. is_error_table = True
  1598. first_table = None
  1599. for idx in range(len(error_table)):
  1600. t_part = error_table[idx]
  1601. # if t_part.name != "table":
  1602. # break
  1603. if idx == 0:
  1604. for _tr in t_part.find("tbody").find_all(recursive=False):
  1605. if _tr.get_text(strip=True) == "":
  1606. _tr.decompose()
  1607. first_table = t_part
  1608. else:
  1609. for _tr in t_part.find("tbody").find_all(recursive=False):
  1610. if _tr.get_text(strip=True) != "":
  1611. first_table.find("tbody").append(_tr)
  1612. t_part.clear()
  1613. elif web_source_no == '18021-2':
  1614. body = sourceContent.find("body")
  1615. body_child = body.find_all(recursive=False)
  1616. pcontent = body
  1617. if 'id' in body_child[0].attrs:
  1618. if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
  1619. pcontent = body_child[0]
  1620. # pcontent = sourceContent.find("div", id="pcontent")
  1621. td = pcontent.find_all("td")
  1622. for _td in td:
  1623. if str(_td.string).strip() == "报价金额":
  1624. _td.string = "单价"
  1625. elif web_source_no == '13740-2':
  1626. # “xxx成为成交供应商”
  1627. re_match = re.search("[^,。]+成为[^,。]*成交供应商", sourceContent)
  1628. if re_match:
  1629. sourceContent = sourceContent.replace(re_match.group(), "成交人:" + re_match.group())
  1630. elif web_source_no == '03786-10':
  1631. ser1 = re.search('中标价:([\d,.]+)', sourceContent)
  1632. ser2 = re.search('合同金额[((]万元[))]:([\d,.]+)', sourceContent)
  1633. if ser1 and ser2:
  1634. m1 = ser1.group(1).replace(',', '')
  1635. m2 = ser2.group(1).replace(',', '')
  1636. if float(m1) < 100000 and (m1.split('.')[0] == m2.split('.')[0] or m2 == '0'):
  1637. new = '中标价(万元):' + m1
  1638. sourceContent = sourceContent.replace(ser1.group(0), new, 1)
  1639. elif web_source_no=='00076-4':
  1640. ser = re.search('主要标的数量:([0-9一]+)\w{,3},主要标的单价:([\d,.]+)元?,合同金额:(.00),', sourceContent)
  1641. if ser:
  1642. num = ser.group(1).replace('一', '1')
  1643. try:
  1644. num = 1 if num == '0' else num
  1645. unit_price = ser.group(2).replace(',', '')
  1646. total_price = str(int(num) * float(unit_price))
  1647. new = '合同金额:' + total_price
  1648. sourceContent = sourceContent.replace('合同金额:.00', new, 1)
  1649. except Exception as e:
  1650. log('preprocessing.py special_treatment exception')
  1651. elif web_source_no=='DX000105-2':
  1652. if re.search("成交公示", sourceContent) and re.search(',投标人:', sourceContent) and re.search(',成交人:', sourceContent)==None:
  1653. sourceContent = sourceContent.replace(',投标人:', ',成交人:')
  1654. elif web_source_no in ['04080-3', '04080-4']:
  1655. ser = re.search('合同金额:([0-9,]+.[0-9]{3,})(.{,4})', sourceContent)
  1656. if ser and '万' not in ser.group(2):
  1657. sourceContent = sourceContent.replace('合同金额:', '合同金额(万元):')
  1658. elif web_source_no=='03761-3':
  1659. ser = re.search('中标价,([0-9]+)[.0-9]*%', sourceContent)
  1660. if ser and int(ser.group(1))>100:
  1661. sourceContent = sourceContent.replace(ser.group(0), ser.group(0)[:-1]+'元')
  1662. elif web_source_no=='00695-7':
  1663. ser = re.search('支付金额:', sourceContent)
  1664. if ser:
  1665. sourceContent = sourceContent.replace('支付金额:', '合同金额:')
  1666. return sourceContent
  1667. except Exception as e:
  1668. log('特殊数据源: %s 预处理特别修改抛出异常: %s'%(web_source_no, e))
  1669. return sourceContent
  1670. def article_limit(soup,limit_words=30000):
  1671. sub_space = re.compile("\s+")
  1672. def soup_limit(_soup,_count,max_count=30000,max_gap=500):
  1673. """
  1674. :param _soup: soup
  1675. :param _count: 当前字数
  1676. :param max_count: 字数最大限制
  1677. :param max_gap: 超过限制后的最大误差
  1678. :return:
  1679. """
  1680. _gap = _count - max_count
  1681. _is_skip = False
  1682. next_soup = None
  1683. while len(_soup.find_all(recursive=False)) == 1 and \
  1684. _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
  1685. _soup = _soup.find_all(recursive=False)[0]
  1686. if len(_soup.find_all(recursive=False)) == 0:
  1687. _soup.string = str(_soup.get_text())[:max_count-_count]
  1688. _count += len(re.sub(sub_space, "", _soup.string))
  1689. _gap = _count - max_count
  1690. next_soup = None
  1691. else:
  1692. for _soup_part in _soup.find_all(recursive=False):
  1693. if not _is_skip:
  1694. _count += len(re.sub(sub_space, "", _soup_part.get_text()))
  1695. if _count >= max_count:
  1696. _gap = _count - max_count
  1697. if _gap <= max_gap:
  1698. _is_skip = True
  1699. else:
  1700. _is_skip = True
  1701. next_soup = _soup_part
  1702. _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
  1703. continue
  1704. else:
  1705. _soup_part.decompose()
  1706. return _count,_gap,next_soup
  1707. text_count = 0
  1708. have_attachment = False
  1709. attachment_part = None
  1710. for child in soup.find_all(recursive=True):
  1711. if child.name == 'div' and 'class' in child.attrs:
  1712. if "richTextFetch" in child['class']:
  1713. child.insert_before("##attachment##")
  1714. attachment_part = child
  1715. have_attachment = True
  1716. break
  1717. if not have_attachment:
  1718. # 无附件
  1719. if len(re.sub(sub_space, "", soup.get_text())) > limit_words:
  1720. text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500)
  1721. while n_soup:
  1722. text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
  1723. else:
  1724. # 有附件
  1725. _text = re.sub(sub_space, "", soup.get_text())
  1726. _text_split = _text.split("##attachment##")
  1727. if len(_text_split[0])>limit_words:
  1728. main_soup = attachment_part.parent
  1729. main_text = main_soup.find_all(recursive=False)[0]
  1730. text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500)
  1731. while n_soup:
  1732. text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
  1733. if len(_text_split[1])>limit_words:
  1734. # attachment_html纯文本,无子结构
  1735. if len(attachment_part.find_all(recursive=False))==0:
  1736. attachment_part.string = str(attachment_part.get_text())[:limit_words]
  1737. else:
  1738. attachment_text_nums = 0
  1739. attachment_skip = False
  1740. for part in attachment_part.find_all(recursive=False):
  1741. if not attachment_skip:
  1742. last_attachment_text_nums = attachment_text_nums
  1743. attachment_text_nums = attachment_text_nums + len(re.sub(sub_space, "", part.get_text()))
  1744. if attachment_text_nums>=limit_words:
  1745. part.string = str(part.get_text())[:limit_words-last_attachment_text_nums]
  1746. attachment_skip = True
  1747. else:
  1748. part.decompose()
  1749. return soup
  1750. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  1751. '''
  1752. :param articles: 待处理的article source html
  1753. :param useselffool: 是否使用selffool
  1754. :return: list_articles
  1755. '''
  1756. list_articles = []
  1757. for article in articles:
  1758. doc_id = article[0]
  1759. sourceContent = article[1]
  1760. sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
  1761. sourceContent = sourceContent.replace('<br/>', '<br>')
  1762. sourceContent = re.sub("<br>(\s{0,}<br>)+","<br>",sourceContent)
  1763. # for br_match in re.findall("[^>]+?<br>",sourceContent):
  1764. # _new = re.sub("<br>","",br_match)
  1765. # # <br>标签替换为<p>标签
  1766. # if not re.search("^\s+$",_new):
  1767. # _new = '<p>'+_new + '</p>'
  1768. # # print(br_match,_new)
  1769. # sourceContent = sourceContent.replace(br_match,_new,1)
  1770. _send_doc_id = article[3]
  1771. _title = article[4]
  1772. page_time = article[5]
  1773. web_source_no = article[6]
  1774. '''特别数据源对 html 做特别修改'''
  1775. if web_source_no in ['DX000202-1']:
  1776. sourceContent = special_treatment(sourceContent, web_source_no)
  1777. #表格处理
  1778. key_preprocess = "tableToText"
  1779. start_time = time.time()
  1780. # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml"))
  1781. article_processed = BeautifulSoup(sourceContent,"lxml")
  1782. '''特别数据源对 BeautifulSoup(html) 做特别修改'''
  1783. if web_source_no in ["00753-14","DX008357-11","18021-2"]:
  1784. article_processed = special_treatment(article_processed, web_source_no)
  1785. for _soup in article_processed.descendants:
  1786. # 识别无标签文本,添加<span>标签
  1787. if not _soup.name and not _soup.parent.string and _soup.string.strip()!="":
  1788. # print(_soup.parent.string,_soup.string.strip())
  1789. _soup.wrap(article_processed.new_tag("span"))
  1790. # print(article_processed)
  1791. # 正文和附件内容限制字数30000
  1792. article_processed = article_limit(article_processed,limit_words=30000)
  1793. article_processed = get_preprocessed_outline(article_processed)
  1794. # print('article_processed')
  1795. article_processed = tableToText(article_processed)
  1796. article_processed = segment(article_processed)
  1797. article_processed = article_processed.replace('.','.') # 2021/12/01 修正OCR识别PDF小数点错误问题
  1798. article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
  1799. article_processed = article_processed.replace('成交工程价款', '成交工程价') # 2021/12/21 修正为中标价
  1800. # 修复OCR金额中“,”、“。”识别错误
  1801. article_processed_list = article_processed.split("##attachment##")
  1802. if len(article_processed_list)>1:
  1803. attachment_text = article_processed_list[1]
  1804. for _match in re.finditer("\d。\d{2}",attachment_text):
  1805. _match_text = _match.group()
  1806. attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1)
  1807. for _match in re.finditer("(\d,\d{3})[,,.]",attachment_text):
  1808. _match_text = _match.group()
  1809. attachment_text = attachment_text.replace(_match_text,_match_text.replace(",",","),1)
  1810. article_processed_list[1] = attachment_text
  1811. article_processed = "##attachment##".join(article_processed_list)
  1812. '''特别数据源对 预处理后文本 做特别修改'''
  1813. if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2"]:
  1814. article_processed = special_treatment(article_processed, web_source_no)
  1815. # 提取bidway
  1816. list_bidway = extract_bidway(article_processed, _title)
  1817. if list_bidway:
  1818. bidway = list_bidway[0].get("body")
  1819. # bidway名称统一规范
  1820. bidway = bidway_integrate(bidway)
  1821. else:
  1822. bidway = ""
  1823. # 修正被","逗号分隔的时间
  1824. repair_time = re.compile("[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d|"
  1825. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[:时点],?[0-6]\d分?|"
  1826. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]?,?(?:上午|下午)?,?[0-2]?\d,?[时点]|"
  1827. "[12]\d,?\d,?\d,?[-—-―/年],?[0-1]?\d,?[-—-―/月],?[0-3]?\d,?[日号]|"
  1828. "[0-2]?\d,?:,?[0-6]\d,?:,?[0-6]\d"
  1829. )
  1830. for _time in set(re.findall(repair_time,article_processed)):
  1831. if re.search(",",_time):
  1832. _time2 = re.sub(",", "", _time)
  1833. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time2)
  1834. if item:
  1835. _time2 = _time2.replace(item.group(),item.group() + " ")
  1836. article_processed = article_processed.replace(_time, _time2)
  1837. else:
  1838. item = re.search("[12]\d{3}[-—-―/][0-1]?\d[-—-―/][0-3]\d(?=\d)", _time)
  1839. if item:
  1840. _time2 = _time.replace(item.group(),item.group() + " ")
  1841. article_processed = article_processed.replace(_time, _time2)
  1842. # print('re_rtime',re.findall(repair_time,article_processed))
  1843. # log(article_processed)
  1844. if key_preprocess not in cost_time:
  1845. cost_time[key_preprocess] = 0
  1846. cost_time[key_preprocess] += round(time.time()-start_time,2)
  1847. #article_processed = article[1]
  1848. _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
  1849. bidway=bidway)
  1850. _article.fingerprint = getFingerprint(_title+sourceContent)
  1851. _article.page_time = page_time
  1852. list_articles.append(_article)
  1853. return list_articles
  1854. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  1855. '''
  1856. :param list_articles: 经过预处理的article text
  1857. :return: list_sentences
  1858. '''
  1859. list_sentences = []
  1860. list_outlines = []
  1861. for article in list_articles:
  1862. list_sentences_temp = []
  1863. list_entitys_temp = []
  1864. doc_id = article.id
  1865. _send_doc_id = article.doc_id
  1866. _title = article.title
  1867. #表格处理
  1868. key_preprocess = "tableToText"
  1869. start_time = time.time()
  1870. article_processed = article.content
  1871. attachment_begin_index = -1
  1872. if key_preprocess not in cost_time:
  1873. cost_time[key_preprocess] = 0
  1874. cost_time[key_preprocess] += time.time()-start_time
  1875. #nlp处理
  1876. if article_processed is not None and len(article_processed)!=0:
  1877. split_patten = "。"
  1878. sentences = []
  1879. _begin = 0
  1880. sentences_set = set()
  1881. for _iter in re.finditer(split_patten,article_processed):
  1882. _sen = article_processed[_begin:_iter.span()[1]]
  1883. if len(_sen)>0 and _sen not in sentences_set:
  1884. # 标识在附件里的句子
  1885. if re.search("##attachment##",_sen):
  1886. attachment_begin_index = len(sentences)
  1887. # _sen = re.sub("##attachment##","",_sen)
  1888. sentences.append(_sen)
  1889. sentences_set.add(_sen)
  1890. _begin = _iter.span()[1]
  1891. _sen = article_processed[_begin:]
  1892. if re.search("##attachment##", _sen):
  1893. # _sen = re.sub("##attachment##", "", _sen)
  1894. attachment_begin_index = len(sentences)
  1895. if len(_sen)>0 and _sen not in sentences_set:
  1896. sentences.append(_sen)
  1897. sentences_set.add(_sen)
  1898. # 解析outline大纲分段
  1899. outline_list = []
  1900. if re.search("##split##",article.content):
  1901. temp_sentences = []
  1902. last_sentence_index = (-1,-1)
  1903. outline_index = 0
  1904. for sentence_index in range(len(sentences)):
  1905. sentence_text = sentences[sentence_index]
  1906. for _ in re.findall("##split##", sentence_text):
  1907. _match = re.search("##split##", sentence_text)
  1908. if last_sentence_index[0] > -1:
  1909. sentence_begin_index,wordOffset_begin = last_sentence_index
  1910. sentence_end_index = sentence_index
  1911. wordOffset_end = _match.start()
  1912. if sentence_begin_index<attachment_begin_index and sentence_end_index>=attachment_begin_index:
  1913. outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,attachment_begin_index-1,wordOffset_begin,len(sentences[attachment_begin_index-1])))
  1914. else:
  1915. outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,sentence_end_index,wordOffset_begin,wordOffset_end))
  1916. outline_index += 1
  1917. sentence_text = re.sub("##split##", "", sentence_text,count=1)
  1918. last_sentence_index = (sentence_index,_match.start())
  1919. temp_sentences.append(sentence_text)
  1920. if attachment_begin_index>-1 and last_sentence_index[0]<attachment_begin_index:
  1921. outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],attachment_begin_index-1,last_sentence_index[1],len(temp_sentences[attachment_begin_index-1])))
  1922. else:
  1923. outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],len(sentences)-1,last_sentence_index[1],len(temp_sentences[-1])))
  1924. sentences = temp_sentences
  1925. #解析outline的outline_text内容
  1926. for _outline in outline_list:
  1927. if _outline.sentence_begin_index==_outline.sentence_end_index:
  1928. _text = sentences[_outline.sentence_begin_index][_outline.wordOffset_begin:_outline.wordOffset_end]
  1929. else:
  1930. _text = ""
  1931. for idx in range(_outline.sentence_begin_index,_outline.sentence_end_index+1):
  1932. if idx==_outline.sentence_begin_index:
  1933. _text += sentences[idx][_outline.wordOffset_begin:]
  1934. elif idx==_outline.sentence_end_index:
  1935. _text += sentences[idx][:_outline.wordOffset_end]
  1936. else:
  1937. _text += sentences[idx]
  1938. _outline.outline_text = _text
  1939. _outline_summary = re.split("[::,]",_text,1)[0]
  1940. if len(_outline_summary)<20:
  1941. _outline.outline_summary = _outline_summary
  1942. # print(_outline.outline_index,_outline.outline_text)
  1943. article.content = "".join(sentences)
  1944. # sentences.append(article_processed[_begin:])
  1945. lemmas = []
  1946. doc_offsets = []
  1947. dep_types = []
  1948. dep_tokens = []
  1949. time1 = time.time()
  1950. '''
  1951. tokens_all = fool.cut(sentences)
  1952. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1953. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1954. ner_entitys_all = fool.ner(sentences)
  1955. '''
  1956. #限流执行
  1957. key_nerToken = "nerToken"
  1958. start_time = time.time()
  1959. tokens_all = getTokens(sentences,useselffool=useselffool)
  1960. if key_nerToken not in cost_time:
  1961. cost_time[key_nerToken] = 0
  1962. cost_time[key_nerToken] += round(time.time()-start_time,2)
  1963. in_attachment = False
  1964. for sentence_index in range(len(sentences)):
  1965. if sentence_index == attachment_begin_index:
  1966. in_attachment = True
  1967. sentence_text = sentences[sentence_index]
  1968. tokens = tokens_all[sentence_index]
  1969. #pos_tag = pos_all[sentence_index]
  1970. pos_tag = ""
  1971. ner_entitys = ""
  1972. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys,in_attachment=in_attachment))
  1973. if len(list_sentences_temp)==0:
  1974. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  1975. list_sentences.append(list_sentences_temp)
  1976. list_outlines.append(outline_list)
  1977. return list_sentences,list_outlines
  1978. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  1979. '''
  1980. :param list_sentences:分局情况
  1981. :param cost_time:
  1982. :return: list_entitys
  1983. '''
  1984. list_entitys = []
  1985. for list_sentence in list_sentences:
  1986. sentences = []
  1987. list_entitys_temp = []
  1988. for _sentence in list_sentence:
  1989. sentences.append(_sentence.sentence_text)
  1990. lemmas = []
  1991. doc_offsets = []
  1992. dep_types = []
  1993. dep_tokens = []
  1994. time1 = time.time()
  1995. '''
  1996. tokens_all = fool.cut(sentences)
  1997. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1998. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1999. ner_entitys_all = fool.ner(sentences)
  2000. '''
  2001. #限流执行
  2002. key_nerToken = "nerToken"
  2003. start_time = time.time()
  2004. found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
  2005. # found_pingbiao = False
  2006. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  2007. if key_nerToken not in cost_time:
  2008. cost_time[key_nerToken] = 0
  2009. cost_time[key_nerToken] += round(time.time()-start_time,2)
  2010. company_dict = set()
  2011. company_index = dict((i,set()) for i in range(len(list_sentence)))
  2012. for sentence_index in range(len(list_sentence)):
  2013. list_sentence_entitys = []
  2014. sentence_text = list_sentence[sentence_index].sentence_text
  2015. tokens = list_sentence[sentence_index].tokens
  2016. doc_id = list_sentence[sentence_index].doc_id
  2017. in_attachment = list_sentence[sentence_index].in_attachment
  2018. list_tokenbegin = []
  2019. begin = 0
  2020. for i in range(0,len(tokens)):
  2021. list_tokenbegin.append(begin)
  2022. begin += len(str(tokens[i]))
  2023. list_tokenbegin.append(begin+1)
  2024. #pos_tag = pos_all[sentence_index]
  2025. pos_tag = ""
  2026. ner_entitys = ner_entitys_all[sentence_index]
  2027. '''正则识别角色实体 经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
  2028. for it in re.finditer(
  2029. '(?P<text_key_word>(((单一来源|中标|中选|中价|成交)(供应商|供货商|服务商|候选人|单位|人))|(供应商|供货商|服务商|候选人))(名称)?[为::]+)(?P<text>([^,。、;《::]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[,。]',
  2030. sentence_text):
  2031. for k, v in it.groupdict().items():
  2032. if k == 'text_key_word':
  2033. keyword = v
  2034. if k == 'text':
  2035. entity = v
  2036. b = it.start() + len(keyword)
  2037. e = it.end() - 1
  2038. if (b, e, 'location', entity) in ner_entitys:
  2039. ner_entitys.remove((b, e, 'location', entity))
  2040. ner_entitys.append((b, e, 'company', entity))
  2041. elif (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  2042. ner_entitys.append((b, e, 'company', entity))
  2043. for it in re.finditer(
  2044. '(?P<text_key_word>((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为::]+)(?P<text>\w{2,4}[省市县区镇]([^,。、;《]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[,。]',
  2045. sentence_text):
  2046. for k, v in it.groupdict().items():
  2047. if k == 'text_key_word':
  2048. keyword = v
  2049. if k == 'text':
  2050. entity = v
  2051. b = it.start() + len(keyword)
  2052. e = it.end() - 1
  2053. if (b, e, 'location', entity) in ner_entitys:
  2054. ner_entitys.remove((b, e, 'location', entity))
  2055. ner_entitys.append((b, e, 'org', entity))
  2056. if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
  2057. ner_entitys.append((b, e, 'org', entity))
  2058. for ner_entity in ner_entitys:
  2059. if ner_entity[2] in ['company','org']:
  2060. company_dict.add((ner_entity[2],ner_entity[3]))
  2061. company_index[sentence_index].add((ner_entity[0],ner_entity[1]))
  2062. #识别package
  2063. #识别实体
  2064. for ner_entity in ner_entitys:
  2065. begin_index_temp = ner_entity[0]
  2066. end_index_temp = ner_entity[1]
  2067. entity_type = ner_entity[2]
  2068. entity_text = ner_entity[3]
  2069. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  2070. continue
  2071. for j in range(len(list_tokenbegin)):
  2072. if list_tokenbegin[j]==begin_index_temp:
  2073. begin_index = j
  2074. break
  2075. elif list_tokenbegin[j]>begin_index_temp:
  2076. begin_index = j-1
  2077. break
  2078. begin_index_temp += len(str(entity_text))
  2079. for j in range(begin_index,len(list_tokenbegin)):
  2080. if list_tokenbegin[j]>=begin_index_temp:
  2081. end_index = j-1
  2082. break
  2083. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2084. #去掉标点符号
  2085. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  2086. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  2087. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
  2088. # 标记文章末尾的"发布人”、“发布时间”实体
  2089. if sentence_index==len(list_sentence)-1:
  2090. if len(list_sentence_entitys[-2:])>2:
  2091. second2last = list_sentence_entitys[-2]
  2092. last = list_sentence_entitys[-1]
  2093. if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
  2094. second2last.entity_type=="time" and last.entity_type in ["company",'org']):
  2095. if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
  2096. last.is_tail = True
  2097. second2last.is_tail = True
  2098. #使用正则识别金额
  2099. entity_type = "money"
  2100. #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  2101. # list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  2102. # "key_word":"((?P<text_key_word>(?:[¥¥]+,?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果:)(?:[,(\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[)\)]?))",
  2103. # "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())",
  2104. # "behind_m":"(()()(?P<money_behind_m>[0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\))]?)"}
  2105. list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  2106. "key_word": "((?P<text_key_word>(?:[¥¥]+,?|[单报标限总]价|金额|成交报?价|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果|成交额|中标额)(?:[,,(\(]*\s*(人民币)?(?P<unit_key_word_before>[万亿]?元?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[::])?(\d+(\*\d+%)+=)?(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]{,1})(?:[(\(]?(?P<filter_>[%])*\s*(单位[::])?(?P<unit_key_word_behind>[万亿]?元?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[)\)]?))",
  2107. "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万亿]?元)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)())",
  2108. "behind_m":"(()()(?P<money_behind_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千]*)(人民币)?[\((]?(?P<unit_behind_m>[万亿]?元(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\))]?)"}
  2109. # 2021/7/19 调整金额,单位提取正则,修复部分金额因为单位提取失败被过滤问题。
  2110. pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
  2111. set_begin = set()
  2112. # for pattern_key in list_money_pattern.keys():
  2113. # for pattern_key in ["cn","key_word","behind_m","front_m"]:
  2114. # # pattern = re.compile(list_money_pattern[pattern_key])
  2115. # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?\s*([万元]*)\s*[)\)]?))*|(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*|((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*")
  2116. # all_match = re.findall(pattern, sentence_text)
  2117. # index = 0
  2118. # for i in range(len(all_match)):
  2119. # if len(all_match[i][0])>0:
  2120. # print("===",all_match[i])
  2121. # #print(all_match[i][0])
  2122. # unit = ""
  2123. # entity_text = all_match[i][3]
  2124. # if pattern_key in ["key_word","front_m"]:
  2125. # unit = all_match[i][1]
  2126. # if pattern_key=="key_word":
  2127. # if all_match[i][1]=="" and all_match[i][4]!="":
  2128. # unit = all_match[i][4]
  2129. # else:
  2130. # unit = all_match[i][4]
  2131. # if entity_text.find("元")>=0:
  2132. # unit = ""
  2133. #
  2134. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  2135. # begin_index_temp = index
  2136. # for j in range(len(list_tokenbegin)):
  2137. # if list_tokenbegin[j]==index:
  2138. # begin_index = j
  2139. # break
  2140. # elif list_tokenbegin[j]>index:
  2141. # begin_index = j-1
  2142. # break
  2143. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  2144. # end_index_temp = index
  2145. # #index += len(str(all_match[i][0]))
  2146. # for j in range(begin_index,len(list_tokenbegin)):
  2147. # if list_tokenbegin[j]>=index:
  2148. # end_index = j-1
  2149. # break
  2150. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2151. #
  2152. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  2153. # if len(unit)>0:
  2154. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  2155. # else:
  2156. # entity_text = str(getUnifyMoney(entity_text))
  2157. #
  2158. # _exists = False
  2159. # for item in list_sentence_entitys:
  2160. # if item.entity_id==entity_id and item.entity_type==entity_type:
  2161. # _exists = True
  2162. # if not _exists:
  2163. # if float(entity_text)>1:
  2164. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  2165. #
  2166. # else:
  2167. # index += 1
  2168. # if re.search('评标结果|候选人公示', sentence_text):
  2169. # found_pingbiao = True
  2170. if re.search('业绩', sentence_text):
  2171. found_yeji += 1
  2172. if found_yeji >= 2: # 过滤掉业绩后面的所有金额
  2173. all_match = []
  2174. else:
  2175. all_match = re.finditer(pattern_money, sentence_text)
  2176. index = 0
  2177. for _match in all_match:
  2178. if len(_match.group())>0:
  2179. # print("===",_match.group())
  2180. # # print(_match.groupdict())
  2181. notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
  2182. unit = ""
  2183. entity_text = ""
  2184. text_beforeMoney = ""
  2185. filter = ""
  2186. filter_unit = False
  2187. notSure = False
  2188. if re.search('业绩', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额
  2189. # print('金额在业绩后面: ', _match.group(0))
  2190. found_yeji += 1
  2191. break
  2192. for k,v in _match.groupdict().items():
  2193. if v!="" and v is not None:
  2194. if k=='text_key_word':
  2195. notSure = True
  2196. if k.split("_")[0]=="money":
  2197. entity_text = v
  2198. if k.split("_")[0]=="unit":
  2199. if v=='万元' or unit=="": # 处理 预算金额(元):160万元 这种出现前后单位不一致情况
  2200. unit = v
  2201. if k.split("_")[0]=="text":
  2202. text_beforeMoney = v
  2203. if k.split("_")[0]=="filter":
  2204. filter = v
  2205. if re.search("filter_unit",k) is not None:
  2206. filter_unit = True
  2207. # print(_match.group())
  2208. # print(entity_text,unit,text_beforeMoney,filter,filter_unit)
  2209. if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()): # 2021/7/19 修正OCR识别小数点为逗号
  2210. if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]):
  2211. entity_text = re.sub('\d+,', '', entity_text)
  2212. else:
  2213. entity_text = entity_text.replace(',', '.')
  2214. # print(' 修正OCR识别小数点为逗号')
  2215. if entity_text.find("元")>=0:
  2216. unit = ""
  2217. if unit == "": #2021/7/21 有明显金额特征的补充单位,避免被过滤
  2218. if ('¥' in text_beforeMoney or '¥' in text_beforeMoney):
  2219. unit = '元'
  2220. # print('明显金额特征补充单位 元')
  2221. elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[::为]+$', text_beforeMoney.strip()) and \
  2222. re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None:
  2223. unit = '元'
  2224. # print('明显金额特征补充单位 元')
  2225. elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7},?)$)|(^\d{,3}(,\d{3})+,?$)',entity_text):
  2226. unit = '元'
  2227. # print('明显金额特征补充单位 元')
  2228. if unit.find("万") >= 0 and entity_text.find("万") >= 0: #2021/7/19修改为金额文本有万,不计算单位
  2229. # print('修正金额及单位都有万, 金额:',entity_text, '单位:',unit)
  2230. unit = "元"
  2231. if re.search('.*万元万元', entity_text): #2021/7/19 修正两个万元
  2232. # print(' 修正两个万元',entity_text)
  2233. entity_text = entity_text.replace('万元万元','万元')
  2234. else:
  2235. if filter_unit:
  2236. continue
  2237. if filter!="":
  2238. continue
  2239. index = _match.span()[0]+len(text_beforeMoney)
  2240. begin_index_temp = index
  2241. for j in range(len(list_tokenbegin)):
  2242. if list_tokenbegin[j]==index:
  2243. begin_index = j
  2244. break
  2245. elif list_tokenbegin[j]>index:
  2246. begin_index = j-1
  2247. break
  2248. index = _match.span()[1]
  2249. end_index_temp = index
  2250. #index += len(str(all_match[i][0]))
  2251. for j in range(begin_index,len(list_tokenbegin)):
  2252. if list_tokenbegin[j]>=index:
  2253. end_index = j-1
  2254. break
  2255. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2256. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
  2257. # print('转换前金额:', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
  2258. if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/8/5过滤掉总投资金额
  2259. # print('总投资金额: ', _match.group(0))
  2260. notes = '总投资'
  2261. elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额
  2262. notes = '投资'
  2263. elif re.search('工程造价', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额
  2264. notes = '工程造价'
  2265. elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
  2266. or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\((]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\))]*[::为]',
  2267. sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
  2268. or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
  2269. sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
  2270. notes = '保证金'
  2271. # print('保证金信息:', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
  2272. elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
  2273. sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
  2274. notes = '成本警戒线'
  2275. elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为:]', sentence_text[_match.span()[0]:_match.span()[1]]):
  2276. cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
  2277. notes = cost_re.group(1)
  2278. elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
  2279. notes = '单价'
  2280. elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
  2281. notes = '大写'
  2282. if entity_text[0] == "拾": # 2021/12/16 修正大写金额省略了数字转换错误问题
  2283. entity_text = "壹"+entity_text
  2284. # print("补充备注:notes = 大写")
  2285. if len(unit)>0:
  2286. if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况
  2287. # print('修正单位万元金额过大的情况 金额:', entity_text, '单位:', unit)
  2288. entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000)
  2289. unit = '元' # 修正金额后单位 重置为元
  2290. else:
  2291. # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
  2292. entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  2293. else:
  2294. if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8:
  2295. entity_text = str(getUnifyMoney(entity_text)/10000)
  2296. # print('修正金额字段含万 过大的情况')
  2297. else:
  2298. entity_text = str(getUnifyMoney(entity_text))
  2299. if float(entity_text)>100000000000 or float(entity_text)<100: # float(entity_text)<100 or 2022/3/4 取消最小金额限制
  2300. # print('过滤掉金额:float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
  2301. continue
  2302. if notSure and unit=="" and float(entity_text)>100*10000:
  2303. # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000:', entity_text, unit)
  2304. continue
  2305. _exists = False
  2306. for item in list_sentence_entitys:
  2307. if item.entity_id==entity_id and item.entity_type==entity_type:
  2308. _exists = True
  2309. if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index):
  2310. _exists = True
  2311. if not _exists:
  2312. if float(entity_text)>1:
  2313. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp,in_attachment=in_attachment))
  2314. list_sentence_entitys[-1].notes = notes # 2021/7/20 新增金额备注
  2315. list_sentence_entitys[-1].money_unit = unit # 2021/7/20 新增金额备注
  2316. # print('预处理中的 金额:%s, 单位:%s'%(entity_text,unit))
  2317. # print(entity_text,unit,notes)
  2318. else:
  2319. index += 1
  2320. # "联系人"正则补充提取 2021/11/15 新增
  2321. list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
  2322. error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
  2323. '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理',
  2324. '代理人','采购','附件','注意','登录','报名','踏勘',"测试"]
  2325. list_person_text = set(list_person_text + error_text)
  2326. re_person = re.compile("联系人[::]([\u4e00-\u9fa5]工)|"
  2327. "联系人[::]([\u4e00-\u9fa5]{2,3})(?=联系)|"
  2328. "联系人[::]([\u4e00-\u9fa5]{2,3})")
  2329. list_person = []
  2330. if not in_attachment:
  2331. for match_result in re_person.finditer(sentence_text):
  2332. match_text = match_result.group()
  2333. entity_text = match_text[4:]
  2334. wordOffset_begin = match_result.start() + 4
  2335. wordOffset_end = match_result.end()
  2336. # print(text[wordOffset_begin:wordOffset_end])
  2337. # 排除一些不为人名的实体
  2338. if re.search("^[\u4e00-\u9fa5]{7,}([,。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
  2339. continue
  2340. if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
  2341. _person = dict()
  2342. _person['body'] = entity_text
  2343. _person['begin_index'] = wordOffset_begin
  2344. _person['end_index'] = wordOffset_end
  2345. list_person.append(_person)
  2346. entity_type = "person"
  2347. for person in list_person:
  2348. begin_index_temp = person['begin_index']
  2349. for j in range(len(list_tokenbegin)):
  2350. if list_tokenbegin[j] == begin_index_temp:
  2351. begin_index = j
  2352. break
  2353. elif list_tokenbegin[j] > begin_index_temp:
  2354. begin_index = j - 1
  2355. break
  2356. index = person['end_index']
  2357. end_index_temp = index
  2358. for j in range(begin_index, len(list_tokenbegin)):
  2359. if list_tokenbegin[j] >= index:
  2360. end_index = j - 1
  2361. break
  2362. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2363. entity_text = person['body']
  2364. list_sentence_entitys.append(
  2365. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2366. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2367. # 资金来源提取 2020/12/30 新增
  2368. list_moneySource = extract_moneySource(sentence_text)
  2369. entity_type = "moneysource"
  2370. for moneySource in list_moneySource:
  2371. begin_index_temp = moneySource['begin_index']
  2372. for j in range(len(list_tokenbegin)):
  2373. if list_tokenbegin[j] == begin_index_temp:
  2374. begin_index = j
  2375. break
  2376. elif list_tokenbegin[j] > begin_index_temp:
  2377. begin_index = j - 1
  2378. break
  2379. index = moneySource['end_index']
  2380. end_index_temp = index
  2381. for j in range(begin_index, len(list_tokenbegin)):
  2382. if list_tokenbegin[j] >= index:
  2383. end_index = j - 1
  2384. break
  2385. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2386. entity_text = moneySource['body']
  2387. list_sentence_entitys.append(
  2388. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2389. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2390. # 电子邮箱提取 2021/11/04 新增
  2391. list_email = extract_email(sentence_text)
  2392. entity_type = "email" # 电子邮箱
  2393. for email in list_email:
  2394. begin_index_temp = email['begin_index']
  2395. for j in range(len(list_tokenbegin)):
  2396. if list_tokenbegin[j] == begin_index_temp:
  2397. begin_index = j
  2398. break
  2399. elif list_tokenbegin[j] > begin_index_temp:
  2400. begin_index = j - 1
  2401. break
  2402. index = email['end_index']
  2403. end_index_temp = index
  2404. for j in range(begin_index, len(list_tokenbegin)):
  2405. if list_tokenbegin[j] >= index:
  2406. end_index = j - 1
  2407. break
  2408. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2409. entity_text = email['body']
  2410. list_sentence_entitys.append(
  2411. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2412. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2413. # 服务期限提取 2020/12/30 新增
  2414. list_servicetime = extract_servicetime(sentence_text)
  2415. entity_type = "serviceTime"
  2416. for servicetime in list_servicetime:
  2417. begin_index_temp = servicetime['begin_index']
  2418. for j in range(len(list_tokenbegin)):
  2419. if list_tokenbegin[j] == begin_index_temp:
  2420. begin_index = j
  2421. break
  2422. elif list_tokenbegin[j] > begin_index_temp:
  2423. begin_index = j - 1
  2424. break
  2425. index = servicetime['end_index']
  2426. end_index_temp = index
  2427. for j in range(begin_index, len(list_tokenbegin)):
  2428. if list_tokenbegin[j] >= index:
  2429. end_index = j - 1
  2430. break
  2431. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2432. entity_text = servicetime['body']
  2433. list_sentence_entitys.append(
  2434. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2435. begin_index_temp, end_index_temp,in_attachment=in_attachment, prob=servicetime["prob"]))
  2436. # 招标方式提取 2020/12/30 新增
  2437. # list_bidway = extract_bidway(sentence_text, )
  2438. # entity_type = "bidway"
  2439. # for bidway in list_bidway:
  2440. # begin_index_temp = bidway['begin_index']
  2441. # end_index_temp = bidway['end_index']
  2442. # begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
  2443. # end_index = changeIndexFromWordToWords(tokens, end_index_temp)
  2444. # if begin_index is None or end_index is None:
  2445. # continue
  2446. # print(begin_index_temp,end_index_temp,begin_index,end_index)
  2447. # entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2448. # entity_text = bidway['body']
  2449. # list_sentence_entitys.append(
  2450. # Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2451. # begin_index_temp, end_index_temp))
  2452. # 2021/12/29 新增比率提取
  2453. list_ratio = extract_ratio(sentence_text)
  2454. entity_type = "ratio"
  2455. for ratio in list_ratio:
  2456. # print("ratio", ratio)
  2457. begin_index_temp = ratio['begin_index']
  2458. for j in range(len(list_tokenbegin)):
  2459. if list_tokenbegin[j] == begin_index_temp:
  2460. begin_index = j
  2461. break
  2462. elif list_tokenbegin[j] > begin_index_temp:
  2463. begin_index = j - 1
  2464. break
  2465. index = ratio['end_index']
  2466. end_index_temp = index
  2467. for j in range(begin_index, len(list_tokenbegin)):
  2468. if list_tokenbegin[j] >= index:
  2469. end_index = j - 1
  2470. break
  2471. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  2472. entity_text = ratio['body']
  2473. list_sentence_entitys.append(
  2474. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  2475. begin_index_temp, end_index_temp,in_attachment=in_attachment))
  2476. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  2477. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  2478. # 补充ner模型未识别全的company/org实体
  2479. for sentence_index in range(len(list_sentence)):
  2480. sentence_text = list_sentence[sentence_index].sentence_text
  2481. tokens = list_sentence[sentence_index].tokens
  2482. doc_id = list_sentence[sentence_index].doc_id
  2483. in_attachment = list_sentence[sentence_index].in_attachment
  2484. list_tokenbegin = []
  2485. begin = 0
  2486. for i in range(0, len(tokens)):
  2487. list_tokenbegin.append(begin)
  2488. begin += len(str(tokens[i]))
  2489. list_tokenbegin.append(begin + 1)
  2490. add_sentence_entitys = []
  2491. company_dict = sorted(list(company_dict),key=lambda x:len(x[1]),reverse=True)
  2492. for company_type,company_text in company_dict:
  2493. begin_index_list = findAllIndex(company_text,sentence_text)
  2494. for begin_index in begin_index_list:
  2495. is_continue = False
  2496. for t_begin,t_end in list(company_index[sentence_index]):
  2497. if begin_index>=t_begin and begin_index+len(company_text)<=t_end:
  2498. is_continue = True
  2499. break
  2500. if not is_continue:
  2501. add_sentence_entitys.append((begin_index,begin_index+len(company_text),company_type,company_text))
  2502. company_index[sentence_index].add((begin_index,begin_index+len(company_text)))
  2503. else:
  2504. continue
  2505. for ner_entity in add_sentence_entitys:
  2506. begin_index_temp = ner_entity[0]
  2507. end_index_temp = ner_entity[1]
  2508. entity_type = ner_entity[2]
  2509. entity_text = ner_entity[3]
  2510. if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
  2511. continue
  2512. for j in range(len(list_tokenbegin)):
  2513. if list_tokenbegin[j]==begin_index_temp:
  2514. begin_index = j
  2515. break
  2516. elif list_tokenbegin[j]>begin_index_temp:
  2517. begin_index = j-1
  2518. break
  2519. begin_index_temp += len(str(entity_text))
  2520. for j in range(begin_index,len(list_tokenbegin)):
  2521. if list_tokenbegin[j]>=begin_index_temp:
  2522. end_index = j-1
  2523. break
  2524. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  2525. #去掉标点符号
  2526. entity_text = re.sub("[,,。:!&@$\*]","",entity_text)
  2527. entity_text = entity_text.replace("(","(").replace(")",")") if isinstance(entity_text,str) else entity_text
  2528. list_entitys_temp.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
  2529. list_entitys_temp.sort(key=lambda x:(x.sentence_index,x.begin_index))
  2530. list_entitys.append(list_entitys_temp)
  2531. return list_entitys
  2532. def union_result(codeName,prem):
  2533. '''
  2534. @summary:模型的结果拼成字典
  2535. @param:
  2536. codeName:编号名称模型的结果字典
  2537. prem:拿到属性的角色的字典
  2538. @return:拼接起来的字典
  2539. '''
  2540. result = []
  2541. assert len(codeName)==len(prem)
  2542. for item_code,item_prem in zip(codeName,prem):
  2543. result.append(dict(item_code,**item_prem))
  2544. return result
  2545. def persistenceData(data):
  2546. '''
  2547. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  2548. '''
  2549. import psycopg2
  2550. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  2551. cursor = conn.cursor()
  2552. for item_index in range(len(data)):
  2553. item = data[item_index]
  2554. doc_id = item[0]
  2555. dic = item[1]
  2556. code = dic['code']
  2557. name = dic['name']
  2558. prem = dic['prem']
  2559. if len(code)==0:
  2560. code_insert = ""
  2561. else:
  2562. code_insert = ";".join(code)
  2563. prem_insert = ""
  2564. for item in prem:
  2565. for x in item:
  2566. if isinstance(x, list):
  2567. if len(x)>0:
  2568. for x1 in x:
  2569. prem_insert+="/".join(x1)+","
  2570. prem_insert+="$"
  2571. else:
  2572. prem_insert+=str(x)+"$"
  2573. prem_insert+=";"
  2574. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  2575. cursor.execute(sql)
  2576. conn.commit()
  2577. conn.close()
  2578. def persistenceData1(list_entitys,list_sentences):
  2579. '''
  2580. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  2581. '''
  2582. import psycopg2
  2583. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  2584. cursor = conn.cursor()
  2585. for list_entity in list_entitys:
  2586. for entity in list_entity:
  2587. if entity.values is not None:
  2588. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  2589. else:
  2590. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  2591. cursor.execute(sql)
  2592. for list_sentence in list_sentences:
  2593. for sentence in list_sentence:
  2594. str_tokens = "["
  2595. for item in sentence.tokens:
  2596. str_tokens += "'"
  2597. if item=="'":
  2598. str_tokens += "''"
  2599. else:
  2600. str_tokens += item
  2601. str_tokens += "',"
  2602. str_tokens = str_tokens[:-1]+"]"
  2603. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  2604. cursor.execute(sql)
  2605. conn.commit()
  2606. conn.close()
  2607. def _handle(item,result_queue):
  2608. dochtml = item["dochtml"]
  2609. docid = item["docid"]
  2610. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  2611. flag = False
  2612. if list_innerTable:
  2613. flag = True
  2614. for table in list_innerTable:
  2615. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  2616. def getPredictTable():
  2617. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  2618. import pandas as pd
  2619. import json
  2620. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  2621. df = pd.read_csv(filename)
  2622. df_data = {"json_table":[],"docid":[]}
  2623. _count = 0
  2624. _sum = len(df["docid"])
  2625. task_queue = Queue()
  2626. result_queue = Queue()
  2627. _index = 0
  2628. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  2629. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  2630. _index += 1
  2631. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  2632. mh.run()
  2633. while True:
  2634. try:
  2635. item = result_queue.get(block=True,timeout=1)
  2636. df_data["docid"].append(item["docid"])
  2637. df_data["json_table"].append(item["json_table"])
  2638. except Exception as e:
  2639. print(e)
  2640. break
  2641. df_1 = pd.DataFrame(df_data)
  2642. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  2643. if __name__=="__main__":
  2644. '''
  2645. import glob
  2646. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  2647. file_txt = str(file).replace("html","txt")
  2648. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  2649. f.write("\n================\n")
  2650. content = codecs.open(file,"r",encoding="utf8").read()
  2651. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2652. '''
  2653. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  2654. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  2655. # getPredictTable()
  2656. with open('D:/138786703.html', 'r', encoding='utf-8') as f:
  2657. sourceContent = f.read()
  2658. # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
  2659. # print(article_processed)
  2660. list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
  2661. for entity in list_entitys[0]:
  2662. print(entity.entity_type, entity.entity_text)