Preprocessing.py 83 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922
  1. from bs4 import BeautifulSoup, Comment
  2. import copy
  3. import sys
  4. import os
  5. import time
  6. import codecs
  7. sys.setrecursionlimit(1000000)
  8. sys.path.append(os.path.abspath("../.."))
  9. sys.path.append(os.path.abspath(".."))
  10. from BiddingKG.dl.common.Utils import *
  11. from BiddingKG.dl.interface.Entitys import *
  12. from BiddingKG.dl.interface.predictor import *
  13. from BiddingKG.dl.foolnltk import selffool
  14. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  15. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  16. from BiddingKG.dl.bidway.re_bidway import extract_bidway
  17. #
  18. def tableToText(soup):
  19. '''
  20. @param:
  21. soup:网页html的soup
  22. @return:处理完表格信息的网页text
  23. '''
  24. def getTrs(tbody):
  25. #获取所有的tr
  26. trs = []
  27. objs = tbody.find_all(recursive=False)
  28. for obj in objs:
  29. if obj.name=="tr":
  30. trs.append(obj)
  31. if obj.name=="tbody":
  32. for tr in obj.find_all("tr",recursive=False):
  33. trs.append(tr)
  34. return trs
  35. def fixSpan(tbody):
  36. # 处理colspan, rowspan信息补全问题
  37. #trs = tbody.findChildren('tr', recursive=False)
  38. trs = getTrs(tbody)
  39. ths_len = 0
  40. ths = list()
  41. trs_set = set()
  42. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  43. # 遍历每一个tr
  44. for indtr, tr in enumerate(trs):
  45. ths_tmp = tr.findChildren('th', recursive=False)
  46. #不补全含有表格的tr
  47. if len(tr.findChildren('table'))>0:
  48. continue
  49. if len(ths_tmp) > 0:
  50. ths_len = ths_len + len(ths_tmp)
  51. for th in ths_tmp:
  52. ths.append(th)
  53. trs_set.add(tr)
  54. # 遍历每行中的element
  55. tds = tr.findChildren(recursive=False)
  56. for indtd, td in enumerate(tds):
  57. # 若有colspan 则补全同一行下一个位置
  58. if 'colspan' in td.attrs:
  59. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  60. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  61. if col<100 and len(td.get_text())<1000:
  62. td['colspan'] = 1
  63. for i in range(1, col, 1):
  64. td.insert_after(copy.copy(td))
  65. for indtr, tr in enumerate(trs):
  66. ths_tmp = tr.findChildren('th', recursive=False)
  67. #不补全含有表格的tr
  68. if len(tr.findChildren('table'))>0:
  69. continue
  70. if len(ths_tmp) > 0:
  71. ths_len = ths_len + len(ths_tmp)
  72. for th in ths_tmp:
  73. ths.append(th)
  74. trs_set.add(tr)
  75. # 遍历每行中的element
  76. tds = tr.findChildren(recursive=False)
  77. for indtd, td in enumerate(tds):
  78. # 若有rowspan 则补全下一行同样位置
  79. if 'rowspan' in td.attrs:
  80. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  81. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  82. td['rowspan'] = 1
  83. for i in range(1, row, 1):
  84. # 获取下一行的所有td, 在对应的位置插入
  85. if indtr+i<len(trs):
  86. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  87. if len(tds1) >= (indtd) and len(tds1)>0:
  88. if indtd > 0:
  89. tds1[indtd - 1].insert_after(copy.copy(td))
  90. else:
  91. tds1[0].insert_before(copy.copy(td))
  92. def getTable(tbody):
  93. #trs = tbody.findChildren('tr', recursive=False)
  94. trs = getTrs(tbody)
  95. inner_table = []
  96. for tr in trs:
  97. tr_line = []
  98. tds = tr.findChildren(['td','th'], recursive=False)
  99. for td in tds:
  100. tr_line.append([re.sub('\xa0','',segment(td)),0])
  101. #tr_line.append([td.get_text(),0])
  102. inner_table.append(tr_line)
  103. return inner_table
  104. #处理表格不对齐的问题
  105. def fixTable(inner_table,fix_value="~~"):
  106. maxWidth = 0
  107. for item in inner_table:
  108. if len(item)>maxWidth:
  109. maxWidth = len(item)
  110. for i in range(len(inner_table)):
  111. if len(inner_table[i])<maxWidth:
  112. for j in range(maxWidth-len(inner_table[i])):
  113. inner_table[i].append([fix_value,0])
  114. return inner_table
  115. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  116. height = len(inner_table)
  117. width = len(inner_table[0])
  118. for i in range(height):
  119. point = ""
  120. for j in range(width):
  121. if inner_table[i][j][0]==point and point!="":
  122. inner_table[i][j][0] = pad_row
  123. else:
  124. if inner_table[i][j][0] not in [pad_row,pad_col]:
  125. point = inner_table[i][j][0]
  126. for j in range(width):
  127. point = ""
  128. for i in range(height):
  129. if inner_table[i][j][0]==point and point!="":
  130. inner_table[i][j][0] = pad_col
  131. else:
  132. if inner_table[i][j][0] not in [pad_row,pad_col]:
  133. point = inner_table[i][j][0]
  134. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  135. height = len(inner_table)
  136. width = len(inner_table[0])
  137. for i in range(height):
  138. for j in range(width):
  139. if inner_table[i][j][0]==pad_row:
  140. inner_table[i][j][0] = inner_table[i][j-1][0]
  141. inner_table[i][j][1] = inner_table[i][j-1][1]
  142. if inner_table[i][j][0]==pad_col:
  143. inner_table[i][j][0] = inner_table[i-1][j][0]
  144. inner_table[i][j][1] = inner_table[i-1][j][1]
  145. def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
  146. '''
  147. @summary: 修复表头识别,将明显错误的进行修正
  148. '''
  149. def repairNeeded(line):
  150. first_1 = -1
  151. last_1 = -1
  152. first_0 = -1
  153. last_0 = -1
  154. count_1 = 0
  155. count_0 = 0
  156. for i in range(len(line)):
  157. if line[i][0]==fix_value:
  158. continue
  159. if line[i][1]==1:
  160. if first_1==-1:
  161. first_1 = i
  162. last_1 = i
  163. count_1 += 1
  164. if line[i][1]==0:
  165. if first_0 == -1:
  166. first_0 = i
  167. last_0 = i
  168. count_0 += 1
  169. if first_1 ==-1 or last_0 == -1:
  170. return False
  171. #异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  172. if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
  173. return True
  174. return False
  175. def getsimilarity(line,line1):
  176. same_count = 0
  177. for item,item1 in zip(line,line1):
  178. if item[1]==item1[1]:
  179. same_count += 1
  180. return same_count/len(line)
  181. def selfrepair(inner_table,index,dye_set,key_set):
  182. '''
  183. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  184. '''
  185. #print("B",inner_table[index])
  186. min_presure = 3
  187. list_dye = []
  188. first = None
  189. count = 0
  190. temp_set = set()
  191. _index = 0
  192. for item in inner_table[index]:
  193. if first is None:
  194. first = item[1]
  195. if item[0] not in temp_set:
  196. count += 1
  197. temp_set.add(item[0])
  198. else:
  199. if first == item[1]:
  200. if item[0] not in temp_set:
  201. temp_set.add(item[0])
  202. count += 1
  203. else:
  204. list_dye.append([first,count,_index])
  205. first = item[1]
  206. temp_set.add(item[0])
  207. count = 1
  208. _index += 1
  209. list_dye.append([first,count,_index])
  210. if len(list_dye)>1:
  211. begin = 0
  212. end = 0
  213. for i in range(len(list_dye)):
  214. end = list_dye[i][2]
  215. dye_flag = False
  216. #首尾要求压力减一
  217. if i==0:
  218. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  219. dye_flag = True
  220. dye_type = list_dye[i+1][0]
  221. elif i==len(list_dye)-1:
  222. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  223. dye_flag = True
  224. dye_type = list_dye[i-1][0]
  225. else:
  226. if list_dye[i][1]>1:
  227. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  228. dye_flag = True
  229. dye_type = list_dye[i+1][0]
  230. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  231. dye_flag = True
  232. dye_type = list_dye[i-1][0]
  233. else:
  234. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  235. dye_flag = True
  236. dye_type = list_dye[i+1][0]
  237. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  238. dye_flag = True
  239. dye_type = list_dye[i-1][0]
  240. if dye_flag:
  241. for h in range(begin,end):
  242. inner_table[index][h][1] = dye_type
  243. dye_set.add((inner_table[index][h][0],dye_type))
  244. key_set.add(inner_table[index][h][0])
  245. begin = end
  246. #print("E",inner_table[index])
  247. def otherrepair(inner_table,index,dye_set,key_set):
  248. list_provide_repair = []
  249. if index==0 and len(inner_table)>1:
  250. list_provide_repair.append(index+1)
  251. elif index==len(inner_table)-1:
  252. list_provide_repair.append(index-1)
  253. else:
  254. list_provide_repair.append(index+1)
  255. list_provide_repair.append(index-1)
  256. for provide_index in list_provide_repair:
  257. if not repairNeeded(inner_table[provide_index]):
  258. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  259. if same_prob>=0.8:
  260. for i in range(len(inner_table[provide_index])):
  261. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  262. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  263. key_set.add(inner_table[index][i][0])
  264. inner_table[index][i][1] = inner_table[provide_index][i][1]
  265. elif same_prob<=0.2:
  266. for i in range(len(inner_table[provide_index])):
  267. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  268. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  269. key_set.add(inner_table[index][i][0])
  270. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  271. len_dye_set = len(dye_set)
  272. height = len(inner_table)
  273. for i in range(height):
  274. if repairNeeded(inner_table[i]):
  275. selfrepair(inner_table,i,dye_set,key_set)
  276. #otherrepair(inner_table,i,dye_set,key_set)
  277. for h in range(len(inner_table)):
  278. for w in range(len(inner_table[0])):
  279. if inner_table[h][w][0] in key_set:
  280. for item in dye_set:
  281. if inner_table[h][w][0]==item[0]:
  282. inner_table[h][w][1] = item[1]
  283. #如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  284. if len(dye_set)!=len(key_set):
  285. for i in range(height):
  286. if repairNeeded(inner_table[i]):
  287. selfrepair(inner_table,i,dye_set,key_set)
  288. #otherrepair(inner_table,i,dye_set,key_set)
  289. return
  290. if len(dye_set)==len_dye_set:
  291. '''
  292. for i in range(height):
  293. if repairNeeded(inner_table[i]):
  294. otherrepair(inner_table,i,dye_set,key_set)
  295. '''
  296. return
  297. repairTable(inner_table, dye_set, key_set)
  298. def sliceTable(inner_table,fix_value="~~"):
  299. #进行分块
  300. height = len(inner_table)
  301. width = len(inner_table[0])
  302. head_list = []
  303. head_list.append(0)
  304. last_head = None
  305. last_is_same_value = False;
  306. for h in range(height):
  307. is_all_key = True#是否是全表头行
  308. is_all_value = True#是否是全属性值
  309. is_same_with_lastHead = True#和上一行的结构是否相同
  310. is_same_value=True#一行的item都一样
  311. #is_same_first_item = True#与上一行的第一项是否相同
  312. same_value = inner_table[h][0][0]
  313. for w in range(width):
  314. if last_head is not None:
  315. if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
  316. is_all_key = False
  317. if inner_table[h][w][0]==1:
  318. is_all_value = False
  319. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  320. is_same_with_lastHead = False
  321. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  322. is_same_value = False
  323. else:
  324. if re.search("\d+",same_value) is not None:
  325. is_same_value = False
  326. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  327. is_same_first_item = False
  328. last_head = h
  329. if last_is_same_value:
  330. last_is_same_value = is_same_value
  331. continue
  332. if is_same_value:
  333. head_list.append(h)
  334. last_is_same_value = is_same_value
  335. continue
  336. if not is_all_key:
  337. if not is_same_with_lastHead:
  338. head_list.append(h)
  339. head_list.append(height)
  340. return head_list
  341. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  342. set_item = set()
  343. height = len(inner_table)
  344. width = len(inner_table[0])
  345. for i in range(height):
  346. for j in range(width):
  347. item = inner_table[i][j][0]
  348. set_item.add(item)
  349. list_item = list(set_item)
  350. x = []
  351. for item in list_item:
  352. x.append(getPredictor("form").encode(item))
  353. predict_y = getPredictor("form").predict(np.array(x),type="item")
  354. _dict = dict()
  355. for item,values in zip(list_item,list(predict_y)):
  356. _dict[item] = values[1]
  357. # print("##",item,values)
  358. #print(_dict)
  359. for i in range(height):
  360. for j in range(width):
  361. item = inner_table[i][j][0]
  362. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  363. # print("=====")
  364. # for item in inner_table:
  365. # print(item)
  366. # print("======")
  367. repairTable(inner_table)
  368. head_list = sliceTable(inner_table)
  369. return inner_table,head_list
  370. def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  371. data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
  372. predict_y = getPredictor("form").getModel("context").predict(data_x)
  373. for _position,_y in zip(data_position,predict_y):
  374. _w = _position[0]
  375. _h = _position[1]
  376. if _y[1]>prob_min:
  377. inner_table[_h][_w][1] = 1
  378. else:
  379. inner_table[_h][_w][1] = 0
  380. # print("=====")
  381. # for item in inner_table:
  382. # print(item)
  383. # print("======")
  384. repairTable(inner_table)
  385. head_list = sliceTable(inner_table)
  386. return inner_table,head_list
  387. #设置表头
  388. def setHead_inline(inner_table,prob_min=0.64):
  389. pad_row = "@@"
  390. pad_col = "##"
  391. removePadding(inner_table, pad_row, pad_col)
  392. pad_pattern = re.compile(pad_row+"|"+pad_col)
  393. height = len(inner_table)
  394. width = len(inner_table[0])
  395. head_list = []
  396. head_list.append(0)
  397. #行表头
  398. is_head_last = False
  399. for i in range(height):
  400. is_head = False
  401. is_long_value = False
  402. #判断是否是全padding值
  403. is_same_value = True
  404. same_value = inner_table[i][0][0]
  405. for j in range(width):
  406. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  407. is_same_value = False
  408. break
  409. #predict is head or not with model
  410. temp_item = ""
  411. for j in range(width):
  412. temp_item += inner_table[i][j][0]+"|"
  413. temp_item = re.sub(pad_pattern,"",temp_item)
  414. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  415. if form_prob is not None:
  416. if form_prob[0][1]>prob_min:
  417. is_head = True
  418. else:
  419. is_head = False
  420. #print(temp_item,form_prob)
  421. if len(inner_table[i][0][0])>40:
  422. is_long_value = True
  423. if is_head or is_long_value or is_same_value:
  424. #不把连续表头分开
  425. if not is_head_last:
  426. head_list.append(i)
  427. if is_long_value or is_same_value:
  428. head_list.append(i+1)
  429. if is_head:
  430. for j in range(width):
  431. inner_table[i][j][1] = 1
  432. is_head_last = is_head
  433. head_list.append(height)
  434. #列表头
  435. for i in range(len(head_list)-1):
  436. head_begin = head_list[i]
  437. head_end = head_list[i+1]
  438. #最后一列不设置为列表头
  439. for i in range(width-1):
  440. is_head = False
  441. #predict is head or not with model
  442. temp_item = ""
  443. for j in range(head_begin,head_end):
  444. temp_item += inner_table[j][i][0]+"|"
  445. temp_item = re.sub(pad_pattern,"",temp_item)
  446. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  447. if form_prob is not None:
  448. if form_prob[0][1]>prob_min:
  449. is_head = True
  450. else:
  451. is_head = False
  452. if is_head:
  453. for j in range(head_begin,head_end):
  454. inner_table[j][i][1] = 2
  455. addPadding(inner_table, pad_row, pad_col)
  456. return inner_table,head_list
  457. #设置表头
  458. def setHead_withRule(inner_table,pattern,pat_value,count):
  459. height = len(inner_table)
  460. width = len(inner_table[0])
  461. head_list = []
  462. head_list.append(0)
  463. #行表头
  464. is_head_last = False
  465. for i in range(height):
  466. set_match = set()
  467. is_head = False
  468. is_long_value = False
  469. is_same_value = True
  470. same_value = inner_table[i][0][0]
  471. for j in range(width):
  472. if inner_table[i][j][0]!=same_value:
  473. is_same_value = False
  474. break
  475. for j in range(width):
  476. if re.search(pat_value,inner_table[i][j][0]) is not None:
  477. is_head = False
  478. break
  479. str_find = re.findall(pattern,inner_table[i][j][0])
  480. if len(str_find)>0:
  481. set_match.add(inner_table[i][j][0])
  482. if len(set_match)>=count:
  483. is_head = True
  484. if len(inner_table[i][0][0])>40:
  485. is_long_value = True
  486. if is_head or is_long_value or is_same_value:
  487. if not is_head_last:
  488. head_list.append(i)
  489. if is_head:
  490. for j in range(width):
  491. inner_table[i][j][1] = 1
  492. is_head_last = is_head
  493. head_list.append(height)
  494. #列表头
  495. for i in range(len(head_list)-1):
  496. head_begin = head_list[i]
  497. head_end = head_list[i+1]
  498. #最后一列不设置为列表头
  499. for i in range(width-1):
  500. set_match = set()
  501. is_head = False
  502. for j in range(head_begin,head_end):
  503. if re.search(pat_value,inner_table[j][i][0]) is not None:
  504. is_head = False
  505. break
  506. str_find = re.findall(pattern,inner_table[j][i][0])
  507. if len(str_find)>0:
  508. set_match.add(inner_table[j][i][0])
  509. if len(set_match)>=count:
  510. is_head = True
  511. if is_head:
  512. for j in range(head_begin,head_end):
  513. inner_table[j][i][1] = 2
  514. return inner_table,head_list
  515. #取得表格的处理方向
  516. def getDirect(inner_table,begin,end):
  517. '''
  518. column_head = set()
  519. row_head = set()
  520. widths = len(inner_table[0])
  521. for height in range(begin,end):
  522. for width in range(widths):
  523. if inner_table[height][width][1] ==1:
  524. row_head.add(height)
  525. if inner_table[height][width][1] ==2:
  526. column_head.add(width)
  527. company_pattern = re.compile("公司")
  528. if 0 in column_head and begin not in row_head:
  529. return "column"
  530. if 0 in column_head and begin in row_head:
  531. for height in range(begin,end):
  532. count = 0
  533. count_flag = True
  534. for width_index in range(width):
  535. if inner_table[height][width_index][1]==0:
  536. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  537. count += 1
  538. else:
  539. count_flag = False
  540. if count_flag and count>=2:
  541. return "column"
  542. return "row"
  543. '''
  544. count_row_keys = 0
  545. count_column_keys = 0
  546. width = len(inner_table[0])
  547. if begin<end:
  548. for w in range(len(inner_table[begin])):
  549. if inner_table[begin][w][1]!=0:
  550. count_row_keys += 1
  551. for h in range(begin,end):
  552. if inner_table[h][0][1]!=0:
  553. count_column_keys += 1
  554. company_pattern = re.compile("有限(责任)?公司")
  555. for height in range(begin,end):
  556. count_set = set()
  557. count_flag = True
  558. for width_index in range(width):
  559. if inner_table[height][width_index][1]==0:
  560. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  561. count_set.add(inner_table[height][width_index][0])
  562. else:
  563. count_flag = False
  564. if count_flag and len(count_set)>=2:
  565. return "column"
  566. if count_column_keys>count_row_keys:
  567. return "column"
  568. return "row"
  569. #根据表格处理方向生成句子,
  570. def getTableText(inner_table,head_list,key_direct=False):
  571. # packPattern = "(标包|[标包][号段名])"
  572. packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  573. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标)" # 2020/11/23 大网站规则,添加序号为排序
  574. entityPattern = "(候选|([中投]标|报价)(人|单位|候选)|单位名称|供应商)"
  575. height = len(inner_table)
  576. width = len(inner_table[0])
  577. text = ""
  578. for head_i in range(len(head_list)-1):
  579. head_begin = head_list[head_i]
  580. head_end = head_list[head_i+1]
  581. direct = getDirect(inner_table, head_begin, head_end)
  582. #若只有一行,则直接按行读取
  583. if head_end-head_begin==1:
  584. text_line = ""
  585. for i in range(head_begin,head_end):
  586. for w in range(len(inner_table[i])):
  587. if inner_table[i][w][1]==1:
  588. _punctuation = ":"
  589. else:
  590. _punctuation = ","
  591. if w>0:
  592. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  593. text_line += inner_table[i][w][0]+_punctuation
  594. else:
  595. text_line += inner_table[i][w][0]+_punctuation
  596. text_line = text_line+"。" if text_line!="" else text_line
  597. text += text_line
  598. else:
  599. if direct=="row":
  600. for i in range(head_begin,head_end):
  601. pack_text = ""
  602. rank_text = ""
  603. entity_text = ""
  604. text_line = ""
  605. #在同一句话中重复的可以去掉
  606. text_set = set()
  607. for j in range(width):
  608. cell = inner_table[i][j]
  609. #是属性值
  610. if cell[1]==0 and cell[0]!="":
  611. head = ""
  612. find_flag = False
  613. temp_head = ""
  614. for loop_i in range(0,i+1-head_begin):
  615. if not key_direct:
  616. key_values = [1,2]
  617. else:
  618. key_values = [1]
  619. if inner_table[i-loop_i][j][1] in key_values:
  620. if find_flag:
  621. if inner_table[i-loop_i][j][0]!=temp_head:
  622. head = inner_table[i-loop_i][j][0]+":"+head
  623. else:
  624. head = inner_table[i-loop_i][j][0]+":"+head
  625. find_flag = True
  626. temp_head = inner_table[i-loop_i][j][0]
  627. else:
  628. #找到表头后遇到属性值就返回
  629. if find_flag:
  630. break
  631. find_flag = False
  632. temp_head = ""
  633. for loop_j in range(1,j+1):
  634. if not key_direct:
  635. key_values = [1,2]
  636. else:
  637. key_values = [2]
  638. if inner_table[i][j-loop_j][1] in key_values:
  639. if find_flag:
  640. if inner_table[i][j-loop_j][0]!=temp_head:
  641. head = inner_table[i][j-loop_j][0]+":"+head
  642. else:
  643. head = inner_table[i][j-loop_j][0]+":"+head
  644. find_flag = True
  645. temp_head = inner_table[i][j-loop_j][0]
  646. else:
  647. if find_flag:
  648. break
  649. if str(head+inner_table[i][j][0]) in text_set:
  650. continue
  651. if re.search(packPattern,head) is not None:
  652. pack_text += head+inner_table[i][j][0]+","
  653. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  654. #排名替换为同一种表达
  655. rank_text += head+inner_table[i][j][0]+","
  656. #print(rank_text)
  657. elif re.search(entityPattern,head) is not None:
  658. entity_text += head+inner_table[i][j][0]+","
  659. #print(entity_text)
  660. else:
  661. text_line += head+inner_table[i][j][0]+","
  662. text_set.add(str(head+inner_table[i][j][0]))
  663. text += pack_text+rank_text+entity_text+text_line
  664. text = text[:-1]+"。" if len(text)>0 else text
  665. else:
  666. for j in range(width):
  667. rank_text = ""
  668. entity_text = ""
  669. text_line = ""
  670. text_set = set()
  671. for i in range(head_begin,head_end):
  672. cell = inner_table[i][j]
  673. #是属性值
  674. if cell[1]==0 and cell[0]!="":
  675. find_flag = False
  676. head = ""
  677. temp_head = ""
  678. for loop_j in range(1,j+1):
  679. if not key_direct:
  680. key_values = [1,2]
  681. else:
  682. key_values = [2]
  683. if inner_table[i][j-loop_j][1] in key_values:
  684. if find_flag:
  685. if inner_table[i][j-loop_j][0]!=temp_head:
  686. head = inner_table[i][j-loop_j][0]+":"+head
  687. else:
  688. head = inner_table[i][j-loop_j][0]+":"+head
  689. find_flag = True
  690. temp_head = inner_table[i][j-loop_j][0]
  691. else:
  692. if find_flag:
  693. break
  694. find_flag = False
  695. temp_head = ""
  696. for loop_i in range(0,i+1-head_begin):
  697. if not key_direct:
  698. key_values = [1,2]
  699. else:
  700. key_values = [1]
  701. if inner_table[i-loop_i][j][1] in key_values:
  702. if find_flag:
  703. if inner_table[i-loop_i][j][0]!=temp_head:
  704. head = inner_table[i-loop_i][j][0]+":"+head
  705. else:
  706. head = inner_table[i-loop_i][j][0]+":"+head
  707. find_flag = True
  708. temp_head = inner_table[i-loop_i][j][0]
  709. else:
  710. if find_flag:
  711. break
  712. if str(head+inner_table[i][j][0]) in text_set:
  713. continue
  714. if re.search(rankPattern,head) is not None:
  715. rank_text += head+inner_table[i][j][0]+","
  716. #print(rank_text)
  717. elif re.search(entityPattern,head) is not None:
  718. entity_text += head+inner_table[i][j][0]+","
  719. #print(entity_text)
  720. else:
  721. text_line += head+inner_table[i][j][0]+","
  722. text_set.add(str(head+inner_table[i][j][0]))
  723. text += rank_text+entity_text+text_line
  724. text = text[:-1]+"。" if len(text)>0 else text
  725. return text
  726. def removeFix(inner_table,fix_value="~~"):
  727. height = len(inner_table)
  728. width = len(inner_table[0])
  729. for h in range(height):
  730. for w in range(width):
  731. if inner_table[h][w][0]==fix_value:
  732. inner_table[h][w][0] = ""
  733. def trunTable(tbody):
  734. fixSpan(tbody)
  735. inner_table = getTable(tbody)
  736. inner_table = fixTable(inner_table)
  737. if len(inner_table)>0 and len(inner_table[0])>0:
  738. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  739. #inner_table,head_list = setHead_inline(inner_table)
  740. # inner_table,head_list = setHead_initem(inner_table,pat_head)
  741. inner_table,head_list = setHead_incontext(inner_table,pat_head)
  742. # print(inner_table)
  743. # for begin in range(len(head_list[:-1])):
  744. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  745. # print(item)
  746. # print("====")
  747. removeFix(inner_table)
  748. # print("----")
  749. # print(head_list)
  750. # for item in inner_table:
  751. # print(item)
  752. tbody.string = getTableText(inner_table,head_list)
  753. #print(tbody.string)
  754. tbody.name = "turntable"
  755. return inner_table
  756. return None
  757. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$')
  758. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  759. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  760. list_innerTable = []
  761. tbodies = soup.find_all('table')
  762. # 遍历表格中的每个tbody
  763. #逆序处理嵌套表格
  764. for tbody_index in range(1,len(tbodies)+1):
  765. tbody = tbodies[len(tbodies)-tbody_index]
  766. inner_table = trunTable(tbody)
  767. list_innerTable.append(inner_table)
  768. tbodies = soup.find_all('tbody')
  769. # 遍历表格中的每个tbody
  770. #逆序处理嵌套表格
  771. for tbody_index in range(1,len(tbodies)+1):
  772. tbody = tbodies[len(tbodies)-tbody_index]
  773. inner_table = trunTable(tbody)
  774. list_innerTable.append(inner_table)
  775. return soup
  776. # return list_innerTable
  777. #数据清洗
  778. def segment(soup):
  779. # print("==")
  780. # print(soup)
  781. # print("====")
  782. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  783. if soup.name=="td":
  784. #判断有值叶子节点数
  785. _count = 0
  786. for child in soup.find_all(recursive=True):
  787. if child.get_text().strip()!="" and len(child.find_all())==0:
  788. _count += 1
  789. if _count<=1:
  790. text = soup.get_text()
  791. # 2020/11/24 大网站规则添加
  792. if 'title' in soup.attrs:
  793. if '...' in soup.get_text() and (soup.get_text()[:-3]).strip() in soup.attrs['title']:
  794. text = soup.attrs['title']
  795. _list = []
  796. for x in re.split("\s+",text):
  797. if x.strip()!="":
  798. _list.append(len(x))
  799. if len(_list)>0:
  800. _minLength = min(_list)
  801. if _minLength>2:
  802. _substr = ","
  803. else:
  804. _substr = ""
  805. else:
  806. _substr = ""
  807. text = _substr.join(re.split("(\s+)",text))
  808. text = text.replace("\r\n",",").replace("\n",",")
  809. text = re.sub("^[,\s]*|[,\s]*$","",text)
  810. return text
  811. segList = ["title"]
  812. commaList = ["div","br","td","p"]
  813. #commaList = []
  814. spaceList = ["span"]
  815. subspaceList = ["td",'a',"span","p"]
  816. tbodies = soup.find_all('tbody')
  817. if len(tbodies) == 0:
  818. tbodies = soup.find_all('table')
  819. # 递归遍历所有节点,插入符号
  820. for child in soup.find_all(recursive=True):
  821. if child.name in segList:
  822. child.insert_after("。")
  823. if child.name in commaList:
  824. child.insert_after(",")
  825. # if child.name in subspaceList:
  826. # child.insert_before("#subs"+str(child.name)+"#")
  827. # child.insert_after("#sube"+str(child.name)+"#")
  828. if child.name in spaceList:
  829. child.insert_after(" ")
  830. text = str(soup.get_text())
  831. #替换英文冒号为中文冒号
  832. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  833. #替换为中文逗号
  834. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  835. #替换为中文分号
  836. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  837. #删除标签中的所有空格
  838. for subs in subspaceList:
  839. patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  840. while(True):
  841. oneMatch = re.search(re.compile(patten),text)
  842. if oneMatch is not None:
  843. _match = oneMatch.group(1)
  844. _minLength = min([len(x) for x in re.split("(\s*)",_match)])
  845. if _minLength>2:
  846. _substr = ","
  847. else:
  848. _substr = ""
  849. text = text.replace("#subs"+str(subs)+"#"+oneMatch.group(1)+"#sube"+str(subs)+"#",re.sub("\s",_substr,oneMatch.group(1)))
  850. else:
  851. break
  852. #替换"""为"“",否则导入deepdive出错
  853. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  854. text = re.sub("\s{4,}",",",text)
  855. #替换标点
  856. while(True):
  857. #替换连续的标点
  858. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  859. if punc is not None:
  860. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  861. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  862. if punc is not None:
  863. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  864. else:
  865. #替换标点之后的空格
  866. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  867. if punc is not None:
  868. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  869. else:
  870. break
  871. #将连续的中文句号替换为一个
  872. text_split = text.split("。")
  873. text_split = [x for x in text_split if len(x)>0]
  874. list_text = []
  875. # for _t in text_split:
  876. # list_text.append(re.sub(")",")",re.sub("(","(",re.sub("\s*","",_t))))
  877. text = "。".join(text_split)
  878. # text = text.replace(')',")").replace("(","(").replace("\s","")
  879. #删除所有空格
  880. # text过大报错
  881. LOOP_LEN = 10000
  882. LOOP_BEGIN = 0
  883. _text = ""
  884. if len(text)<10000000:
  885. while(LOOP_BEGIN<len(text)):
  886. _text += re.sub(")",")",re.sub("(","(",re.sub("\s*","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  887. LOOP_BEGIN += LOOP_LEN
  888. else:
  889. return text
  890. # text = re.sub("\s*","",text)
  891. # #替换中文括号为英文括号
  892. # text = re.sub("(","(",text)
  893. # text = re.sub(")",")",text)
  894. return _text
  895. '''
  896. #数据清洗
  897. def segment(soup):
  898. segList = ["title"]
  899. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  900. spaceList = ["span"]
  901. tbodies = soup.find_all('tbody')
  902. if len(tbodies) == 0:
  903. tbodies = soup.find_all('table')
  904. # 递归遍历所有节点,插入符号
  905. for child in soup.find_all(recursive=True):
  906. if child.name == 'br':
  907. child.insert_before(',')
  908. child_text = re.sub('\s', '', child.get_text())
  909. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  910. continue
  911. if child.name in segList:
  912. child.insert_after("。")
  913. if child.name in commaList:
  914. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  915. child.insert_after(",")
  916. elif len(child_text) >=50:
  917. child.insert_after("。")
  918. #if child.name in spaceList:
  919. #child.insert_after(" ")
  920. text = str(soup.get_text())
  921. text = re.sub("\s{5,}",",",text)
  922. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  923. #替换"""为"“",否则导入deepdive出错
  924. text = text.replace('"',"“")
  925. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  926. #删除所有空格
  927. text = re.sub("\s+","#nbsp#",text)
  928. text_list = text.split('#nbsp#')
  929. new_text = ''
  930. for i in range(len(text_list)-1):
  931. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  932. new_text += text_list[i]
  933. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  934. new_text += text_list[i] + '。'
  935. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  936. new_text += text_list[i] + ';'
  937. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  938. new_text += text_list[i] + ' '
  939. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  940. new_text += text_list[i]
  941. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  942. new_text += text_list[i] + ','
  943. else:
  944. new_text += text_list[i]
  945. new_text += text_list[-1]
  946. text = new_text
  947. #替换英文冒号为中文冒号
  948. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  949. #替换为中文逗号
  950. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  951. #替换为中文分号
  952. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  953. #替换标点
  954. while(True):
  955. #替换连续的标点
  956. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  957. if punc is not None:
  958. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  959. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  960. if punc is not None:
  961. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  962. else:
  963. #替换标点之后的空格
  964. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  965. if punc is not None:
  966. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  967. else:
  968. break
  969. #将连续的中文句号替换为一个
  970. text_split = text.split("。")
  971. text_split = [x for x in text_split if len(x)>0]
  972. text = "。".join(text_split)
  973. #替换中文括号为英文括号
  974. text = re.sub("(","(",text)
  975. text = re.sub(")",")",text)
  976. return text
  977. '''
  978. #连续实体合并(弃用)
  979. def union_ner(list_ner):
  980. result_list = []
  981. union_index = []
  982. union_index_set = set()
  983. for i in range(len(list_ner)-1):
  984. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  985. if list_ner[i][1]-list_ner[i+1][0]==1:
  986. union_index_set.add(i)
  987. union_index_set.add(i+1)
  988. union_index.append((i,i+1))
  989. for i in range(len(list_ner)):
  990. if i not in union_index_set:
  991. result_list.append(list_ner[i])
  992. for item in union_index:
  993. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  994. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  995. return result_list
  996. def getTokensAndNers(sentences,MAXAREA = 10000,useselffool=False):
  997. '''
  998. @param: sentences:句子数
  999. @return 限流执行后的分词和实体识别list
  1000. '''
  1001. def getData(tokens,ners,process_data):
  1002. process_sentences = [item[1] for item in process_data]
  1003. token_ = selffool.cut(process_sentences)
  1004. if useselffool:
  1005. ner_ = selffool.self_ner(process_sentences)
  1006. else:
  1007. ner_ = selffool.ner(process_sentences)
  1008. for i in range(len(token_)):
  1009. the_index = process_data[i][0]
  1010. tokens[the_index] = token_[i]
  1011. ners[the_index] = ner_[i]
  1012. sents = []
  1013. for i in range(len(sentences)):
  1014. sents.append([i,sentences[i]])
  1015. sents.sort(key=lambda x:len(x[1]),reverse=True)
  1016. index_ = 0
  1017. tokens = [[]for i in range(len(sentences))]
  1018. ners = [[]for i in range(len(sentences))]
  1019. while(True):
  1020. width = len(sents[index_][1])
  1021. height = MAXAREA//width+1
  1022. if height>len(sents)-index_:
  1023. height = len(sents)-index_
  1024. process_data = sents[index_:index_+height]
  1025. getData(tokens, ners, process_data)
  1026. index_ += height
  1027. if index_>=len(sents):
  1028. break
  1029. return tokens,ners
  1030. def getTokens(sentences,MAXAREA = 10000,useselffool=True):
  1031. '''
  1032. @param: sentences:句子数
  1033. @return 限流执行后的分词list
  1034. '''
  1035. def getData(tokens,process_data):
  1036. process_sentences = [item[1] for item in process_data]
  1037. token_ = selffool.cut(process_sentences)
  1038. for i in range(len(token_)):
  1039. the_index = process_data[i][0]
  1040. tokens[the_index] = token_[i]
  1041. sents = []
  1042. for i in range(len(sentences)):
  1043. sents.append([i,sentences[i]])
  1044. sents.sort(key=lambda x:len(x[1]),reverse=True)
  1045. index_ = 0
  1046. tokens = [[]for i in range(len(sentences))]
  1047. while(True):
  1048. width = len(sents[index_][1])
  1049. height = MAXAREA//width+1
  1050. if height>len(sents)-index_:
  1051. height = len(sents)-index_
  1052. process_data = sents[index_:index_+height]
  1053. getData(tokens, process_data)
  1054. index_ += height
  1055. if index_>=len(sents):
  1056. break
  1057. return tokens
  1058. def getNers(sentences,MAXAREA = 10000,useselffool=False):
  1059. '''
  1060. @param: sentences:句子数
  1061. @return 限流执行后的实体识别list
  1062. '''
  1063. def getData(ners,process_data):
  1064. process_sentences = [item[1] for item in process_data]
  1065. if useselffool:
  1066. ner_ = selffool.self_ner(process_sentences)
  1067. else:
  1068. ner_ = selffool.ner(process_sentences)
  1069. for i in range(len(ner_)):
  1070. the_index = process_data[i][0]
  1071. ners[the_index] = ner_[i]
  1072. sents = []
  1073. for i in range(len(sentences)):
  1074. sents.append([i,sentences[i]])
  1075. sents.sort(key=lambda x:len(x[1]),reverse=True)
  1076. index_ = 0
  1077. ners = [[]for i in range(len(sentences))]
  1078. while(True):
  1079. width = len(sents[index_][1])
  1080. height = MAXAREA//width+1
  1081. if height>len(sents)-index_:
  1082. height = len(sents)-index_
  1083. process_data = sents[index_:index_+height]
  1084. getData( ners, process_data)
  1085. index_ += height
  1086. if index_>=len(sents):
  1087. break
  1088. return ners
  1089. # def get_preprocessed(articles,useselffool=False):
  1090. # '''
  1091. # @summary:预处理步骤,NLP处理、实体识别
  1092. # @param:
  1093. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1094. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1095. # '''
  1096. # list_articles = []
  1097. # list_sentences = []
  1098. # list_entitys = []
  1099. # cost_time = dict()
  1100. # for article in articles:
  1101. # list_sentences_temp = []
  1102. # list_entitys_temp = []
  1103. # doc_id = article[0]
  1104. # sourceContent = article[1]
  1105. # _send_doc_id = article[3]
  1106. # _title = article[4]
  1107. # #表格处理
  1108. # key_preprocess = "tableToText"
  1109. # start_time = time.time()
  1110. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1111. #
  1112. # # log(article_processed)
  1113. #
  1114. # if key_preprocess not in cost_time:
  1115. # cost_time[key_preprocess] = 0
  1116. # cost_time[key_preprocess] += time.time()-start_time
  1117. #
  1118. # #article_processed = article[1]
  1119. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1120. # #nlp处理
  1121. # if article_processed is not None and len(article_processed)!=0:
  1122. # split_patten = "。"
  1123. # sentences = []
  1124. # _begin = 0
  1125. # for _iter in re.finditer(split_patten,article_processed):
  1126. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1127. # _begin = _iter.span()[1]
  1128. # sentences.append(article_processed[_begin:])
  1129. #
  1130. # lemmas = []
  1131. # doc_offsets = []
  1132. # dep_types = []
  1133. # dep_tokens = []
  1134. #
  1135. # time1 = time.time()
  1136. #
  1137. # '''
  1138. # tokens_all = fool.cut(sentences)
  1139. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1140. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1141. # ner_entitys_all = fool.ner(sentences)
  1142. # '''
  1143. # #限流执行
  1144. # key_nerToken = "nerToken"
  1145. # start_time = time.time()
  1146. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1147. # if key_nerToken not in cost_time:
  1148. # cost_time[key_nerToken] = 0
  1149. # cost_time[key_nerToken] += time.time()-start_time
  1150. #
  1151. #
  1152. # for sentence_index in range(len(sentences)):
  1153. #
  1154. #
  1155. #
  1156. # list_sentence_entitys = []
  1157. # sentence_text = sentences[sentence_index]
  1158. # tokens = tokens_all[sentence_index]
  1159. #
  1160. # list_tokenbegin = []
  1161. # begin = 0
  1162. # for i in range(0,len(tokens)):
  1163. # list_tokenbegin.append(begin)
  1164. # begin += len(str(tokens[i]))
  1165. # list_tokenbegin.append(begin+1)
  1166. # #pos_tag = pos_all[sentence_index]
  1167. # pos_tag = ""
  1168. #
  1169. # ner_entitys = ner_entitys_all[sentence_index]
  1170. #
  1171. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1172. #
  1173. # #识别package
  1174. #
  1175. #
  1176. # #识别实体
  1177. # for ner_entity in ner_entitys:
  1178. # begin_index_temp = ner_entity[0]
  1179. # end_index_temp = ner_entity[1]
  1180. # entity_type = ner_entity[2]
  1181. # entity_text = ner_entity[3]
  1182. #
  1183. # for j in range(len(list_tokenbegin)):
  1184. # if list_tokenbegin[j]==begin_index_temp:
  1185. # begin_index = j
  1186. # break
  1187. # elif list_tokenbegin[j]>begin_index_temp:
  1188. # begin_index = j-1
  1189. # break
  1190. # begin_index_temp += len(str(entity_text))
  1191. # for j in range(begin_index,len(list_tokenbegin)):
  1192. # if list_tokenbegin[j]>=begin_index_temp:
  1193. # end_index = j-1
  1194. # break
  1195. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1196. #
  1197. # #去掉标点符号
  1198. # entity_text = re.sub("[,,。:]","",entity_text)
  1199. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1200. #
  1201. #
  1202. # #使用正则识别金额
  1203. # entity_type = "money"
  1204. #
  1205. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1206. #
  1207. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1208. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1209. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1210. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1211. #
  1212. # set_begin = set()
  1213. # for pattern_key in list_money_pattern.keys():
  1214. # pattern = re.compile(list_money_pattern[pattern_key])
  1215. # all_match = re.findall(pattern, sentence_text)
  1216. # index = 0
  1217. # for i in range(len(all_match)):
  1218. # if len(all_match[i][0])>0:
  1219. # # print("===",all_match[i])
  1220. # #print(all_match[i][0])
  1221. # unit = ""
  1222. # entity_text = all_match[i][3]
  1223. # if pattern_key in ["key_word","front_m"]:
  1224. # unit = all_match[i][1]
  1225. # else:
  1226. # unit = all_match[i][4]
  1227. # if entity_text.find("元")>=0:
  1228. # unit = ""
  1229. #
  1230. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1231. #
  1232. # begin_index_temp = index
  1233. # for j in range(len(list_tokenbegin)):
  1234. # if list_tokenbegin[j]==index:
  1235. # begin_index = j
  1236. # break
  1237. # elif list_tokenbegin[j]>index:
  1238. # begin_index = j-1
  1239. # break
  1240. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1241. # end_index_temp = index
  1242. # #index += len(str(all_match[i][0]))
  1243. # for j in range(begin_index,len(list_tokenbegin)):
  1244. # if list_tokenbegin[j]>=index:
  1245. # end_index = j-1
  1246. # break
  1247. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1248. #
  1249. #
  1250. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1251. # if len(unit)>0:
  1252. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1253. # else:
  1254. # entity_text = str(getUnifyMoney(entity_text))
  1255. #
  1256. # _exists = False
  1257. # for item in list_sentence_entitys:
  1258. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1259. # _exists = True
  1260. # if not _exists:
  1261. # if float(entity_text)>10:
  1262. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1263. #
  1264. # else:
  1265. # index += 1
  1266. #
  1267. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1268. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1269. # list_sentences.append(list_sentences_temp)
  1270. # list_entitys.append(list_entitys_temp)
  1271. # return list_articles,list_sentences,list_entitys,cost_time
  1272. def get_preprocessed(articles,useselffool=False):
  1273. '''
  1274. @summary:预处理步骤,NLP处理、实体识别
  1275. @param:
  1276. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1277. @return:list of articles,list of each article of sentences,list of each article of entitys
  1278. '''
  1279. cost_time = dict()
  1280. list_articles = get_preprocessed_article(articles,cost_time)
  1281. list_sentences = get_preprocessed_sentences(list_articles,True,cost_time)
  1282. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1283. return list_articles,list_sentences,list_entitys,cost_time
  1284. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  1285. '''
  1286. :param articles: 待处理的article source html
  1287. :param useselffool: 是否使用selffool
  1288. :return: list_articles
  1289. '''
  1290. list_articles = []
  1291. for article in articles:
  1292. doc_id = article[0]
  1293. sourceContent = article[1]
  1294. _send_doc_id = article[3]
  1295. _title = article[4]
  1296. #表格处理
  1297. key_preprocess = "tableToText"
  1298. start_time = time.time()
  1299. article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1300. # log(article_processed)
  1301. if key_preprocess not in cost_time:
  1302. cost_time[key_preprocess] = 0
  1303. cost_time[key_preprocess] += time.time()-start_time
  1304. #article_processed = article[1]
  1305. list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1306. return list_articles
  1307. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  1308. '''
  1309. :param list_articles: 经过预处理的article text
  1310. :return: list_sentences
  1311. '''
  1312. list_sentences = []
  1313. for article in list_articles:
  1314. list_sentences_temp = []
  1315. list_entitys_temp = []
  1316. doc_id = article.id
  1317. _send_doc_id = article.doc_id
  1318. _title = article.title
  1319. #表格处理
  1320. key_preprocess = "tableToText"
  1321. start_time = time.time()
  1322. article_processed = article.content
  1323. if key_preprocess not in cost_time:
  1324. cost_time[key_preprocess] = 0
  1325. cost_time[key_preprocess] += time.time()-start_time
  1326. #nlp处理
  1327. if article_processed is not None and len(article_processed)!=0:
  1328. split_patten = "。"
  1329. sentences = []
  1330. _begin = 0
  1331. sentences_set = set()
  1332. for _iter in re.finditer(split_patten,article_processed):
  1333. _sen = article_processed[_begin:_iter.span()[1]]
  1334. if len(_sen)>0 and _sen not in sentences_set:
  1335. sentences.append(_sen)
  1336. sentences_set.add(_sen)
  1337. _begin = _iter.span()[1]
  1338. _sen = article_processed[_begin:]
  1339. if len(_sen)>0 and _sen not in sentences_set:
  1340. sentences.append(_sen)
  1341. sentences_set.add(_sen)
  1342. article.content = "".join(sentences)
  1343. # sentences.append(article_processed[_begin:])
  1344. lemmas = []
  1345. doc_offsets = []
  1346. dep_types = []
  1347. dep_tokens = []
  1348. time1 = time.time()
  1349. '''
  1350. tokens_all = fool.cut(sentences)
  1351. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1352. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1353. ner_entitys_all = fool.ner(sentences)
  1354. '''
  1355. #限流执行
  1356. key_nerToken = "nerToken"
  1357. start_time = time.time()
  1358. tokens_all = getTokens(sentences,useselffool=useselffool)
  1359. if key_nerToken not in cost_time:
  1360. cost_time[key_nerToken] = 0
  1361. cost_time[key_nerToken] += time.time()-start_time
  1362. for sentence_index in range(len(sentences)):
  1363. sentence_text = sentences[sentence_index]
  1364. tokens = tokens_all[sentence_index]
  1365. #pos_tag = pos_all[sentence_index]
  1366. pos_tag = ""
  1367. ner_entitys = ""
  1368. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1369. if len(list_sentences_temp)==0:
  1370. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  1371. list_sentences.append(list_sentences_temp)
  1372. return list_sentences
  1373. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  1374. '''
  1375. :param list_sentences:分局情况
  1376. :param cost_time:
  1377. :return: list_entitys
  1378. '''
  1379. list_entitys = []
  1380. for list_sentence in list_sentences:
  1381. sentences = []
  1382. list_entitys_temp = []
  1383. for _sentence in list_sentence:
  1384. sentences.append(_sentence.sentence_text)
  1385. lemmas = []
  1386. doc_offsets = []
  1387. dep_types = []
  1388. dep_tokens = []
  1389. time1 = time.time()
  1390. '''
  1391. tokens_all = fool.cut(sentences)
  1392. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1393. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1394. ner_entitys_all = fool.ner(sentences)
  1395. '''
  1396. #限流执行
  1397. key_nerToken = "nerToken"
  1398. start_time = time.time()
  1399. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  1400. if key_nerToken not in cost_time:
  1401. cost_time[key_nerToken] = 0
  1402. cost_time[key_nerToken] += time.time()-start_time
  1403. for sentence_index in range(len(list_sentence)):
  1404. list_sentence_entitys = []
  1405. sentence_text = list_sentence[sentence_index].sentence_text
  1406. tokens = list_sentence[sentence_index].tokens
  1407. doc_id = list_sentence[sentence_index].doc_id
  1408. list_tokenbegin = []
  1409. begin = 0
  1410. for i in range(0,len(tokens)):
  1411. list_tokenbegin.append(begin)
  1412. begin += len(str(tokens[i]))
  1413. list_tokenbegin.append(begin+1)
  1414. #pos_tag = pos_all[sentence_index]
  1415. pos_tag = ""
  1416. ner_entitys = ner_entitys_all[sentence_index]
  1417. #识别package
  1418. #识别实体
  1419. for ner_entity in ner_entitys:
  1420. begin_index_temp = ner_entity[0]
  1421. end_index_temp = ner_entity[1]
  1422. entity_type = ner_entity[2]
  1423. entity_text = ner_entity[3]
  1424. for j in range(len(list_tokenbegin)):
  1425. if list_tokenbegin[j]==begin_index_temp:
  1426. begin_index = j
  1427. break
  1428. elif list_tokenbegin[j]>begin_index_temp:
  1429. begin_index = j-1
  1430. break
  1431. begin_index_temp += len(str(entity_text))
  1432. for j in range(begin_index,len(list_tokenbegin)):
  1433. if list_tokenbegin[j]>=begin_index_temp:
  1434. end_index = j-1
  1435. break
  1436. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1437. #去掉标点符号
  1438. entity_text = re.sub("[,,。:]","",entity_text)
  1439. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1440. #使用正则识别金额
  1441. entity_type = "money"
  1442. #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1443. list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  1444. "key_word":"((?P<text_key_word>(?:[¥¥]+,?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果:)(?:[,(\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[)\)]?))",
  1445. "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())",
  1446. "behind_m":"(()()(?P<money_behind_m>[0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\))]?)"}
  1447. pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
  1448. set_begin = set()
  1449. # for pattern_key in list_money_pattern.keys():
  1450. # for pattern_key in ["cn","key_word","behind_m","front_m"]:
  1451. # # pattern = re.compile(list_money_pattern[pattern_key])
  1452. # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?\s*([万元]*)\s*[)\)]?))*|(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*|((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*")
  1453. # all_match = re.findall(pattern, sentence_text)
  1454. # index = 0
  1455. # for i in range(len(all_match)):
  1456. # if len(all_match[i][0])>0:
  1457. # print("===",all_match[i])
  1458. # #print(all_match[i][0])
  1459. # unit = ""
  1460. # entity_text = all_match[i][3]
  1461. # if pattern_key in ["key_word","front_m"]:
  1462. # unit = all_match[i][1]
  1463. # if pattern_key=="key_word":
  1464. # if all_match[i][1]=="" and all_match[i][4]!="":
  1465. # unit = all_match[i][4]
  1466. # else:
  1467. # unit = all_match[i][4]
  1468. # if entity_text.find("元")>=0:
  1469. # unit = ""
  1470. #
  1471. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1472. # begin_index_temp = index
  1473. # for j in range(len(list_tokenbegin)):
  1474. # if list_tokenbegin[j]==index:
  1475. # begin_index = j
  1476. # break
  1477. # elif list_tokenbegin[j]>index:
  1478. # begin_index = j-1
  1479. # break
  1480. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1481. # end_index_temp = index
  1482. # #index += len(str(all_match[i][0]))
  1483. # for j in range(begin_index,len(list_tokenbegin)):
  1484. # if list_tokenbegin[j]>=index:
  1485. # end_index = j-1
  1486. # break
  1487. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1488. #
  1489. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1490. # if len(unit)>0:
  1491. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1492. # else:
  1493. # entity_text = str(getUnifyMoney(entity_text))
  1494. #
  1495. # _exists = False
  1496. # for item in list_sentence_entitys:
  1497. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1498. # _exists = True
  1499. # if not _exists:
  1500. # if float(entity_text)>1:
  1501. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1502. #
  1503. # else:
  1504. # index += 1
  1505. all_match = re.finditer(pattern_money, sentence_text)
  1506. index = 0
  1507. for _match in all_match:
  1508. if len(_match.group())>0:
  1509. # print("===",_match.group())
  1510. # print(_match.groupdict())
  1511. unit = ""
  1512. entity_text = ""
  1513. text_beforeMoney = ""
  1514. filter = ""
  1515. filter_unit = False
  1516. for k,v in _match.groupdict().items():
  1517. if v!="" and v is not None:
  1518. if k.split("_")[0]=="money":
  1519. entity_text = v
  1520. if k.split("_")[0]=="unit":
  1521. unit = v
  1522. if k.split("_")[0]=="text":
  1523. text_beforeMoney = v
  1524. if k.split("_")[0]=="filter":
  1525. filter = v
  1526. if re.search("filter_unit",k) is not None:
  1527. filter_unit = True
  1528. if entity_text.find("元")>=0:
  1529. unit = ""
  1530. else:
  1531. if filter_unit:
  1532. continue
  1533. if filter!="":
  1534. continue
  1535. index = _match.span()[0]+len(text_beforeMoney)
  1536. begin_index_temp = index
  1537. for j in range(len(list_tokenbegin)):
  1538. if list_tokenbegin[j]==index:
  1539. begin_index = j
  1540. break
  1541. elif list_tokenbegin[j]>index:
  1542. begin_index = j-1
  1543. break
  1544. index = _match.span()[1]
  1545. end_index_temp = index
  1546. #index += len(str(all_match[i][0]))
  1547. for j in range(begin_index,len(list_tokenbegin)):
  1548. if list_tokenbegin[j]>=index:
  1549. end_index = j-1
  1550. break
  1551. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1552. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
  1553. if len(unit)>0:
  1554. entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1555. else:
  1556. entity_text = str(getUnifyMoney(entity_text))
  1557. if float(entity_text)<100 or float(entity_text)>100000000000:
  1558. continue
  1559. _exists = False
  1560. for item in list_sentence_entitys:
  1561. if item.entity_id==entity_id and item.entity_type==entity_type:
  1562. _exists = True
  1563. if not _exists:
  1564. if float(entity_text)>1:
  1565. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1566. else:
  1567. index += 1
  1568. # 资金来源提取 2020/12/30 新增
  1569. list_moneySource = extract_moneySource(sentence_text)
  1570. entity_type = "moneysource"
  1571. for moneySource in list_moneySource:
  1572. begin_index_temp = moneySource['begin_index']
  1573. for j in range(len(list_tokenbegin)):
  1574. if list_tokenbegin[j] == begin_index_temp:
  1575. begin_index = j
  1576. break
  1577. elif list_tokenbegin[j] > begin_index_temp:
  1578. begin_index = j - 1
  1579. break
  1580. index = moneySource['end_index']
  1581. end_index_temp = index
  1582. for j in range(begin_index, len(list_tokenbegin)):
  1583. if list_tokenbegin[j] >= index:
  1584. end_index = j - 1
  1585. break
  1586. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1587. entity_text = moneySource['body']
  1588. list_sentence_entitys.append(
  1589. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1590. begin_index_temp, end_index_temp))
  1591. # 服务期限提取 2020/12/30 新增
  1592. list_servicetime = extract_servicetime(sentence_text)
  1593. entity_type = "serviceTime"
  1594. for servicetime in list_servicetime:
  1595. begin_index_temp = servicetime['begin_index']
  1596. for j in range(len(list_tokenbegin)):
  1597. if list_tokenbegin[j] == begin_index_temp:
  1598. begin_index = j
  1599. break
  1600. elif list_tokenbegin[j] > begin_index_temp:
  1601. begin_index = j - 1
  1602. break
  1603. index = servicetime['end_index']
  1604. end_index_temp = index
  1605. for j in range(begin_index, len(list_tokenbegin)):
  1606. if list_tokenbegin[j] >= index:
  1607. end_index = j - 1
  1608. break
  1609. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1610. entity_text = servicetime['body']
  1611. list_sentence_entitys.append(
  1612. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1613. begin_index_temp, end_index_temp))
  1614. # 招标方式提取 2020/12/30 新增
  1615. list_bidway = extract_bidway(sentence_text)
  1616. entity_type = "bidway"
  1617. for bidway in list_bidway:
  1618. begin_index_temp = bidway['begin_index']
  1619. end_index_temp = bidway['end_index']
  1620. begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
  1621. end_index = changeIndexFromWordToWords(tokens, end_index_temp)
  1622. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1623. entity_text = bidway['body']
  1624. list_sentence_entitys.append(
  1625. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1626. begin_index_temp, end_index_temp))
  1627. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1628. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1629. list_entitys.append(list_entitys_temp)
  1630. return list_entitys
  1631. def union_result(codeName,prem):
  1632. '''
  1633. @summary:模型的结果拼成字典
  1634. @param:
  1635. codeName:编号名称模型的结果字典
  1636. prem:拿到属性的角色的字典
  1637. @return:拼接起来的字典
  1638. '''
  1639. result = []
  1640. assert len(codeName)==len(prem)
  1641. for item_code,item_prem in zip(codeName,prem):
  1642. result.append(dict(item_code,**item_prem))
  1643. return result
  1644. def persistenceData(data):
  1645. '''
  1646. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  1647. '''
  1648. import psycopg2
  1649. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  1650. cursor = conn.cursor()
  1651. for item_index in range(len(data)):
  1652. item = data[item_index]
  1653. doc_id = item[0]
  1654. dic = item[1]
  1655. code = dic['code']
  1656. name = dic['name']
  1657. prem = dic['prem']
  1658. if len(code)==0:
  1659. code_insert = ""
  1660. else:
  1661. code_insert = ";".join(code)
  1662. prem_insert = ""
  1663. for item in prem:
  1664. for x in item:
  1665. if isinstance(x, list):
  1666. if len(x)>0:
  1667. for x1 in x:
  1668. prem_insert+="/".join(x1)+","
  1669. prem_insert+="$"
  1670. else:
  1671. prem_insert+=str(x)+"$"
  1672. prem_insert+=";"
  1673. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  1674. cursor.execute(sql)
  1675. conn.commit()
  1676. conn.close()
  1677. def persistenceData1(list_entitys,list_sentences):
  1678. '''
  1679. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  1680. '''
  1681. import psycopg2
  1682. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  1683. cursor = conn.cursor()
  1684. for list_entity in list_entitys:
  1685. for entity in list_entity:
  1686. if entity.values is not None:
  1687. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  1688. else:
  1689. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  1690. cursor.execute(sql)
  1691. for list_sentence in list_sentences:
  1692. for sentence in list_sentence:
  1693. str_tokens = "["
  1694. for item in sentence.tokens:
  1695. str_tokens += "'"
  1696. if item=="'":
  1697. str_tokens += "''"
  1698. else:
  1699. str_tokens += item
  1700. str_tokens += "',"
  1701. str_tokens = str_tokens[:-1]+"]"
  1702. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  1703. cursor.execute(sql)
  1704. conn.commit()
  1705. conn.close()
  1706. def _handle(item,result_queue):
  1707. dochtml = item["dochtml"]
  1708. docid = item["docid"]
  1709. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  1710. flag = False
  1711. if list_innerTable:
  1712. flag = True
  1713. for table in list_innerTable:
  1714. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  1715. def getPredictTable():
  1716. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  1717. import pandas as pd
  1718. import json
  1719. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  1720. df = pd.read_csv(filename)
  1721. df_data = {"json_table":[],"docid":[]}
  1722. _count = 0
  1723. _sum = len(df["docid"])
  1724. task_queue = Queue()
  1725. result_queue = Queue()
  1726. _index = 0
  1727. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  1728. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  1729. _index += 1
  1730. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  1731. mh.run()
  1732. while True:
  1733. try:
  1734. item = result_queue.get(block=True,timeout=1)
  1735. df_data["docid"].append(item["docid"])
  1736. df_data["json_table"].append(item["json_table"])
  1737. except Exception as e:
  1738. print(e)
  1739. break
  1740. df_1 = pd.DataFrame(df_data)
  1741. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  1742. if __name__=="__main__":
  1743. '''
  1744. import glob
  1745. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  1746. file_txt = str(file).replace("html","txt")
  1747. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  1748. f.write("\n================\n")
  1749. content = codecs.open(file,"r",encoding="utf8").read()
  1750. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  1751. '''
  1752. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  1753. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  1754. getPredictTable()