Preprocessing.py 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884
  1. from bs4 import BeautifulSoup, Comment
  2. import copy
  3. import sys
  4. import os
  5. import time
  6. import codecs
  7. sys.setrecursionlimit(1000000)
  8. sys.path.append(os.path.abspath("../.."))
  9. sys.path.append(os.path.abspath(".."))
  10. from BiddingKG.dl.common.Utils import *
  11. from BiddingKG.dl.interface.Entitys import *
  12. from BiddingKG.dl.interface.predictor import *
  13. from BiddingKG.dl.foolnltk import selffool
  14. from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
  15. from BiddingKG.dl.time.re_servicetime import extract_servicetime
  16. from BiddingKG.dl.bidway.re_bidway import extract_bidway
  17. #
  18. def tableToText(soup):
  19. '''
  20. @param:
  21. soup:网页html的soup
  22. @return:处理完表格信息的网页text
  23. '''
  24. def getTrs(tbody):
  25. #获取所有的tr
  26. trs = []
  27. objs = tbody.find_all(recursive=False)
  28. for obj in objs:
  29. if obj.name=="tr":
  30. trs.append(obj)
  31. if obj.name=="tbody":
  32. for tr in obj.find_all("tr",recursive=False):
  33. trs.append(tr)
  34. return trs
  35. def fixSpan(tbody):
  36. # 处理colspan, rowspan信息补全问题
  37. #trs = tbody.findChildren('tr', recursive=False)
  38. trs = getTrs(tbody)
  39. ths_len = 0
  40. ths = list()
  41. trs_set = set()
  42. #修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  43. # 遍历每一个tr
  44. for indtr, tr in enumerate(trs):
  45. ths_tmp = tr.findChildren('th', recursive=False)
  46. #不补全含有表格的tr
  47. if len(tr.findChildren('table'))>0:
  48. continue
  49. if len(ths_tmp) > 0:
  50. ths_len = ths_len + len(ths_tmp)
  51. for th in ths_tmp:
  52. ths.append(th)
  53. trs_set.add(tr)
  54. # 遍历每行中的element
  55. tds = tr.findChildren(recursive=False)
  56. for indtd, td in enumerate(tds):
  57. # 若有colspan 则补全同一行下一个位置
  58. if 'colspan' in td.attrs:
  59. if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
  60. col = int(re.sub("[^0-9]","",str(td['colspan'])))
  61. if col<100 and len(td.get_text())<1000:
  62. td['colspan'] = 1
  63. for i in range(1, col, 1):
  64. td.insert_after(copy.copy(td))
  65. for indtr, tr in enumerate(trs):
  66. ths_tmp = tr.findChildren('th', recursive=False)
  67. #不补全含有表格的tr
  68. if len(tr.findChildren('table'))>0:
  69. continue
  70. if len(ths_tmp) > 0:
  71. ths_len = ths_len + len(ths_tmp)
  72. for th in ths_tmp:
  73. ths.append(th)
  74. trs_set.add(tr)
  75. # 遍历每行中的element
  76. tds = tr.findChildren(recursive=False)
  77. for indtd, td in enumerate(tds):
  78. # 若有rowspan 则补全下一行同样位置
  79. if 'rowspan' in td.attrs:
  80. if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
  81. row = int(re.sub("[^0-9]","",str(td['rowspan'])))
  82. td['rowspan'] = 1
  83. for i in range(1, row, 1):
  84. # 获取下一行的所有td, 在对应的位置插入
  85. if indtr+i<len(trs):
  86. tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
  87. if len(tds1) >= (indtd) and len(tds1)>0:
  88. if indtd > 0:
  89. tds1[indtd - 1].insert_after(copy.copy(td))
  90. else:
  91. tds1[0].insert_before(copy.copy(td))
  92. def getTable(tbody):
  93. #trs = tbody.findChildren('tr', recursive=False)
  94. trs = getTrs(tbody)
  95. inner_table = []
  96. for tr in trs:
  97. tr_line = []
  98. tds = tr.findChildren(['td','th'], recursive=False)
  99. for td in tds:
  100. tr_line.append([re.sub('\xa0','',segment(td)),0])
  101. #tr_line.append([td.get_text(),0])
  102. inner_table.append(tr_line)
  103. return inner_table
  104. #处理表格不对齐的问题
  105. def fixTable(inner_table,fix_value="~~"):
  106. maxWidth = 0
  107. for item in inner_table:
  108. if len(item)>maxWidth:
  109. maxWidth = len(item)
  110. for i in range(len(inner_table)):
  111. if len(inner_table[i])<maxWidth:
  112. for j in range(maxWidth-len(inner_table[i])):
  113. inner_table[i].append([fix_value,0])
  114. return inner_table
  115. def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
  116. height = len(inner_table)
  117. width = len(inner_table[0])
  118. for i in range(height):
  119. point = ""
  120. for j in range(width):
  121. if inner_table[i][j][0]==point and point!="":
  122. inner_table[i][j][0] = pad_row
  123. else:
  124. if inner_table[i][j][0] not in [pad_row,pad_col]:
  125. point = inner_table[i][j][0]
  126. for j in range(width):
  127. point = ""
  128. for i in range(height):
  129. if inner_table[i][j][0]==point and point!="":
  130. inner_table[i][j][0] = pad_col
  131. else:
  132. if inner_table[i][j][0] not in [pad_row,pad_col]:
  133. point = inner_table[i][j][0]
  134. def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
  135. height = len(inner_table)
  136. width = len(inner_table[0])
  137. for i in range(height):
  138. for j in range(width):
  139. if inner_table[i][j][0]==pad_row:
  140. inner_table[i][j][0] = inner_table[i][j-1][0]
  141. inner_table[i][j][1] = inner_table[i][j-1][1]
  142. if inner_table[i][j][0]==pad_col:
  143. inner_table[i][j][0] = inner_table[i-1][j][0]
  144. inner_table[i][j][1] = inner_table[i-1][j][1]
  145. def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
  146. def repairTable(inner_table,dye_set = set(),key_set = set()):
  147. '''
  148. @summary: 修复表头识别,将明显错误的进行修正
  149. '''
  150. def repairNeeded(line):
  151. first_1 = -1
  152. last_1 = -1
  153. first_0 = -1
  154. last_0 = -1
  155. count_1 = 0
  156. count_0 = 0
  157. for i in range(len(line)):
  158. if line[i][0]==fix_value:
  159. continue
  160. if line[i][1]==1:
  161. if first_1==-1:
  162. first_1 = i
  163. last_1 = i
  164. count_1 += 1
  165. if line[i][1]==0:
  166. if first_0 == -1:
  167. first_0 = i
  168. last_0 = i
  169. count_0 += 1
  170. if first_1 ==-1 or last_0 == -1:
  171. return False
  172. #异常情况:第一个不是表头;最后一个是表头;表头个数远大于属性值个数
  173. if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
  174. return True
  175. return False
  176. def getsimilarity(line,line1):
  177. same_count = 0
  178. for item,item1 in zip(line,line1):
  179. if item[1]==item1[1]:
  180. same_count += 1
  181. return same_count/len(line)
  182. def selfrepair(inner_table,index,dye_set,key_set):
  183. '''
  184. @summary: 计算每个节点受到的挤压度来判断是否需要染色
  185. '''
  186. #print("B",inner_table[index])
  187. min_presure = 3
  188. list_dye = []
  189. first = None
  190. count = 0
  191. temp_set = set()
  192. _index = 0
  193. for item in inner_table[index]:
  194. if first is None:
  195. first = item[1]
  196. if item[0] not in temp_set:
  197. count += 1
  198. temp_set.add(item[0])
  199. else:
  200. if first == item[1]:
  201. if item[0] not in temp_set:
  202. temp_set.add(item[0])
  203. count += 1
  204. else:
  205. list_dye.append([first,count,_index])
  206. first = item[1]
  207. temp_set.add(item[0])
  208. count = 1
  209. _index += 1
  210. list_dye.append([first,count,_index])
  211. if len(list_dye)>1:
  212. begin = 0
  213. end = 0
  214. for i in range(len(list_dye)):
  215. end = list_dye[i][2]
  216. dye_flag = False
  217. #首尾要求压力减一
  218. if i==0:
  219. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
  220. dye_flag = True
  221. dye_type = list_dye[i+1][0]
  222. elif i==len(list_dye)-1:
  223. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
  224. dye_flag = True
  225. dye_type = list_dye[i-1][0]
  226. else:
  227. if list_dye[i][1]>1:
  228. if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
  229. dye_flag = True
  230. dye_type = list_dye[i+1][0]
  231. if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  232. dye_flag = True
  233. dye_type = list_dye[i-1][0]
  234. else:
  235. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  236. dye_flag = True
  237. dye_type = list_dye[i+1][0]
  238. if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
  239. dye_flag = True
  240. dye_type = list_dye[i-1][0]
  241. if dye_flag:
  242. for h in range(begin,end):
  243. inner_table[index][h][1] = dye_type
  244. dye_set.add((inner_table[index][h][0],dye_type))
  245. key_set.add(inner_table[index][h][0])
  246. begin = end
  247. #print("E",inner_table[index])
  248. def otherrepair(inner_table,index,dye_set,key_set):
  249. list_provide_repair = []
  250. if index==0 and len(inner_table)>1:
  251. list_provide_repair.append(index+1)
  252. elif index==len(inner_table)-1:
  253. list_provide_repair.append(index-1)
  254. else:
  255. list_provide_repair.append(index+1)
  256. list_provide_repair.append(index-1)
  257. for provide_index in list_provide_repair:
  258. if not repairNeeded(inner_table[provide_index]):
  259. same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
  260. if same_prob>=0.8:
  261. for i in range(len(inner_table[provide_index])):
  262. if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
  263. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  264. key_set.add(inner_table[index][i][0])
  265. inner_table[index][i][1] = inner_table[provide_index][i][1]
  266. elif same_prob<=0.2:
  267. for i in range(len(inner_table[provide_index])):
  268. if inner_table[index][i][1]==inner_table[provide_index][i][1]:
  269. dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
  270. key_set.add(inner_table[index][i][0])
  271. inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
  272. len_dye_set = len(dye_set)
  273. height = len(inner_table)
  274. for i in range(height):
  275. if repairNeeded(inner_table[i]):
  276. selfrepair(inner_table,i,dye_set,key_set)
  277. #otherrepair(inner_table,i,dye_set,key_set)
  278. for h in range(len(inner_table)):
  279. for w in range(len(inner_table[0])):
  280. if inner_table[h][w][0] in key_set:
  281. for item in dye_set:
  282. if inner_table[h][w][0]==item[0]:
  283. inner_table[h][w][1] = item[1]
  284. #如果两个set长度不相同,则有同一个key被反复染色,将导致无限迭代
  285. if len(dye_set)!=len(key_set):
  286. for i in range(height):
  287. if repairNeeded(inner_table[i]):
  288. selfrepair(inner_table,i,dye_set,key_set)
  289. #otherrepair(inner_table,i,dye_set,key_set)
  290. return
  291. if len(dye_set)==len_dye_set:
  292. '''
  293. for i in range(height):
  294. if repairNeeded(inner_table[i]):
  295. otherrepair(inner_table,i,dye_set,key_set)
  296. '''
  297. return
  298. repairTable(inner_table, dye_set, key_set)
  299. set_item = set()
  300. height = len(inner_table)
  301. width = len(inner_table[0])
  302. for i in range(height):
  303. for j in range(width):
  304. item = inner_table[i][j][0]
  305. set_item.add(item)
  306. list_item = list(set_item)
  307. x = []
  308. for item in list_item:
  309. x.append(getPredictor("form").encode(item))
  310. predict_y = getPredictor("form").predict(np.array(x),type="item")
  311. _dict = dict()
  312. for item,values in zip(list_item,list(predict_y)):
  313. _dict[item] = values[1]
  314. # print("##",item,values)
  315. #print(_dict)
  316. for i in range(height):
  317. for j in range(width):
  318. item = inner_table[i][j][0]
  319. inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
  320. repairTable(inner_table)
  321. #进行分块
  322. head_list = []
  323. head_list.append(0)
  324. last_head = None
  325. for h in range(height):
  326. is_all_key = True#是否是全表头行
  327. is_all_value = True#是否是全属性值
  328. is_same_with_lastHead = True#和上一行的结构是否相同
  329. is_same_value=True#一行的item都一样
  330. #is_same_first_item = True#与上一行的第一项是否相同
  331. same_value = inner_table[h][0][0]
  332. for w in range(width):
  333. if last_head is not None:
  334. if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
  335. is_all_key = False
  336. if inner_table[h][w][0]==1:
  337. is_all_value = False
  338. if inner_table[h][w][1]!= inner_table[h-1][w][1]:
  339. is_same_with_lastHead = False
  340. if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
  341. is_same_value = False
  342. else:
  343. if re.search("\d+",same_value) is not None:
  344. is_same_value = False
  345. if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
  346. is_same_first_item = False
  347. last_head = h
  348. if is_same_value:
  349. head_list.append(h)
  350. head_list.append(h+1)
  351. continue
  352. if not is_all_key:
  353. if not is_same_with_lastHead:
  354. head_list.append(h)
  355. head_list.append(height)
  356. return inner_table,head_list
  357. #设置表头
  358. def setHead_inline(inner_table,prob_min=0.64):
  359. pad_row = "@@"
  360. pad_col = "##"
  361. removePadding(inner_table, pad_row, pad_col)
  362. pad_pattern = re.compile(pad_row+"|"+pad_col)
  363. height = len(inner_table)
  364. width = len(inner_table[0])
  365. head_list = []
  366. head_list.append(0)
  367. #行表头
  368. is_head_last = False
  369. for i in range(height):
  370. is_head = False
  371. is_long_value = False
  372. #判断是否是全padding值
  373. is_same_value = True
  374. same_value = inner_table[i][0][0]
  375. for j in range(width):
  376. if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
  377. is_same_value = False
  378. break
  379. #predict is head or not with model
  380. temp_item = ""
  381. for j in range(width):
  382. temp_item += inner_table[i][j][0]+"|"
  383. temp_item = re.sub(pad_pattern,"",temp_item)
  384. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  385. if form_prob is not None:
  386. if form_prob[0][1]>prob_min:
  387. is_head = True
  388. else:
  389. is_head = False
  390. #print(temp_item,form_prob)
  391. if len(inner_table[i][0][0])>40:
  392. is_long_value = True
  393. if is_head or is_long_value or is_same_value:
  394. #不把连续表头分开
  395. if not is_head_last:
  396. head_list.append(i)
  397. if is_long_value or is_same_value:
  398. head_list.append(i+1)
  399. if is_head:
  400. for j in range(width):
  401. inner_table[i][j][1] = 1
  402. is_head_last = is_head
  403. head_list.append(height)
  404. #列表头
  405. for i in range(len(head_list)-1):
  406. head_begin = head_list[i]
  407. head_end = head_list[i+1]
  408. #最后一列不设置为列表头
  409. for i in range(width-1):
  410. is_head = False
  411. #predict is head or not with model
  412. temp_item = ""
  413. for j in range(head_begin,head_end):
  414. temp_item += inner_table[j][i][0]+"|"
  415. temp_item = re.sub(pad_pattern,"",temp_item)
  416. form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
  417. if form_prob is not None:
  418. if form_prob[0][1]>prob_min:
  419. is_head = True
  420. else:
  421. is_head = False
  422. if is_head:
  423. for j in range(head_begin,head_end):
  424. inner_table[j][i][1] = 2
  425. addPadding(inner_table, pad_row, pad_col)
  426. return inner_table,head_list
  427. #设置表头
  428. def setHead_withRule(inner_table,pattern,pat_value,count):
  429. height = len(inner_table)
  430. width = len(inner_table[0])
  431. head_list = []
  432. head_list.append(0)
  433. #行表头
  434. is_head_last = False
  435. for i in range(height):
  436. set_match = set()
  437. is_head = False
  438. is_long_value = False
  439. is_same_value = True
  440. same_value = inner_table[i][0][0]
  441. for j in range(width):
  442. if inner_table[i][j][0]!=same_value:
  443. is_same_value = False
  444. break
  445. for j in range(width):
  446. if re.search(pat_value,inner_table[i][j][0]) is not None:
  447. is_head = False
  448. break
  449. str_find = re.findall(pattern,inner_table[i][j][0])
  450. if len(str_find)>0:
  451. set_match.add(inner_table[i][j][0])
  452. if len(set_match)>=count:
  453. is_head = True
  454. if len(inner_table[i][0][0])>40:
  455. is_long_value = True
  456. if is_head or is_long_value or is_same_value:
  457. if not is_head_last:
  458. head_list.append(i)
  459. if is_head:
  460. for j in range(width):
  461. inner_table[i][j][1] = 1
  462. is_head_last = is_head
  463. head_list.append(height)
  464. #列表头
  465. for i in range(len(head_list)-1):
  466. head_begin = head_list[i]
  467. head_end = head_list[i+1]
  468. #最后一列不设置为列表头
  469. for i in range(width-1):
  470. set_match = set()
  471. is_head = False
  472. for j in range(head_begin,head_end):
  473. if re.search(pat_value,inner_table[j][i][0]) is not None:
  474. is_head = False
  475. break
  476. str_find = re.findall(pattern,inner_table[j][i][0])
  477. if len(str_find)>0:
  478. set_match.add(inner_table[j][i][0])
  479. if len(set_match)>=count:
  480. is_head = True
  481. if is_head:
  482. for j in range(head_begin,head_end):
  483. inner_table[j][i][1] = 2
  484. return inner_table,head_list
  485. #取得表格的处理方向
  486. def getDirect(inner_table,begin,end):
  487. '''
  488. column_head = set()
  489. row_head = set()
  490. widths = len(inner_table[0])
  491. for height in range(begin,end):
  492. for width in range(widths):
  493. if inner_table[height][width][1] ==1:
  494. row_head.add(height)
  495. if inner_table[height][width][1] ==2:
  496. column_head.add(width)
  497. company_pattern = re.compile("公司")
  498. if 0 in column_head and begin not in row_head:
  499. return "column"
  500. if 0 in column_head and begin in row_head:
  501. for height in range(begin,end):
  502. count = 0
  503. count_flag = True
  504. for width_index in range(width):
  505. if inner_table[height][width_index][1]==0:
  506. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  507. count += 1
  508. else:
  509. count_flag = False
  510. if count_flag and count>=2:
  511. return "column"
  512. return "row"
  513. '''
  514. count_row_keys = 0
  515. count_column_keys = 0
  516. width = len(inner_table[0])
  517. if begin<end:
  518. for w in range(len(inner_table[begin])):
  519. if inner_table[begin][w][1]!=0:
  520. count_row_keys += 1
  521. for h in range(begin,end):
  522. if inner_table[h][0][1]!=0:
  523. count_column_keys += 1
  524. company_pattern = re.compile("有限(责任)?公司")
  525. for height in range(begin,end):
  526. count_set = set()
  527. count_flag = True
  528. for width_index in range(width):
  529. if inner_table[height][width_index][1]==0:
  530. if re.search(company_pattern,inner_table[height][width_index][0]) is not None:
  531. count_set.add(inner_table[height][width_index][0])
  532. else:
  533. count_flag = False
  534. if count_flag and len(count_set)>=2:
  535. return "column"
  536. if count_column_keys>count_row_keys:
  537. return "column"
  538. return "row"
  539. #根据表格处理方向生成句子,
  540. def getTableText(inner_table,head_list,key_direct=False):
  541. # packPattern = "(标包|[标包][号段名])"
  542. packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则,补充采购类包名
  543. rankPattern = "(排名|排序|名次|序号|评标结果|评审结果)" # 2020/11/23 大网站规则,添加序号为排序
  544. entityPattern = "(候选|([中投]标|报价)(人|单位|候选)|单位名称|供应商)"
  545. height = len(inner_table)
  546. width = len(inner_table[0])
  547. text = ""
  548. for head_i in range(len(head_list)-1):
  549. head_begin = head_list[head_i]
  550. head_end = head_list[head_i+1]
  551. direct = getDirect(inner_table, head_begin, head_end)
  552. #若只有一行,则直接按行读取
  553. if head_end-head_begin==1:
  554. text_line = ""
  555. for i in range(head_begin,head_end):
  556. for w in range(len(inner_table[i])):
  557. if inner_table[i][w][1]==1:
  558. _punctuation = ":"
  559. else:
  560. _punctuation = ","
  561. if w>0:
  562. if inner_table[i][w][0]!= inner_table[i][w-1][0]:
  563. text_line += inner_table[i][w][0]+_punctuation
  564. else:
  565. text_line += inner_table[i][w][0]+_punctuation
  566. text_line = text_line+"。" if text_line!="" else text_line
  567. text += text_line
  568. else:
  569. if direct=="row":
  570. for i in range(head_begin,head_end):
  571. pack_text = ""
  572. rank_text = ""
  573. entity_text = ""
  574. text_line = ""
  575. #在同一句话中重复的可以去掉
  576. text_set = set()
  577. for j in range(width):
  578. cell = inner_table[i][j]
  579. #是属性值
  580. if cell[1]==0 and cell[0]!="":
  581. head = ""
  582. find_flag = False
  583. temp_head = ""
  584. for loop_i in range(0,i+1-head_begin):
  585. if not key_direct:
  586. key_values = [1,2]
  587. else:
  588. key_values = [1]
  589. if inner_table[i-loop_i][j][1] in key_values:
  590. if find_flag:
  591. if inner_table[i-loop_i][j][0]!=temp_head:
  592. head = inner_table[i-loop_i][j][0]+":"+head
  593. else:
  594. head = inner_table[i-loop_i][j][0]+":"+head
  595. find_flag = True
  596. temp_head = inner_table[i-loop_i][j][0]
  597. else:
  598. #找到表头后遇到属性值就返回
  599. if find_flag:
  600. break
  601. find_flag = False
  602. temp_head = ""
  603. for loop_j in range(1,j+1):
  604. if not key_direct:
  605. key_values = [1,2]
  606. else:
  607. key_values = [2]
  608. if inner_table[i][j-loop_j][1] in key_values:
  609. if find_flag:
  610. if inner_table[i][j-loop_j][0]!=temp_head:
  611. head = inner_table[i][j-loop_j][0]+":"+head
  612. else:
  613. head = inner_table[i][j-loop_j][0]+":"+head
  614. find_flag = True
  615. temp_head = inner_table[i][j-loop_j][0]
  616. else:
  617. if find_flag:
  618. break
  619. if str(head+inner_table[i][j][0]) in text_set:
  620. continue
  621. if re.search(packPattern,head) is not None:
  622. pack_text += head+inner_table[i][j][0]+","
  623. elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题,if 改elif
  624. #排名替换为同一种表达
  625. rank_text += head+inner_table[i][j][0]+","
  626. #print(rank_text)
  627. elif re.search(entityPattern,head) is not None:
  628. entity_text += head+inner_table[i][j][0]+","
  629. #print(entity_text)
  630. else:
  631. text_line += head+inner_table[i][j][0]+","
  632. text_set.add(str(head+inner_table[i][j][0]))
  633. text += pack_text+rank_text+entity_text+text_line
  634. text = text[:-1]+"。" if len(text)>0 else text
  635. else:
  636. for j in range(width):
  637. rank_text = ""
  638. entity_text = ""
  639. text_line = ""
  640. text_set = set()
  641. for i in range(head_begin,head_end):
  642. cell = inner_table[i][j]
  643. #是属性值
  644. if cell[1]==0 and cell[0]!="":
  645. find_flag = False
  646. head = ""
  647. temp_head = ""
  648. for loop_j in range(1,j+1):
  649. if not key_direct:
  650. key_values = [1,2]
  651. else:
  652. key_values = [2]
  653. if inner_table[i][j-loop_j][1] in key_values:
  654. if find_flag:
  655. if inner_table[i][j-loop_j][0]!=temp_head:
  656. head = inner_table[i][j-loop_j][0]+":"+head
  657. else:
  658. head = inner_table[i][j-loop_j][0]+":"+head
  659. find_flag = True
  660. temp_head = inner_table[i][j-loop_j][0]
  661. else:
  662. if find_flag:
  663. break
  664. find_flag = False
  665. temp_head = ""
  666. for loop_i in range(0,i+1-head_begin):
  667. if not key_direct:
  668. key_values = [1,2]
  669. else:
  670. key_values = [1]
  671. if inner_table[i-loop_i][j][1] in key_values:
  672. if find_flag:
  673. if inner_table[i-loop_i][j][0]!=temp_head:
  674. head = inner_table[i-loop_i][j][0]+":"+head
  675. else:
  676. head = inner_table[i-loop_i][j][0]+":"+head
  677. find_flag = True
  678. temp_head = inner_table[i-loop_i][j][0]
  679. else:
  680. if find_flag:
  681. break
  682. if str(head+inner_table[i][j][0]) in text_set:
  683. continue
  684. if re.search(rankPattern,head) is not None:
  685. rank_text += head+inner_table[i][j][0]+","
  686. #print(rank_text)
  687. elif re.search(entityPattern,head) is not None:
  688. entity_text += head+inner_table[i][j][0]+","
  689. #print(entity_text)
  690. else:
  691. text_line += head+inner_table[i][j][0]+","
  692. text_set.add(str(head+inner_table[i][j][0]))
  693. text += rank_text+entity_text+text_line
  694. text = text[:-1]+"。" if len(text)>0 else text
  695. return text
  696. def removeFix(inner_table,fix_value="~~"):
  697. height = len(inner_table)
  698. width = len(inner_table[0])
  699. for h in range(height):
  700. for w in range(width):
  701. if inner_table[h][w][0]==fix_value:
  702. inner_table[h][w][0] = ""
  703. def trunTable(tbody):
  704. fixSpan(tbody)
  705. inner_table = getTable(tbody)
  706. inner_table = fixTable(inner_table)
  707. if len(inner_table)>0 and len(inner_table[0])>0:
  708. #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
  709. #inner_table,head_list = setHead_inline(inner_table)
  710. inner_table,head_list = setHead_initem(inner_table,pat_head)
  711. # print(inner_table)
  712. # for begin in range(len(head_list[:-1])):
  713. # for item in inner_table[head_list[begin]:head_list[begin+1]]:
  714. # print(item)
  715. # print("====")
  716. removeFix(inner_table)
  717. '''
  718. print("----")
  719. print(head_list)
  720. for item in inner_table:
  721. print(item)
  722. '''
  723. tbody.string = getTableText(inner_table,head_list)
  724. #print(tbody.string)
  725. tbody.name = "turntable"
  726. return inner_table
  727. return None
  728. pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$')
  729. #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
  730. pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")
  731. list_innerTable = []
  732. tbodies = soup.find_all('table')
  733. # 遍历表格中的每个tbody
  734. #逆序处理嵌套表格
  735. for tbody_index in range(1,len(tbodies)+1):
  736. tbody = tbodies[len(tbodies)-tbody_index]
  737. inner_table = trunTable(tbody)
  738. list_innerTable.append(inner_table)
  739. tbodies = soup.find_all('tbody')
  740. # 遍历表格中的每个tbody
  741. #逆序处理嵌套表格
  742. for tbody_index in range(1,len(tbodies)+1):
  743. tbody = tbodies[len(tbodies)-tbody_index]
  744. inner_table = trunTable(tbody)
  745. list_innerTable.append(inner_table)
  746. return soup
  747. # return list_innerTable
  748. #数据清洗
  749. def segment(soup):
  750. # print("==")
  751. # print(soup)
  752. # print("====")
  753. #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
  754. if soup.name=="td":
  755. #判断有值叶子节点数
  756. _count = 0
  757. for child in soup.find_all(recursive=True):
  758. if child.get_text().strip()!="" and len(child.find_all())==0:
  759. _count += 1
  760. if _count<=1:
  761. text = soup.get_text()
  762. # 2020/11/24 大网站规则添加
  763. if 'title' in soup.attrs:
  764. if '...' in soup.get_text() and (soup.get_text()[:-3]).strip() in soup.attrs['title']:
  765. text = soup.attrs['title']
  766. _list = []
  767. for x in re.split("\s+",text):
  768. if x.strip()!="":
  769. _list.append(len(x))
  770. if len(_list)>0:
  771. _minLength = min(_list)
  772. if _minLength>2:
  773. _substr = ","
  774. else:
  775. _substr = ""
  776. else:
  777. _substr = ""
  778. text = _substr.join(re.split("(\s+)",text))
  779. text = text.replace("\r\n",",").replace("\n",",")
  780. text = re.sub("^[,\s]*|[,\s]*$","",text)
  781. return text
  782. segList = ["title"]
  783. commaList = ["div","br","td","p"]
  784. #commaList = []
  785. spaceList = ["span"]
  786. subspaceList = ["td",'a',"span","p"]
  787. tbodies = soup.find_all('tbody')
  788. if len(tbodies) == 0:
  789. tbodies = soup.find_all('table')
  790. # 递归遍历所有节点,插入符号
  791. for child in soup.find_all(recursive=True):
  792. if child.name in segList:
  793. child.insert_after("。")
  794. if child.name in commaList:
  795. child.insert_after(",")
  796. # if child.name in subspaceList:
  797. # child.insert_before("#subs"+str(child.name)+"#")
  798. # child.insert_after("#sube"+str(child.name)+"#")
  799. if child.name in spaceList:
  800. child.insert_after(" ")
  801. text = str(soup.get_text())
  802. #替换英文冒号为中文冒号
  803. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  804. #替换为中文逗号
  805. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  806. #替换为中文分号
  807. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  808. #删除标签中的所有空格
  809. for subs in subspaceList:
  810. patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
  811. while(True):
  812. oneMatch = re.search(re.compile(patten),text)
  813. if oneMatch is not None:
  814. _match = oneMatch.group(1)
  815. _minLength = min([len(x) for x in re.split("(\s*)",_match)])
  816. if _minLength>2:
  817. _substr = ","
  818. else:
  819. _substr = ""
  820. text = text.replace("#subs"+str(subs)+"#"+oneMatch.group(1)+"#sube"+str(subs)+"#",re.sub("\s",_substr,oneMatch.group(1)))
  821. else:
  822. break
  823. #替换"""为"“",否则导入deepdive出错
  824. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  825. text = re.sub("\s{4,}",",",text)
  826. #替换标点
  827. while(True):
  828. #替换连续的标点
  829. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  830. if punc is not None:
  831. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  832. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  833. if punc is not None:
  834. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  835. else:
  836. #替换标点之后的空格
  837. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  838. if punc is not None:
  839. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  840. else:
  841. break
  842. #将连续的中文句号替换为一个
  843. text_split = text.split("。")
  844. text_split = [x for x in text_split if len(x)>0]
  845. list_text = []
  846. # for _t in text_split:
  847. # list_text.append(re.sub(")",")",re.sub("(","(",re.sub("\s*","",_t))))
  848. text = "。".join(text_split)
  849. # text = text.replace(')',")").replace("(","(").replace("\s","")
  850. #删除所有空格
  851. # text过大报错
  852. LOOP_LEN = 10000
  853. LOOP_BEGIN = 0
  854. _text = ""
  855. if len(text)<10000000:
  856. while(LOOP_BEGIN<len(text)):
  857. _text += re.sub(")",")",re.sub("(","(",re.sub("\s*","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
  858. LOOP_BEGIN += LOOP_LEN
  859. else:
  860. return text
  861. # text = re.sub("\s*","",text)
  862. # #替换中文括号为英文括号
  863. # text = re.sub("(","(",text)
  864. # text = re.sub(")",")",text)
  865. return _text
  866. '''
  867. #数据清洗
  868. def segment(soup):
  869. segList = ["title"]
  870. commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]
  871. spaceList = ["span"]
  872. tbodies = soup.find_all('tbody')
  873. if len(tbodies) == 0:
  874. tbodies = soup.find_all('table')
  875. # 递归遍历所有节点,插入符号
  876. for child in soup.find_all(recursive=True):
  877. if child.name == 'br':
  878. child.insert_before(',')
  879. child_text = re.sub('\s', '', child.get_text())
  880. if child_text == '' or child_text[-1] in ['。',',',':',';']:
  881. continue
  882. if child.name in segList:
  883. child.insert_after("。")
  884. if child.name in commaList:
  885. if len(child_text)>3 and len(child_text) <50: # 先判断是否字数少于50,成立加逗号,否则加句号
  886. child.insert_after(",")
  887. elif len(child_text) >=50:
  888. child.insert_after("。")
  889. #if child.name in spaceList:
  890. #child.insert_after(" ")
  891. text = str(soup.get_text())
  892. text = re.sub("\s{5,}",",",text)
  893. text = text.replace('"',"“").replace("\r","").replace("\n",",")
  894. #替换"""为"“",否则导入deepdive出错
  895. text = text.replace('"',"“")
  896. #text = text.replace('"',"“").replace("\r","").replace("\n","")
  897. #删除所有空格
  898. text = re.sub("\s+","#nbsp#",text)
  899. text_list = text.split('#nbsp#')
  900. new_text = ''
  901. for i in range(len(text_list)-1):
  902. if text_list[i] == '' or text_list[i][-1] in [',','。',';',':']:
  903. new_text += text_list[i]
  904. elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
  905. new_text += text_list[i] + '。'
  906. elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
  907. new_text += text_list[i] + ';'
  908. elif text_list[i].isdigit() and text_list[i+1].isdigit():
  909. new_text += text_list[i] + ' '
  910. elif text_list[i][-1] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','¥'] or text_list[i+1][0] in ['-',':','(',')','/','(',')','——','年','月','日','时','分','元','万元']:
  911. new_text += text_list[i]
  912. elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3:
  913. new_text += text_list[i] + ','
  914. else:
  915. new_text += text_list[i]
  916. new_text += text_list[-1]
  917. text = new_text
  918. #替换英文冒号为中文冒号
  919. text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])",":",text)
  920. #替换为中文逗号
  921. text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])",",",text)
  922. #替换为中文分号
  923. text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])",";",text)
  924. #替换标点
  925. while(True):
  926. #替换连续的标点
  927. punc = re.search(",(?P<punc>:|。|,|;)\s*",text)
  928. if punc is not None:
  929. text = re.sub(","+punc.group("punc")+"\s*",punc.group("punc"),text)
  930. punc = re.search("(?P<punc>:|。|,|;)\s*,",text)
  931. if punc is not None:
  932. text = re.sub(punc.group("punc")+"\s*,",punc.group("punc"),text)
  933. else:
  934. #替换标点之后的空格
  935. punc = re.search("(?P<punc>:|。|,|;)\s+",text)
  936. if punc is not None:
  937. text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
  938. else:
  939. break
  940. #将连续的中文句号替换为一个
  941. text_split = text.split("。")
  942. text_split = [x for x in text_split if len(x)>0]
  943. text = "。".join(text_split)
  944. #替换中文括号为英文括号
  945. text = re.sub("(","(",text)
  946. text = re.sub(")",")",text)
  947. return text
  948. '''
  949. #连续实体合并(弃用)
  950. def union_ner(list_ner):
  951. result_list = []
  952. union_index = []
  953. union_index_set = set()
  954. for i in range(len(list_ner)-1):
  955. if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
  956. if list_ner[i][1]-list_ner[i+1][0]==1:
  957. union_index_set.add(i)
  958. union_index_set.add(i+1)
  959. union_index.append((i,i+1))
  960. for i in range(len(list_ner)):
  961. if i not in union_index_set:
  962. result_list.append(list_ner[i])
  963. for item in union_index:
  964. #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
  965. result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
  966. return result_list
  967. def getTokensAndNers(sentences,MAXAREA = 10000,useselffool=False):
  968. '''
  969. @param: sentences:句子数
  970. @return 限流执行后的分词和实体识别list
  971. '''
  972. def getData(tokens,ners,process_data):
  973. process_sentences = [item[1] for item in process_data]
  974. token_ = selffool.cut(process_sentences)
  975. if useselffool:
  976. ner_ = selffool.self_ner(process_sentences)
  977. else:
  978. ner_ = selffool.ner(process_sentences)
  979. for i in range(len(token_)):
  980. the_index = process_data[i][0]
  981. tokens[the_index] = token_[i]
  982. ners[the_index] = ner_[i]
  983. sents = []
  984. for i in range(len(sentences)):
  985. sents.append([i,sentences[i]])
  986. sents.sort(key=lambda x:len(x[1]),reverse=True)
  987. index_ = 0
  988. tokens = [[]for i in range(len(sentences))]
  989. ners = [[]for i in range(len(sentences))]
  990. while(True):
  991. width = len(sents[index_][1])
  992. height = MAXAREA//width+1
  993. if height>len(sents)-index_:
  994. height = len(sents)-index_
  995. process_data = sents[index_:index_+height]
  996. getData(tokens, ners, process_data)
  997. index_ += height
  998. if index_>=len(sents):
  999. break
  1000. return tokens,ners
  1001. def getTokens(sentences,MAXAREA = 10000,useselffool=True):
  1002. '''
  1003. @param: sentences:句子数
  1004. @return 限流执行后的分词list
  1005. '''
  1006. def getData(tokens,process_data):
  1007. process_sentences = [item[1] for item in process_data]
  1008. token_ = selffool.cut(process_sentences)
  1009. for i in range(len(token_)):
  1010. the_index = process_data[i][0]
  1011. tokens[the_index] = token_[i]
  1012. sents = []
  1013. for i in range(len(sentences)):
  1014. sents.append([i,sentences[i]])
  1015. sents.sort(key=lambda x:len(x[1]),reverse=True)
  1016. index_ = 0
  1017. tokens = [[]for i in range(len(sentences))]
  1018. while(True):
  1019. width = len(sents[index_][1])
  1020. height = MAXAREA//width+1
  1021. if height>len(sents)-index_:
  1022. height = len(sents)-index_
  1023. process_data = sents[index_:index_+height]
  1024. getData(tokens, process_data)
  1025. index_ += height
  1026. if index_>=len(sents):
  1027. break
  1028. return tokens
  1029. def getNers(sentences,MAXAREA = 10000,useselffool=False):
  1030. '''
  1031. @param: sentences:句子数
  1032. @return 限流执行后的实体识别list
  1033. '''
  1034. def getData(ners,process_data):
  1035. process_sentences = [item[1] for item in process_data]
  1036. if useselffool:
  1037. ner_ = selffool.self_ner(process_sentences)
  1038. else:
  1039. ner_ = selffool.ner(process_sentences)
  1040. for i in range(len(ner_)):
  1041. the_index = process_data[i][0]
  1042. ners[the_index] = ner_[i]
  1043. sents = []
  1044. for i in range(len(sentences)):
  1045. sents.append([i,sentences[i]])
  1046. sents.sort(key=lambda x:len(x[1]),reverse=True)
  1047. index_ = 0
  1048. ners = [[]for i in range(len(sentences))]
  1049. while(True):
  1050. width = len(sents[index_][1])
  1051. height = MAXAREA//width+1
  1052. if height>len(sents)-index_:
  1053. height = len(sents)-index_
  1054. process_data = sents[index_:index_+height]
  1055. getData( ners, process_data)
  1056. index_ += height
  1057. if index_>=len(sents):
  1058. break
  1059. return ners
  1060. # def get_preprocessed(articles,useselffool=False):
  1061. # '''
  1062. # @summary:预处理步骤,NLP处理、实体识别
  1063. # @param:
  1064. # articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1065. # @return:list of articles,list of each article of sentences,list of each article of entitys
  1066. # '''
  1067. # list_articles = []
  1068. # list_sentences = []
  1069. # list_entitys = []
  1070. # cost_time = dict()
  1071. # for article in articles:
  1072. # list_sentences_temp = []
  1073. # list_entitys_temp = []
  1074. # doc_id = article[0]
  1075. # sourceContent = article[1]
  1076. # _send_doc_id = article[3]
  1077. # _title = article[4]
  1078. # #表格处理
  1079. # key_preprocess = "tableToText"
  1080. # start_time = time.time()
  1081. # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1082. #
  1083. # # log(article_processed)
  1084. #
  1085. # if key_preprocess not in cost_time:
  1086. # cost_time[key_preprocess] = 0
  1087. # cost_time[key_preprocess] += time.time()-start_time
  1088. #
  1089. # #article_processed = article[1]
  1090. # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1091. # #nlp处理
  1092. # if article_processed is not None and len(article_processed)!=0:
  1093. # split_patten = "。"
  1094. # sentences = []
  1095. # _begin = 0
  1096. # for _iter in re.finditer(split_patten,article_processed):
  1097. # sentences.append(article_processed[_begin:_iter.span()[1]])
  1098. # _begin = _iter.span()[1]
  1099. # sentences.append(article_processed[_begin:])
  1100. #
  1101. # lemmas = []
  1102. # doc_offsets = []
  1103. # dep_types = []
  1104. # dep_tokens = []
  1105. #
  1106. # time1 = time.time()
  1107. #
  1108. # '''
  1109. # tokens_all = fool.cut(sentences)
  1110. # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1111. # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1112. # ner_entitys_all = fool.ner(sentences)
  1113. # '''
  1114. # #限流执行
  1115. # key_nerToken = "nerToken"
  1116. # start_time = time.time()
  1117. # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
  1118. # if key_nerToken not in cost_time:
  1119. # cost_time[key_nerToken] = 0
  1120. # cost_time[key_nerToken] += time.time()-start_time
  1121. #
  1122. #
  1123. # for sentence_index in range(len(sentences)):
  1124. #
  1125. #
  1126. #
  1127. # list_sentence_entitys = []
  1128. # sentence_text = sentences[sentence_index]
  1129. # tokens = tokens_all[sentence_index]
  1130. #
  1131. # list_tokenbegin = []
  1132. # begin = 0
  1133. # for i in range(0,len(tokens)):
  1134. # list_tokenbegin.append(begin)
  1135. # begin += len(str(tokens[i]))
  1136. # list_tokenbegin.append(begin+1)
  1137. # #pos_tag = pos_all[sentence_index]
  1138. # pos_tag = ""
  1139. #
  1140. # ner_entitys = ner_entitys_all[sentence_index]
  1141. #
  1142. # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1143. #
  1144. # #识别package
  1145. #
  1146. #
  1147. # #识别实体
  1148. # for ner_entity in ner_entitys:
  1149. # begin_index_temp = ner_entity[0]
  1150. # end_index_temp = ner_entity[1]
  1151. # entity_type = ner_entity[2]
  1152. # entity_text = ner_entity[3]
  1153. #
  1154. # for j in range(len(list_tokenbegin)):
  1155. # if list_tokenbegin[j]==begin_index_temp:
  1156. # begin_index = j
  1157. # break
  1158. # elif list_tokenbegin[j]>begin_index_temp:
  1159. # begin_index = j-1
  1160. # break
  1161. # begin_index_temp += len(str(entity_text))
  1162. # for j in range(begin_index,len(list_tokenbegin)):
  1163. # if list_tokenbegin[j]>=begin_index_temp:
  1164. # end_index = j-1
  1165. # break
  1166. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1167. #
  1168. # #去掉标点符号
  1169. # entity_text = re.sub("[,,。:]","",entity_text)
  1170. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1171. #
  1172. #
  1173. # #使用正则识别金额
  1174. # entity_type = "money"
  1175. #
  1176. # #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1177. #
  1178. # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
  1179. # "key_word":"((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1180. # "front_m":"((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*",
  1181. # "behind_m":"(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*"}
  1182. #
  1183. # set_begin = set()
  1184. # for pattern_key in list_money_pattern.keys():
  1185. # pattern = re.compile(list_money_pattern[pattern_key])
  1186. # all_match = re.findall(pattern, sentence_text)
  1187. # index = 0
  1188. # for i in range(len(all_match)):
  1189. # if len(all_match[i][0])>0:
  1190. # # print("===",all_match[i])
  1191. # #print(all_match[i][0])
  1192. # unit = ""
  1193. # entity_text = all_match[i][3]
  1194. # if pattern_key in ["key_word","front_m"]:
  1195. # unit = all_match[i][1]
  1196. # else:
  1197. # unit = all_match[i][4]
  1198. # if entity_text.find("元")>=0:
  1199. # unit = ""
  1200. #
  1201. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1202. #
  1203. # begin_index_temp = index
  1204. # for j in range(len(list_tokenbegin)):
  1205. # if list_tokenbegin[j]==index:
  1206. # begin_index = j
  1207. # break
  1208. # elif list_tokenbegin[j]>index:
  1209. # begin_index = j-1
  1210. # break
  1211. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1212. # end_index_temp = index
  1213. # #index += len(str(all_match[i][0]))
  1214. # for j in range(begin_index,len(list_tokenbegin)):
  1215. # if list_tokenbegin[j]>=index:
  1216. # end_index = j-1
  1217. # break
  1218. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1219. #
  1220. #
  1221. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1222. # if len(unit)>0:
  1223. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1224. # else:
  1225. # entity_text = str(getUnifyMoney(entity_text))
  1226. #
  1227. # _exists = False
  1228. # for item in list_sentence_entitys:
  1229. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1230. # _exists = True
  1231. # if not _exists:
  1232. # if float(entity_text)>10:
  1233. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1234. #
  1235. # else:
  1236. # index += 1
  1237. #
  1238. # list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1239. # list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1240. # list_sentences.append(list_sentences_temp)
  1241. # list_entitys.append(list_entitys_temp)
  1242. # return list_articles,list_sentences,list_entitys,cost_time
  1243. def get_preprocessed(articles,useselffool=False):
  1244. '''
  1245. @summary:预处理步骤,NLP处理、实体识别
  1246. @param:
  1247. articles:待处理的文章list [[id,source,jointime,doc_id,title]]
  1248. @return:list of articles,list of each article of sentences,list of each article of entitys
  1249. '''
  1250. cost_time = dict()
  1251. list_articles = get_preprocessed_article(articles,cost_time)
  1252. list_sentences = get_preprocessed_sentences(list_articles,True,cost_time)
  1253. list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)
  1254. return list_articles,list_sentences,list_entitys,cost_time
  1255. def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
  1256. '''
  1257. :param articles: 待处理的article source html
  1258. :param useselffool: 是否使用selffool
  1259. :return: list_articles
  1260. '''
  1261. list_articles = []
  1262. for article in articles:
  1263. doc_id = article[0]
  1264. sourceContent = article[1]
  1265. _send_doc_id = article[3]
  1266. _title = article[4]
  1267. #表格处理
  1268. key_preprocess = "tableToText"
  1269. start_time = time.time()
  1270. article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
  1271. # log(article_processed)
  1272. if key_preprocess not in cost_time:
  1273. cost_time[key_preprocess] = 0
  1274. cost_time[key_preprocess] += time.time()-start_time
  1275. #article_processed = article[1]
  1276. list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
  1277. return list_articles
  1278. def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
  1279. '''
  1280. :param list_articles: 经过预处理的article text
  1281. :return: list_sentences
  1282. '''
  1283. list_sentences = []
  1284. for article in list_articles:
  1285. list_sentences_temp = []
  1286. list_entitys_temp = []
  1287. doc_id = article.id
  1288. _send_doc_id = article.doc_id
  1289. _title = article.title
  1290. #表格处理
  1291. key_preprocess = "tableToText"
  1292. start_time = time.time()
  1293. article_processed = article.content
  1294. if key_preprocess not in cost_time:
  1295. cost_time[key_preprocess] = 0
  1296. cost_time[key_preprocess] += time.time()-start_time
  1297. #nlp处理
  1298. if article_processed is not None and len(article_processed)!=0:
  1299. split_patten = "。"
  1300. sentences = []
  1301. _begin = 0
  1302. sentences_set = set()
  1303. for _iter in re.finditer(split_patten,article_processed):
  1304. _sen = article_processed[_begin:_iter.span()[1]]
  1305. if len(_sen)>0 and _sen not in sentences_set:
  1306. sentences.append(_sen)
  1307. sentences_set.add(_sen)
  1308. _begin = _iter.span()[1]
  1309. _sen = article_processed[_begin:]
  1310. if len(_sen)>0 and _sen not in sentences_set:
  1311. sentences.append(_sen)
  1312. sentences_set.add(_sen)
  1313. article.content = "".join(sentences)
  1314. # sentences.append(article_processed[_begin:])
  1315. lemmas = []
  1316. doc_offsets = []
  1317. dep_types = []
  1318. dep_tokens = []
  1319. time1 = time.time()
  1320. '''
  1321. tokens_all = fool.cut(sentences)
  1322. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1323. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1324. ner_entitys_all = fool.ner(sentences)
  1325. '''
  1326. #限流执行
  1327. key_nerToken = "nerToken"
  1328. start_time = time.time()
  1329. tokens_all = getTokens(sentences,useselffool=useselffool)
  1330. if key_nerToken not in cost_time:
  1331. cost_time[key_nerToken] = 0
  1332. cost_time[key_nerToken] += time.time()-start_time
  1333. for sentence_index in range(len(sentences)):
  1334. sentence_text = sentences[sentence_index]
  1335. tokens = tokens_all[sentence_index]
  1336. #pos_tag = pos_all[sentence_index]
  1337. pos_tag = ""
  1338. ner_entitys = ""
  1339. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
  1340. if len(list_sentences_temp)==0:
  1341. list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
  1342. list_sentences.append(list_sentences_temp)
  1343. return list_sentences
  1344. def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
  1345. '''
  1346. :param list_sentences:分局情况
  1347. :param cost_time:
  1348. :return: list_entitys
  1349. '''
  1350. list_entitys = []
  1351. for list_sentence in list_sentences:
  1352. sentences = []
  1353. list_entitys_temp = []
  1354. for _sentence in list_sentence:
  1355. sentences.append(_sentence.sentence_text)
  1356. lemmas = []
  1357. doc_offsets = []
  1358. dep_types = []
  1359. dep_tokens = []
  1360. time1 = time.time()
  1361. '''
  1362. tokens_all = fool.cut(sentences)
  1363. #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
  1364. #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
  1365. ner_entitys_all = fool.ner(sentences)
  1366. '''
  1367. #限流执行
  1368. key_nerToken = "nerToken"
  1369. start_time = time.time()
  1370. ner_entitys_all = getNers(sentences,useselffool=useselffool)
  1371. if key_nerToken not in cost_time:
  1372. cost_time[key_nerToken] = 0
  1373. cost_time[key_nerToken] += time.time()-start_time
  1374. for sentence_index in range(len(list_sentence)):
  1375. list_sentence_entitys = []
  1376. sentence_text = list_sentence[sentence_index].sentence_text
  1377. tokens = list_sentence[sentence_index].tokens
  1378. doc_id = list_sentence[sentence_index].doc_id
  1379. list_tokenbegin = []
  1380. begin = 0
  1381. for i in range(0,len(tokens)):
  1382. list_tokenbegin.append(begin)
  1383. begin += len(str(tokens[i]))
  1384. list_tokenbegin.append(begin+1)
  1385. #pos_tag = pos_all[sentence_index]
  1386. pos_tag = ""
  1387. ner_entitys = ner_entitys_all[sentence_index]
  1388. #识别package
  1389. #识别实体
  1390. for ner_entity in ner_entitys:
  1391. begin_index_temp = ner_entity[0]
  1392. end_index_temp = ner_entity[1]
  1393. entity_type = ner_entity[2]
  1394. entity_text = ner_entity[3]
  1395. for j in range(len(list_tokenbegin)):
  1396. if list_tokenbegin[j]==begin_index_temp:
  1397. begin_index = j
  1398. break
  1399. elif list_tokenbegin[j]>begin_index_temp:
  1400. begin_index = j-1
  1401. break
  1402. begin_index_temp += len(str(entity_text))
  1403. for j in range(begin_index,len(list_tokenbegin)):
  1404. if list_tokenbegin[j]>=begin_index_temp:
  1405. end_index = j-1
  1406. break
  1407. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1408. #去掉标点符号
  1409. entity_text = re.sub("[,,。:]","",entity_text)
  1410. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
  1411. #使用正则识别金额
  1412. entity_type = "money"
  1413. #money_patten_str = "(([1-9][\d,,]*(?:\.\d+)?[百千万亿]?[\(\)()元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[¥¥]+,?|报价|标价)[(\(]?([万])?元?[)\)]?[::]?.{,7}?([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)|([1-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]?)[\((]?([万元]{1,2}))*"
  1414. list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
  1415. "key_word":"((?P<text_key_word>(?:[¥¥]+,?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果:)(?:[,(\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[)\)]?)\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[)\)]?))",
  1416. "front_m":"((?P<text_front_m>(?:[(\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[)\)])\s*[,,::]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())",
  1417. "behind_m":"(()()(?P<money_behind_m>[0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\))]?)"}
  1418. pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
  1419. set_begin = set()
  1420. # for pattern_key in list_money_pattern.keys():
  1421. # for pattern_key in ["cn","key_word","behind_m","front_m"]:
  1422. # # pattern = re.compile(list_money_pattern[pattern_key])
  1423. # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[¥¥]+,?|[报标限]价|金额)(?:[(\(]?\s*([万元]*)\s*[)\)]?)\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)(?:[(\(]?\s*([万元]*)\s*[)\)]?))*|(()()([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿]*)[\((]?([万元]+)[\))]?)*|((?:[(\(]?\s*([万元]+)\s*[)\)])\s*[::]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,,]*(?:\.\d+)?(?:,?)[百千万亿元]*)())*")
  1424. # all_match = re.findall(pattern, sentence_text)
  1425. # index = 0
  1426. # for i in range(len(all_match)):
  1427. # if len(all_match[i][0])>0:
  1428. # print("===",all_match[i])
  1429. # #print(all_match[i][0])
  1430. # unit = ""
  1431. # entity_text = all_match[i][3]
  1432. # if pattern_key in ["key_word","front_m"]:
  1433. # unit = all_match[i][1]
  1434. # if pattern_key=="key_word":
  1435. # if all_match[i][1]=="" and all_match[i][4]!="":
  1436. # unit = all_match[i][4]
  1437. # else:
  1438. # unit = all_match[i][4]
  1439. # if entity_text.find("元")>=0:
  1440. # unit = ""
  1441. #
  1442. # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体,否则会丢失特征
  1443. # begin_index_temp = index
  1444. # for j in range(len(list_tokenbegin)):
  1445. # if list_tokenbegin[j]==index:
  1446. # begin_index = j
  1447. # break
  1448. # elif list_tokenbegin[j]>index:
  1449. # begin_index = j-1
  1450. # break
  1451. # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
  1452. # end_index_temp = index
  1453. # #index += len(str(all_match[i][0]))
  1454. # for j in range(begin_index,len(list_tokenbegin)):
  1455. # if list_tokenbegin[j]>=index:
  1456. # end_index = j-1
  1457. # break
  1458. # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1459. #
  1460. # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
  1461. # if len(unit)>0:
  1462. # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1463. # else:
  1464. # entity_text = str(getUnifyMoney(entity_text))
  1465. #
  1466. # _exists = False
  1467. # for item in list_sentence_entitys:
  1468. # if item.entity_id==entity_id and item.entity_type==entity_type:
  1469. # _exists = True
  1470. # if not _exists:
  1471. # if float(entity_text)>1:
  1472. # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1473. #
  1474. # else:
  1475. # index += 1
  1476. all_match = re.finditer(pattern_money, sentence_text)
  1477. index = 0
  1478. for _match in all_match:
  1479. if len(_match.group())>0:
  1480. # print("===",_match.group())
  1481. # print(_match.groupdict())
  1482. unit = ""
  1483. entity_text = ""
  1484. text_beforeMoney = ""
  1485. filter = ""
  1486. filter_unit = False
  1487. for k,v in _match.groupdict().items():
  1488. if v!="" and v is not None:
  1489. if k.split("_")[0]=="money":
  1490. entity_text = v
  1491. if k.split("_")[0]=="unit":
  1492. unit = v
  1493. if k.split("_")[0]=="text":
  1494. text_beforeMoney = v
  1495. if k.split("_")[0]=="filter":
  1496. filter = v
  1497. if re.search("filter_unit",k) is not None:
  1498. filter_unit = True
  1499. if entity_text.find("元")>=0:
  1500. unit = ""
  1501. else:
  1502. if filter_unit:
  1503. continue
  1504. if filter!="":
  1505. continue
  1506. index = _match.span()[0]+len(text_beforeMoney)
  1507. begin_index_temp = index
  1508. for j in range(len(list_tokenbegin)):
  1509. if list_tokenbegin[j]==index:
  1510. begin_index = j
  1511. break
  1512. elif list_tokenbegin[j]>index:
  1513. begin_index = j-1
  1514. break
  1515. index = _match.span()[1]
  1516. end_index_temp = index
  1517. #index += len(str(all_match[i][0]))
  1518. for j in range(begin_index,len(list_tokenbegin)):
  1519. if list_tokenbegin[j]>=index:
  1520. end_index = j-1
  1521. break
  1522. entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
  1523. entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
  1524. if len(unit)>0:
  1525. entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
  1526. else:
  1527. entity_text = str(getUnifyMoney(entity_text))
  1528. if float(entity_text)<100 or float(entity_text)>100000000000:
  1529. continue
  1530. _exists = False
  1531. for item in list_sentence_entitys:
  1532. if item.entity_id==entity_id and item.entity_type==entity_type:
  1533. _exists = True
  1534. if not _exists:
  1535. if float(entity_text)>1:
  1536. list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
  1537. else:
  1538. index += 1
  1539. # 资金来源提取 2020/12/30 新增
  1540. list_moneySource = extract_moneySource(sentence_text)
  1541. entity_type = "moneySource"
  1542. for moneySource in list_moneySource:
  1543. begin_index_temp = moneySource['begin_index']
  1544. for j in range(len(list_tokenbegin)):
  1545. if list_tokenbegin[j] == begin_index_temp:
  1546. begin_index = j
  1547. break
  1548. elif list_tokenbegin[j] > begin_index_temp:
  1549. begin_index = j - 1
  1550. break
  1551. index = moneySource['end_index']
  1552. end_index_temp = index
  1553. for j in range(begin_index, len(list_tokenbegin)):
  1554. if list_tokenbegin[j] >= index:
  1555. end_index = j - 1
  1556. break
  1557. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1558. entity_text = moneySource['body']
  1559. list_sentence_entitys.append(
  1560. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1561. begin_index_temp, end_index_temp))
  1562. # 服务期限提取 2020/12/30 新增
  1563. list_servicetime = extract_servicetime(sentence_text)
  1564. entity_type = "servicetime"
  1565. for servicetime in list_servicetime:
  1566. begin_index_temp = servicetime['begin_index']
  1567. for j in range(len(list_tokenbegin)):
  1568. if list_tokenbegin[j] == begin_index_temp:
  1569. begin_index = j
  1570. break
  1571. elif list_tokenbegin[j] > begin_index_temp:
  1572. begin_index = j - 1
  1573. break
  1574. index = servicetime['end_index']
  1575. end_index_temp = index
  1576. for j in range(begin_index, len(list_tokenbegin)):
  1577. if list_tokenbegin[j] >= index:
  1578. end_index = j - 1
  1579. break
  1580. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1581. entity_text = servicetime['body']
  1582. list_sentence_entitys.append(
  1583. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1584. begin_index_temp, end_index_temp))
  1585. # 招标方式提取 2020/12/30 新增
  1586. list_bidway = extract_bidway(sentence_text)
  1587. entity_type = "bidway"
  1588. for bidway in list_bidway:
  1589. begin_index_temp = bidway['begin_index']
  1590. end_index_temp = bidway['end_index']
  1591. begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
  1592. end_index = changeIndexFromWordToWords(tokens, end_index_temp)
  1593. entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
  1594. entity_text = bidway['body']
  1595. list_sentence_entitys.append(
  1596. Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
  1597. begin_index_temp, end_index_temp))
  1598. list_sentence_entitys.sort(key=lambda x:x.begin_index)
  1599. list_entitys_temp = list_entitys_temp+list_sentence_entitys
  1600. list_entitys.append(list_entitys_temp)
  1601. return list_entitys
  1602. def union_result(codeName,prem):
  1603. '''
  1604. @summary:模型的结果拼成字典
  1605. @param:
  1606. codeName:编号名称模型的结果字典
  1607. prem:拿到属性的角色的字典
  1608. @return:拼接起来的字典
  1609. '''
  1610. result = []
  1611. assert len(codeName)==len(prem)
  1612. for item_code,item_prem in zip(codeName,prem):
  1613. if item_code[0]==item_prem[0]:
  1614. result.append([item_code[0],dict(item_code[1],**item_prem[1])])
  1615. return result
  1616. def persistenceData(data):
  1617. '''
  1618. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  1619. '''
  1620. import psycopg2
  1621. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  1622. cursor = conn.cursor()
  1623. for item_index in range(len(data)):
  1624. item = data[item_index]
  1625. doc_id = item[0]
  1626. dic = item[1]
  1627. code = dic['code']
  1628. name = dic['name']
  1629. prem = dic['prem']
  1630. if len(code)==0:
  1631. code_insert = ""
  1632. else:
  1633. code_insert = ";".join(code)
  1634. prem_insert = ""
  1635. for item in prem:
  1636. for x in item:
  1637. if isinstance(x, list):
  1638. if len(x)>0:
  1639. for x1 in x:
  1640. prem_insert+="/".join(x1)+","
  1641. prem_insert+="$"
  1642. else:
  1643. prem_insert+=str(x)+"$"
  1644. prem_insert+=";"
  1645. sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
  1646. cursor.execute(sql)
  1647. conn.commit()
  1648. conn.close()
  1649. def persistenceData1(list_entitys,list_sentences):
  1650. '''
  1651. @summary:将中间结果保存到数据库-线上生产的时候不需要执行
  1652. '''
  1653. import psycopg2
  1654. conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
  1655. cursor = conn.cursor()
  1656. for list_entity in list_entitys:
  1657. for entity in list_entity:
  1658. if entity.values is not None:
  1659. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
  1660. else:
  1661. sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
  1662. cursor.execute(sql)
  1663. for list_sentence in list_sentences:
  1664. for sentence in list_sentence:
  1665. str_tokens = "["
  1666. for item in sentence.tokens:
  1667. str_tokens += "'"
  1668. if item=="'":
  1669. str_tokens += "''"
  1670. else:
  1671. str_tokens += item
  1672. str_tokens += "',"
  1673. str_tokens = str_tokens[:-1]+"]"
  1674. sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
  1675. cursor.execute(sql)
  1676. conn.commit()
  1677. conn.close()
  1678. def _handle(item,result_queue):
  1679. dochtml = item["dochtml"]
  1680. docid = item["docid"]
  1681. list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
  1682. flag = False
  1683. if list_innerTable:
  1684. flag = True
  1685. for table in list_innerTable:
  1686. result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})
  1687. def getPredictTable():
  1688. filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
  1689. import pandas as pd
  1690. import json
  1691. from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
  1692. df = pd.read_csv(filename)
  1693. df_data = {"json_table":[],"docid":[]}
  1694. _count = 0
  1695. _sum = len(df["docid"])
  1696. task_queue = Queue()
  1697. result_queue = Queue()
  1698. _index = 0
  1699. for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
  1700. task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
  1701. _index += 1
  1702. mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
  1703. mh.run()
  1704. while True:
  1705. try:
  1706. item = result_queue.get(block=True,timeout=1)
  1707. df_data["docid"].append(item["docid"])
  1708. df_data["json_table"].append(item["json_table"])
  1709. except Exception as e:
  1710. print(e)
  1711. break
  1712. df_1 = pd.DataFrame(df_data)
  1713. df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])
  1714. if __name__=="__main__":
  1715. '''
  1716. import glob
  1717. for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
  1718. file_txt = str(file).replace("html","txt")
  1719. with codecs.open(file_txt,"a+",encoding="utf8") as f:
  1720. f.write("\n================\n")
  1721. content = codecs.open(file,"r",encoding="utf8").read()
  1722. f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
  1723. '''
  1724. # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
  1725. # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
  1726. getPredictTable()