对结果json非法字符进行去除

.gitignore

@@ -8,8 +8,7 @@
 /BiddingKG/dl_dev/test/traindata/
 /BiddingKG/dl/product/data/
 /BiddingKG/dl/channel/data/
-/BiddingKG/dl/test/
-/BiddingKG/dl_dev/test/
+/BiddingKG/dl_dev/test
 node_modules
 /BiddingKG/dl/table_head/train_data/
 /BiddingKG/dl/table_head/check_user_result/

BiddingKG/dl/interface/extract.py

@@ -264,7 +264,9 @@ def predict(doc_id,text,title="",page_time="",web_source_no='',web_source_name="
     #         log("type:%s,text:%s,label:%s,values:%s,sentence:%s,begin_index:%s,end_index:%s"%
     #               (str(_entity.entity_type),str(_entity.entity_text),str(_entity.label),str(_entity.values),str(_entity.sentence_index),
     #                str(_entity.begin_index),str(_entity.end_index)))
-    return json.dumps(data_res,cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)#, list_articles[0].content, get_ent_context(list_sentences, list_entitys)
+    _extract_json = json.dumps(data_res,cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)
+    _extract_json = _extract_json.replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '')
+    return _extract_json#, list_articles[0].content, get_ent_context(list_sentences, list_entitys)
 
 
 def test(name,content):

BiddingKG/dl_dev/test/12.py

@@ -1,4 +1,7 @@
-print("243705217")
+import re
 
 
+import time
 
+
+print(time.localtime(1663878377786/1000))

BiddingKG/dl_dev/test/test4.py

@@ -111,7 +111,7 @@ def run_one():
     # print("takes",time.time()-a)
     print("start")
     _time1 = time.time()
-    print(predict("12", content,"打印机",original_docchannel=52))
+    print(predict("12", content,"打印机",original_docchannel=101))
     # test(12,content)
     # test(12,text)
     print("takes",time.time()-a)

BiddingKG/maxcompute/documentDumplicate.py

@@ -2141,7 +2141,7 @@ if __name__ == '__main__':
 #     c = f_get_nlp_enterprise()
 #     print(c.evaluate("山东东岳项目管理有限公司",_json))
 #     print(c.evaluate(_json))
-#     c = f_set_docid()
+#     c = f_set_docid()f_get_single_merged_bychannel
 #     _s = '''
 #     154064190	1512489600	4	03689-11	1	大连市妇女儿童医疗中心
 #     154064188	1512489600	4	03689-11	1	大连市妇女儿童医疗中心

BiddingKG/maxcompute/documentMerge.py

@@ -364,6 +364,16 @@ class f_get_single_merged_bychannel(BaseUDTF):
         _d = {"data":{str(docid):[]},"process_time":getCurrent_date()}
         self.forward(json.dumps(_d))
 
+@annotate('string->string')
+class f_get_single_merged_docids(object):
+
+    def evaluate(self,_json):
+        if _json!="" and _json is not None:
+            _d = json.loads(_json)
+            _keys = _d.get("data",{}).keys()
+            return ",".join(list(_keys))
+        return ""
+
 
 
 
@@ -1252,14 +1262,8 @@ if __name__ == '__main__':
     a = f_remege_limit_num_contain_bychannel()
     buffer = a.new_buffer()
     tmp_s = '''
-    234858920	229011768	2022-03-25	1648137600		横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工招标文件.pdf	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工文件.pdf	珠海大横琴公共设施建设管理有限公司	珠海德联工程咨询有限公司				103	0	7	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "2022-04-29", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	232745950	2022-04-12	1649692800	E4404000001002779001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工招标答疑	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工答疑	珠海大横琴公共设施建设管理有限公司	珠海德联工程咨询有限公司				103	0	8	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	234858920	2022-04-21	1650470400	E4404000001002779001001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工						101	1	2	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	234595980	2022-04-20	1650384000	E4404000001002779001001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工	珠海大横琴公共设施建设管理有限公司	珠海德联工程咨询有限公司				105	0	10	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "2022-04-22", "time_publicity_start": "2022-04-21", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	228908786	2022-03-25	1648137600	E4404000001002779001001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工	珠海大横琴公共设施建设管理有限公司	珠海德联工程咨询有限公司			1795743.68	52	0	8	"{"time_bidclose": "2022-04-20", "time_bidopen": "2022-04-20", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "2022-04-20", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "2022-03-26", "time_publicity_end": "2022-04-26", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	234523333	2022-04-20	1650384000	E4404000001002779001001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工						101	0	2	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	234787082	2022-04-20	1650384000	E4404000001002779001001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工开标记录表	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工开标记录表					1795743.68	101	0	6	"{"time_bidclose": "", "time_bidopen": "2022-04-20", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
-    234858920	235240618	2022-04-22	1650556800	E4404000001002779001001	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化工程施工	横琴新区2号消防站暨新区消防宣教培训体验中心项目智能化施工			广东博思信息技术股份有限公司	1775136.23		101	0	12	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "2022-04-26", "time_publicity_start": "2022-04-24", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
+    266523906	266539038	2022-09-08	1662566400	SDGP371525000202201000421_A	冠县第二实验小学平台教育信息化设备采购智慧屏	冠县第二实验小学平台教育信息化设备采购智慧屏成交公告	冠县第二实验小学平台教育信息化设备智慧屏	冠县第二实验小学	聊城市采购中心	山东润博网络有限公司	246890.0		101	0	12	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
+    266523906	266523906	2022-09-15	1663171200	SDGP371525000202201000421_A	冠县第二实验小学平台教育信息化设备采购智慧屏	冠县第二实验小学平台教育信息化设备采购智慧屏成交公告	冠县第二实验小学平台教育信息化设备智慧屏	冠县第二实验小学	聊城市采购中心	山东润博网络有限公司	246890.0		101	999	12	"{"time_bidclose": "", "time_bidopen": "", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "", "time_publicity_start": "", "time_registration_end": "", "time_registration_start": "", "time_release": ""}"
 
     '''
     for _s in tmp_s.split("\n"):
@@ -1272,4 +1276,6 @@ if __name__ == '__main__':
     # a.iterate(buffer,219957825,101,86400*4,"1","1","1","1","1","1","1",0,5,'{"time_bidclose": "", "time_bidopen": "2022-02-10", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "2022-02-21", "time_publicity_start": "2022-02-11", "time_registration_end": "", "time_registration_start": "", "time_release": ""}')
     # a.iterate(buffer,219957825,101,86400*4,"1","1","1","1","1","1","1",0,5,'{"time_bidclose": "", "time_bidopen": "2022-02-10", "time_bidstart": "", "time_commencement": "", "time_completion": "", "time_earnest_money_end": "", "time_earnest_money_start": "", "time_get_file_end": "", "time_get_file_start": "", "time_publicity_end": "2022-02-22", "time_publicity_start": "2022-02-11", "time_registration_end": "", "time_registration_start": "", "time_release": ""}')
     print(a.terminate(buffer))
-    print(1)
+    print(1)
+
+    print(getSimilarityOfString('37168100014015220220012_40785671','SDGP371681000202201000912'))