解决导入fool包在云上无法运行的问题

BiddingKG/dl/channel/doc_type.py

@@ -432,6 +432,7 @@ def lstm_att_model_withoutEmb(class_num):
     backward = tf.contrib.rnn.BasicLSTMCell(lstm_dim, state_is_tuple=True, dtype=tf.float32)
     # forward = tf.nn.rnn_cell.DropoutWrapper(forward, output_keep_prob=prob)
     # backward = tf.nn.rnn_cell.DropoutWrapper(backward, output_keep_prob=prob)
+    
     outputs,state = tf.nn.bidirectional_dynamic_rnn(
       forward,
       backward,

BiddingKG/dl/interface/extract.py

@@ -27,8 +27,6 @@ import json
 
 
 
-
-
 #自定义jsonEncoder
 class MyEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -104,14 +102,14 @@ def predict(doc_id,text,title=""):
     data_res["success"] = True
 
 
-    for _article in list_articles:
-        log(_article.content)
-
-    for list_entity in list_entitys:
-        for _entity in list_entity:
-            log("type:%s,text:%s,label:%s,values:%s,sentence:%s,begin_index:%s,end_index:%s"%
-                  (str(_entity.entity_type),str(_entity.entity_text),str(_entity.label),str(_entity.values),str(_entity.sentence_index),
-                   str(_entity.begin_index),str(_entity.end_index)))
+    # for _article in list_articles:
+    #     log(_article.content)
+    #
+    # for list_entity in list_entitys:
+    #     for _entity in list_entity:
+    #         log("type:%s,text:%s,label:%s,values:%s,sentence:%s,begin_index:%s,end_index:%s"%
+    #               (str(_entity.entity_type),str(_entity.entity_text),str(_entity.label),str(_entity.values),str(_entity.sentence_index),
+    #                str(_entity.begin_index),str(_entity.end_index)))
 
     return json.dumps(data_res,cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)
 

BiddingKG/dl/interface/predictor.py

@@ -6,7 +6,7 @@ Created on 2018年12月26日
 
 import os
 import sys
-import fool
+from BiddingKG.dl.common.nerUtils import *
 sys.path.append(os.path.abspath("../.."))
 # from keras.engine import topology
 # from keras import models
@@ -1582,7 +1582,7 @@ class DocChannel():
     datas = []
     datas_title = []
     try:
-      segword_title = ' '.join(fool.cut(doctitle)[0])
+      segword_title = ' '.join(selffool.cut(doctitle)[0])
       segword_content = dochtmlcon
     except:
       segword_content = ''

BiddingKG/maxcompute/evaluates.py

@@ -11,6 +11,9 @@ import logging
 logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 import time
 
+def log(msg):
+    logging.info(msg)
+
 
 # 配置pandas依赖包
 def include_package_path(res_name):
@@ -18,9 +21,10 @@ def include_package_path(res_name):
     archive_files = get_cache_archive(res_name)
     dir_names = sorted([os.path.dirname(os.path.normpath(f.name)) for f in archive_files
                         if '.dist_info' not in f.name], key=lambda v: len(v))
-    sys.path.append(dir_names[0])
-
-    return os.path.dirname(dir_names[0])
+    _path = dir_names[0].split(".zip/files")[0]+".zip/files"
+    log("add path:%s"%(_path))
+    sys.path.append(_path)
+    return _path
 
 # 可能出现类似RuntimeError: xxx has been blocked by sandbox
 # 这是因为包含C的库，会被沙盘block，可设置set odps.isolation.session.enable = true
@@ -67,8 +71,11 @@ def init_env(list_files,package_name):
 def multiLoadEnv():
     def load_project():
         start_time = time.time()
-        init_env(["BiddingKG.zip.env.baseline"],str(uuid.uuid4()))
+        ## init_env(["BiddingKG.zip.env.baseline"],str(uuid.uuid4()))
         # init_env(["BiddingKG.zip.env.backup"],str(uuid.uuid4()))
+        #改为zip引入
+        log("=======")
+        include_package_path("BiddingKG.baseline.zip")
         logging.info("init biddingkg.zip.env.line cost %d"%(time.time()-start_time))
 
     def load_vector():
@@ -113,26 +120,29 @@ class Extract(BaseUDTF):
 
         multiLoadEnv()
 
-        # logging.info("time5"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
-        # import BiddingKG.dl.interface.predictor as predictor
-        # logging.info("time6"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
-        # import BiddingKG.dl.interface.Entitys as Entitys
-        # logging.info("time6.1"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
-        # import BiddingKG.dl.interface.getAttributes as getAttributes
-        # logging.info("time6.2"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
-        # import BiddingKG.dl.entityLink.entityLink as entityLink
-        # logging.info("time6.2"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
-        # import BiddingKG.dl.interface.Preprocessing as Preprocessing
-        # logging.info("time6.3"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
-
+        import BiddingKG.dl.common.nerUtils
+        log("time5"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
+        import BiddingKG.dl.interface.predictor as predictor
+        log("time6"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
+        import BiddingKG.dl.interface.Entitys as Entitys
+        log("time6.1"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
+        import BiddingKG.dl.interface.getAttributes as getAttributes
+        log("time6.2"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
+        import BiddingKG.dl.entityLink.entityLink as entityLink
+        log("time6.2"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
+        import BiddingKG.dl.interface.Preprocessing as Preprocessing
+        log("time6.3"+str(datetime.datetime.now().strftime('%y-%m-%d %H:%M:%S')))
+        
+        log("=======")
+        time.sleep(5)
         from BiddingKG.dl.interface.extract import predict as predict
+        log("=======import done")
         import json
 
-        from BiddingKG.dl.common.Utils import log
         import numpy as np
 
 
-        global predictor,Entitys,getAttributes,entityLink,json,MyEncoder,Preprocessing,log,MyEncoder,np,predict
+        global predictor,Entitys,getAttributes,entityLink,json,MyEncoder,Preprocessing,MyEncoder,np,predict
         class MyEncoder(json.JSONEncoder):
 
             def default(self, obj):