
import oss2

from BaseDataMaintenance.common.ossUtils import *
import queue
from BaseDataMaintenance.common.Utils import *
from apscheduler.schedulers.blocking import BlockingScheduler
from BaseDataMaintenance.common.multiThread import MultiThreadHandler
import time
import os
import traceback
from itertools import islice
from threading import RLock
from oss2.models import OSS_TRAFFIC_LIMIT
from BaseDataMaintenance.model.ots.attachment import *
from BaseDataMaintenance.dataSource.source import getConnect_ots,getAuth
from BaseDataMaintenance.dataSource.pool import ConnectorPool
import base64

import platform

class AttachmentUpload():


    def initBucket(self,bucket_name):
        self.bucket = oss2.Bucket(self.auth,"http://oss-cn-hangzhou.aliyuncs.com",bucket_name)
        if not does_bucket_exist(self.bucket):
            # 创建存储空间。
            # 如果需要在创建存储空间时设置存储类型、存储空间访问权限、数据容灾类型，请参考以下代码。
            # 以下以配置存储空间为标准存储类型，访问权限为私有，数据容灾类型为同城冗余存储为例。
            bucketConfig = oss2.models.BucketCreateConfig(oss2.BUCKET_STORAGE_CLASS_STANDARD, oss2.BUCKET_DATA_REDUNDANCY_TYPE_ZRS)
            self.bucket.create_bucket(oss2.BUCKET_ACL_PRIVATE, bucketConfig)
            self.bucket.create_bucket()


    def __init__(self):
        self.attachment_bucket_name = "attachment-hub"
        self.auth = getAuth()
        self.task_queue = queue.Queue(maxsize=100)
        self.task_set = set()
        self.task_lock = RLock()
        if platform.system()=="Windows":
            self.HOME_PATH = "F:\\Workspace2016\\BaseDataMaintenance\\BaseDataMaintenance\\maintenance\\attachment\\test\\"
        else:
            self.HOME_PATH = "/datafile2/file/"
        self.initBucket(self.attachment_bucket_name)
        self.ots_client = getConnect_ots()
        self.task_num = 6
        self.md5_lock = RLock()
        self.list_md5 = []


    def process(self):
        #每分钟启动producer
        _schedule = BlockingScheduler()
        _schedule.add_job(self.producer,'cron',minute="*/1")
        #每分钟启动comsumer
        _schedule.add_job(self.comsumer,"cron",minute="*/1")
        _schedule.start()


    def comsumer(self):
        def _handle(_data,result_queue):
            ots_client = self.ots_client
            try:

                #上传附件
                filepath = _data.get("filepath")
                relapath = _data.get("relapath")

                filemd5,size = getMDFFromFile(filepath)

                if filemd5 is not None:
                    _is_exists = True
                    with self.md5_lock:
                        if self.list_md5.count(filemd5)==0:
                            _is_exists = False
                            self.list_md5.insert(0,filemd5)
                            self.list_md5 = self.list_md5[:10000]
                        else:
                            _is_exists = True
                            log("file:%s is uploading,ignore:%s"%(filemd5,filepath))

                    if not _is_exists:
                        filetype = ""
                        relapath = "%s/%s"%(filemd5[:4],relapath)
                        _t = relapath.split(".")
                        if len(_t)>0:
                            filetype = _t[-1]
                        filetype = filetype.lower()
                        print("filemd5",filemd5)
                        _dict = {attachment_filemd5:filemd5,attachment_size:size,attachment_path:relapath,attachment_crtime:getCurrent_date("%Y-%m-%d %H:%M:%S"),attachment_filetype:filetype,attachment_status:ATTACHMENT_INIT}

                        _attach = attachment(_dict)

                        if _attach.getProperties().get(attachment_size)<ATTACHMENT_SMALL_SIZE:
                            #写入base64到attachment
                            # _data_base64 = base64.b64encode(open(filepath,"rb").read())
                            # _attach.setValue(attachment_base64,_data_base64,True)
                            _attach.setStatusToMCDeal()
                        else:
                            _attach.setValue(attachment_status,ATTACHMENT_TOOLARGE,True)

                        #切割附件
                        # if size<ATTACHMENT_LARGESIZE:
                        #     _data_base64 = base64.b64encode(open(filepath,"rb").read())
                        #     list_attach = splitBase64(_data_base64,_dict)
                        #     for _attach in list_attach:
                        #         _attach.setStatusToMCDeal()
                        # else:
                        #     _attach = attachment(_dict)
                        #     _attach.setValue(attachment_status,ATTACHMENT_TOOLARGE,True)
                        #     list_attach = [_attach]


                        _upload_succeed = True
                        if not _attach.exists_row(ots_client):
                            #限速上传
                            headers = {}
                            # headers[OSS_TRAFFIC_LIMIT] = str(100*1024*8)
                            _upload_succeed &= uploadFileByPath(self.bucket,filepath,relapath,headers)

                            #ots attachment插入记录，先获取MD5值,文件路径
                            # for _attach in list_attach:
                            if _upload_succeed:
                                _upload_succeed &= _attach.update_row(ots_client,retrytimes=10)


                        # 删除文件
                        if _upload_succeed:
                            os.remove(filepath)
                            pass
                        else:
                            log("upload failed of filename:%s"%(filepath))
                    else:
                        os.remove(filepath)
                        pass
                else:
                    log("upload failed of filemd5 is None ")

            except queue.Empty as e:
                pass
            except Exception as e:
                traceback.print_exc()
                log("other Exception %s of filename:%s"%(str(e),filepath))
            finally:
                with self.task_lock:
                    if filepath in self.task_set:
                        self.task_set.remove(filepath)


        mt = MultiThreadHandler(self.task_queue,_handle,None,self.task_num)
        mt.run()



    def producer(self):
        for home,dirs,files in os.walk(self.HOME_PATH):
            try:
                #删除空文件夹
                if len(dirs)==0 and len(files)==0  and os.path.isdir(home) and "".join(os.path.split(home))!="".join(os.path.split(self.HOME_PATH)):
                    os.rmdir(home)
                for file in files:
                    filename = os.path.join(home,file)
                    relative_path = filename[len(self.HOME_PATH):]
                    if not os.path.isfile(filename):
                        continue
                    if filename.find("null")>=0:
                        os.remove(filename)
                        continue
                    if filename.find("log")>=0:
                        os.remove(filename)
                        continue
                    if self.isDone(filename):
                        _add = False
                        with self.task_lock:
                            if filename not in self.task_set:
                                _add = True
                                self.task_set.add(filename)
                        if _add:
                            self.task_queue.put({"filepath":filename,"relapath":relative_path},True)
            except Exception as e:
                traceback.print_exc()

    def isDone(self,path):
        #判断文件是否下载完成
        d_time = time.time()-os.stat(path).st_mtime
        if d_time<3600*10:
            return False
        return True

    def listObjects(self):
        for b in islice(oss2.ObjectIterator(self.bucket),10):
            log(b.key)


def start_attachUpload():
    au = AttachmentUpload()
    au.process()

if __name__=="__main__":
    # au = AttachmentUpload()
    # # Endpoint以杭州为例，其它Region请按实际情况填写。
    # service = oss2.Service(au.auth, 'http://oss-cn-hangzhou.aliyuncs.com')
    #
    # # 列举所有的存储空间。
    # for b in oss2.BucketIterator(service):
    #     print(b.creation_date)
    #     print(b.storage_class)
    #     print(b.name)
    # print(getMDFFromFile('C:\\Users\\Administrator\\Desktop\\temp.tar.gz'))
    # import os
    # import time
    # HOME_PATH = "F:\\Workspace2016\\BaseDataMaintenance\\BaseDataMaintenance\\maintenance\\attachment\\"
    # for home,dirs,files in os.walk(HOME_PATH):
    #
    #     relative_path =home[len(HOME_PATH):]
    #     print(len(relative_path))
    #     print(home,dirs,files)
    #     print(len(home.split("\\")))
    #     for f in files:
    #         print(time.time()-os.stat(home+"\\"+f).st_mtime)
    #         print("=",os.stat(home+"\\"+f))
    au = AttachmentUpload()
    au.task_queue.put({"filepath":"download/639f21947d7d48919d7584c4de689e88.swf","relapath":"639f/20210802/2021-08-02/639f21947d7d48919d7584c4de689e88.swf"},True)
    au.process()
    # uploadFileByPath(au.bucket,"download/639f21947d7d48919d7584c4de689e88.swf","639f/20210802/2021-08-02/639f21947d7d48919d7584c4de689e88.swf")
    # print(au.bucket.sign_url('GET', 'temp.tar.gz', 60))


