#encoding:UTF8

from BaseDataMaintenance.common.multiProcess import MultiHandler,MultiProcessHandler
from BaseDataMaintenance.common.multiThread import MultiThreadHandler
from BaseDataMaintenance.dataSource.pool import ConnectorPool
from BaseDataMaintenance.dataSource.source import *
from BaseDataMaintenance.common.Utils import *
import queue
from tablestore import *
from multiprocessing import RLock
from multiprocessing import Queue
from threading import Thread
from apscheduler.schedulers.blocking import BlockingScheduler

from BaseDataMaintenance.model.mysql.attach_document_richtext import attach_document_richtext
from BaseDataMaintenance.model.mysql.BaseModel import BaseModel
from BaseDataMaintenance.model.ots.document import *
import traceback
from BaseDataMaintenance.dataSource.download import download
import base64
from BaseDataMaintenance.dataSource.interface import getAttachDealInterface,sentMsgToDD
from BaseDataMaintenance.model.ots.attachment import *
from BaseDataMaintenance.common.ossUtils import *
from uuid import uuid4
from bs4 import BeautifulSoup
import random
import platform
from BaseDataMaintenance.common.SWFUtils import swf2images
from multiprocessing import Process

STATUS_TODEAL = 10
STATUS_DEALING = 20
STATUS_DONE = 30
STATUS_FAILED = 40
MAX_DEAL_COUNT = 5

class AttachProcess():

    def __init__(self):

        self.set_docid = set()
        self.docid_lock = RLock()
        # self.task_queue = queue.Queue(100)
        # self.task_queue_his = queue.Queue(100)
        self.q_size = 3000
        self.task_queue = Queue(self.q_size)
        self.task_queue_his = Queue(self.q_size)
        self.document_table = "document"
        self.document_table_index = "document_index"
        self.ots_client = getConnect_ots()
        self.swf_queue = Queue(self.q_size)
        self.attachment_table = "attachment"
        self.attachment_table_index = "attachment_index"
        self.attachment_bucket_name = "attachment-hub"
        self.auth = getAuth()
        check_url = "oss-cn-hangzhou-internal.aliyuncs.com"
        if is_internal:
            self.bucket_url = "http://oss-cn-hangzhou-internal.aliyuncs.com"
        else:
            self.bucket_url = "http://oss-cn-hangzhou.aliyuncs.com"
        self.attachment_hub_url = "https://attachment-hub.oss-cn-hangzhou.aliyuncs.com/"
        log("bucket_url:%s"%(self.bucket_url))
        self.bucket = oss2.Bucket(self.auth,self.bucket_url,self.attachment_bucket_name)
        self.current_path = os.path.dirname(__file__)
        self.current_crtime = "2021-07-08 12:00:00"
        self.current_process_time = "2021-07-09 05:40:00"
        self.current_process_his_time = "2021-07-09 05:40:00"
        self.crtime_file = os.path.join(self.current_path,"crtime.txt")
        self.process_file = os.path.join(self.current_path,"process_time.txt")
        self.process_his_file = os.path.join(self.current_path,"process_time_his.txt")
        if os.path.exists(self.crtime_file):
            with open(self.crtime_file,"r",encoding="utf8") as f:
                line = f.readline()
                if line:
                    self.current_crtime = line.strip()
        if os.path.exists(self.process_file):
            with open(self.process_file,"r",encoding="utf8") as f:
                line = f.readline()
                if line:
                    self.current_process_time = line.strip()
        if os.path.exists(self.process_his_file):
            with open(self.process_his_file,"r",encoding="utf8") as f:
                line = f.readline()
                if line:
                    self.current_process_his_time = line.strip()
        else:
            self.current_process_his_time = self.current_process_time


    def producer(self):
        #直接从ots_document表查询待处理数据
        #查询有附件内容并且还没有处理的

        try:
            exists_q = BoolQuery(should_queries=[RangeQuery(document_attachment_extract_status,0),
                                               ])

            list_must_not = [exists_q]
            # with self.docid_lock:
            self.set_docid = set(list(self.set_docid)[:100])
            for docid in list(self.set_docid):
                list_must_not.append(TermQuery(document_docid,docid))

            bool_query = BoolQuery(must_queries=[NestedQuery(document_attachment_path,WildcardQuery("%s.%s"%(document_attachment_path,document_attachment_path_filemd5),"*")),
                                                 # RangeQuery(document_crtime,range_from=self.current_crtime,include_lower=False)
                                                 #,TermQuery("docid",171428592)
                                                 ],
                                   must_not_queries=list_must_not)

            columns = [document_partitionkey,document_docid,document_attachment_path,document_crtime,document_attachment_extract_status]
            rows, next_token, total_count, is_all_succeed = self.ots_client.search(self.document_table, self.document_table_index,
                                                                              SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort(document_crtime,SortOrder.DESC)]), limit=100, get_total_count=True),
                                                                              ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))

            _size = self.q_size
            log("document producer total_count:%d"%total_count)
            _set_filemd5 = set()
            list_data = getRow_ots(rows)
            for _data in list_data:
                _add = True
                # _document = Document(_data)
                # with self.docid_lock:
                if _data.get(document_docid) in self.set_docid:
                    _add = False
                else:
                    _add = True
                    self.set_docid.add(_data.get(document_docid))
                if _add:
                    if _size>0:
                        _size -= 1
                    else:
                        self.current_crtime = _data.get("crtime","")

                    while True:
                        try:
                            self.task_queue.put((_data,_set_filemd5),True,timeout=0.2)
                            break
                        except Exception as e:
                            time.sleep(1)
                        # if self.task_queue.qsize()<self.q_size:
                        #     log("now add %d"%(_data.get(document_docid)))
                        #     self.task_queue.put(_data,True)
                        #     break
                        # else:
                        #     time.sleep(random.random()+2)
            # if _size==0:
            #     with open(self.crtime_file,"w",encoding="utf8") as f:
            #         f.write(self.current_crtime)

            _count = 1
            while next_token:
                _count += 1
                # if _count>=10:
                #     break
                rows, next_token, total_count, is_all_succeed = self.ots_client.search(self.document_table, self.document_table_index,
                                                                                       SearchQuery(bool_query ,next_token=next_token, limit=100, get_total_count=True),
                                                                                       ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
                list_data = getRow_ots(rows)
                for _data in list_data:
                    _add = True
                    # _document = Document(_data)
                    # with self.docid_lock:
                    if _data.get(document_docid) in self.set_docid:
                        _add = False
                    else:
                        _add = True
                        self.set_docid.add(_data.get(document_docid))
                    if _add:
                        if _size>0:
                            _size -= 1
                        else:
                            self.current_crtime = _data.get("crtime","")

                        while True:
                            try:
                                self.task_queue.put((_data,_set_filemd5),True,timeout=0.2)
                                break
                            except Exception as e:
                                time.sleep(1)
                            # if self.task_queue.qsize()<self.q_size:
                            #     log("now add %d"%(_data.get(document_docid)))
                            #     self.task_queue.put(_data,True)
                            #     break
                            # else:
                            #     time.sleep(random.random()+2)
                # if _size==0:
                #     with open(self.crtime_file,"w",encoding="utf8") as f:
                #         f.write(self.current_crtime)

        except Exception as e:
            log("attachProcess producer error %s"%(str(e)))

    def addDocumentWithFilemd5(self,list_filemd5,_queue):


        list_should_q = []

        set_d_md5 = set()

        if len(list_filemd5)==0:
            return

        for _filemd5 in list_filemd5:
            list_should_q.append(NestedQuery(document_attachment_path,TermQuery("%s.%s"%(document_attachment_path,document_attachment_path_filemd5),_filemd5)))

        bool_query = BoolQuery(must_queries=[BoolQuery(should_queries=list_should_q),
                                             BoolQuery(must_not_queries=[RangeQuery(document_attachment_extract_status,1)])
                                             ]
                               )

        columns = [document_partitionkey,document_docid,document_attachment_path,document_crtime,document_attachment_extract_status,document_status]
        rows, next_token, total_count, is_all_succeed = self.ots_client.search(self.document_table, self.document_table_index,
                                                                               SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort(document_crtime,SortOrder.ASC)]), limit=100, get_total_count=True),
                                                                               ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
        log("addDocumentWithFilemd5  total_count:%d"%total_count)
        _set_filemd5 = set(list_filemd5)
        list_data = getRow_ots(rows)
        print(list_data[0])
        for _data in list_data:

            page_attachments = json.loads(_data.get(document_attachment_path,"[]"))
            for _atta in page_attachments:
                set_d_md5.add(_atta.get(document_attachment_path_filemd5))
            _add = True
            # log("%d"%_data.get(document_docid))
            _document = Document(_data)
            # with self.docid_lock:
            if _data.get(document_docid) in self.set_docid:
                _add = False
            else:
                _add = True
                self.set_docid.add(_data.get(document_docid))
            if _add:
                while True:
                    try:
                        _queue.put((_data,_set_filemd5),True,timeout=1)
                        break
                    except Exception as e:
                        time.sleep(0.5)
                    # if _queue.qsize()<self.q_size:
                    #     log("now add %d"%(_data.get(document_docid)))
                    #     _queue.put(_data,True)
                    #     break
                    # else:
                    #     time.sleep(random.random()+2)
        while next_token:
            rows, next_token, total_count, is_all_succeed = self.ots_client.search(self.document_table, self.document_table_index,
                                                                                   SearchQuery(bool_query ,next_token=next_token, limit=100, get_total_count=True),
                                                                                   ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
            list_data = getRow_ots(rows)
            for _data in list_data:

                page_attachments = json.loads(_data.get(document_attachment_path,"[]"))
                for _atta in page_attachments:
                    set_d_md5.add(_atta.get(document_attachment_path_filemd5))
                _add = True
                # log("%d"%_data.get(document_docid))
                _document = Document(_data)
                # with self.docid_lock:
                if _data.get(document_docid) in self.set_docid:
                    _add = False
                else:
                    _add = True
                    self.set_docid.add(_data.get(document_docid))
                if _add:
                    while True:
                        try:
                            _queue.put((_data,_set_filemd5),True,timeout=1)
                            break
                        except Exception as e:
                            time.sleep(0.5)
                        # if _queue.qsize()<self.q_size:
                        #     log("now add %d"%(_data.get(document_docid)))
                        #     _queue.put(_data,True)
                        #     break
                        # else:
                        #     time.sleep(random.random()+2)
        log("not in d_md5 %d-%d"%(len(list_filemd5),len(set_d_md5)))
        for _filemd5 in list_filemd5:
            if _filemd5 not in set_d_md5:
                _dict = {attachment_filemd5:_filemd5}
                _attach = attachment(_dict)
                _attach.setValue(attachment_link_status,2,True)
                _attach.update_row(self.ots_client)
                log("%s not in d_md5"%(_filemd5))


    def producer_attachment_update(self):

        not_link_status = BoolQuery(must_not_queries=[TermQuery(attachment_link_status,1),
                                                      TermQuery(attachment_link_status,2)])
        # not_his = BoolQuery(must_not_queries=[WildcardQuery(attachment_path,"*fix*")])
        # not_swf = BoolQuery(must_not_queries=[TermQuery(attachment_filetype,"swf")])
        # bool_query = BoolQuery(must_queries=[RangeQuery(attachment_process_time,self.current_process_time,include_lower=False)
        #                                      ,TermQuery("status",10)])
        bool_query = BoolQuery(must_queries=[not_link_status,
                                             # not_his,
                                            # not_swf,
                                            #  RangeQuery(attachment_process_time,self.current_process_time,include_lower=False),
                                             TermQuery("status",10)])

        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort(attachment_process_time,SortOrder.DESC)]),limit=50,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet([attachment_process_time],ColumnReturnType.SPECIFIED))

        log("attachment producer_update total_count:%d"%total_count)
        _size = self.q_size
        list_dict = getRow_ots(rows)
        list_filemd5 = []
        for _dict in list_dict:
            list_filemd5.append(_dict.get(attachment_filemd5))
            if _size>0:
                _size -= 1
            else:
                self.current_process_time = _dict.get(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"))
        self.addDocumentWithFilemd5(list_filemd5,self.task_queue)
        # if _size==0:
        #     log("current_process_time:%s"%(self.current_process_time))
        #     with open(self.process_file,"w",encoding="utf8") as f:
        #         f.write(self.current_process_time)
        _count = 1
        while next_token:
            _count += 1
            # if _count>=10:
            #     break
            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=50,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet([attachment_process_time],ColumnReturnType.SPECIFIED))
            list_dict = getRow_ots(rows)
            list_filemd5 = []
            for _dict in list_dict:
                list_filemd5.append(_dict.get(attachment_filemd5))
                if _size>0:
                    _size -= 1
                else:
                    self.current_process_time = _dict.get(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"))
            self.addDocumentWithFilemd5(list_filemd5,self.task_queue)
            # if _size==0:
            #     log("current_process_time:%s"%(self.current_process_time))
            #     with open(self.process_file,"w",encoding="utf8") as f:
            #         f.write(self.current_process_time)

    def producer_attachment_update_swf(self):

        not_link_status = BoolQuery(must_not_queries=[TermQuery(attachment_link_status,1)])
        not_his = BoolQuery(must_not_queries=[WildcardQuery(attachment_path,"*fix*")])
        is_swf = BoolQuery(must_queries=[TermQuery(attachment_filetype,"swf")])
        # bool_query = BoolQuery(must_queries=[RangeQuery(attachment_process_time,self.current_process_time,include_lower=False)
        #                                      ,TermQuery("status",10)])
        bool_query = BoolQuery(must_queries=[#not_link_status,
            not_his,
            is_swf,
            not_link_status,
            TermQuery("status",10)])

        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort(attachment_process_time,SortOrder.DESC)]),limit=100,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet([attachment_process_time],ColumnReturnType.SPECIFIED))

        _size = self.q_size
        list_dict = getRow_ots(rows)
        list_filemd5 = []
        for _dict in list_dict:
            list_filemd5.append(_dict.get(attachment_filemd5))
        self.addDocumentWithFilemd5(list_filemd5,self.swf_queue)

        while next_token:
            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=100,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet([attachment_process_time],ColumnReturnType.SPECIFIED))
            list_dict = getRow_ots(rows)
            list_filemd5 = []
            for _dict in list_dict:
                list_filemd5.append(_dict.get(attachment_filemd5))
            self.addDocumentWithFilemd5(list_filemd5,self.swf_queue)


    def producer_attachment_his(self):

        not_link_status = BoolQuery(must_not_queries=[TermQuery(attachment_link_status,1)])
        is_his = BoolQuery(should_queries=[WildcardQuery(attachment_path,"*fix*"),
                                           WildcardQuery(attachment_path,"*sup*")])
        # bool_query = BoolQuery(must_queries=[RangeQuery(attachment_process_time,self.current_process_time,include_lower=False)
        #                                      ,TermQuery("status",10)])
        bool_query = BoolQuery(must_queries=[#not_link_status,
                                             is_his,
                                             RangeQuery(attachment_process_time,self.current_process_his_time,include_lower=False),
                                             TermQuery("status",10)])

        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort(attachment_process_time,SortOrder.ASC)]),limit=100,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet([attachment_process_time],ColumnReturnType.SPECIFIED))
        _size = self.q_size
        list_dict = getRow_ots(rows)
        list_filemd5 = []
        log("getting dicts %d"%_size)
        for _dict in list_dict:
            list_filemd5.append(_dict.get(attachment_filemd5))
            if _size>0:
                _size -= 1
            else:
                self.current_process_his_time = _dict.get(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"))
        self.addDocumentWithFilemd5(list_filemd5,self.task_queue_his)
        if _size==0:
            log("current_process_his_time:%s"%(self.current_process_his_time))
            with open(self.process_his_file,"w",encoding="utf8") as f:
                f.write(self.current_process_his_time)

        while next_token:
            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=100,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet([attachment_process_time],ColumnReturnType.SPECIFIED))
            log("getting dicts %d"%_size)
            list_dict = getRow_ots(rows)
            list_filemd5 = []
            for _dict in list_dict:
                list_filemd5.append(_dict.get(attachment_filemd5))
                if _size>0:
                    _size -= 1
                else:
                    self.current_process_his_time = _dict.get(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"))
            self.addDocumentWithFilemd5(list_filemd5,self.task_queue_his)
            if _size==0:
                log("current_process_his_time:%s"%(self.current_process_his_time))
                with open(self.process_his_file,"w",encoding="utf8") as f:
                    f.write(self.current_process_his_time)




    def getAttachments(self,list_filemd5,columns_to_get=[attachment_filemd5,attachment_path,attachment_size,attachment_attachmenthtml,attachment_attachmentcon,attachment_filetype,attachment_docids,attachment_status,attachment_swfUrls]):
        list_attachment = []
        rows_to_get = []
        for _md5 in list_filemd5[:50]:
            primary_key = [(attachment_filemd5,_md5)]
            rows_to_get.append(primary_key)
        req = BatchGetRowRequest()
        req.add(TableInBatchGetRowItem(self.attachment_table,rows_to_get,columns_to_get,None,1))
        try:
            result = self.ots_client.batch_get_row(req)
            attach_result = result.get_result_by_table(self.attachment_table)
            for item in attach_result:
                if item.is_ok:
                    _dict = getRow_ots_primary(item.row)
                    if _dict is not None:
                        list_attachment.append(attachment(_dict))

        except Exception as e:
            log(str(list_filemd5))
            log("attachProcess comsumer error %s"%str(e))
        return list_attachment

    def getTitleFromHtml(self,filemd5,_html):
        _soup = BeautifulSoup(_html,"lxml")

        _find = _soup.find("a",attrs={"data":filemd5})
        _title = ""
        if _find is not None:
            _title = _find.get_text()
        return _title

    def getSourceLinkFromHtml(self,filemd5,_html):
        _soup = BeautifulSoup(_html,"lxml")

        _find = _soup.find("a",attrs={"filelink":filemd5})
        filelink = ""
        if _find is None:
            _find = _soup.find("img",attrs={"filelink":filemd5})
            if _find is not None:
                filelink = _find.attrs.get("src","")
        else:
            filelink = _find.attrs.get("href","")
        return filelink

    def transformSWF(self,bucket,attachment_hub_url,objectPath,localpath,swf_dir):
        swf_urls = []
        try:
            swf2images(localpath,swf_dir)
            list_files = os.listdir(swf_dir)
            list_files.sort(key=lambda x:x)
            headers = dict()
            headers["x-oss-object-acl"] = oss2.OBJECT_ACL_PUBLIC_READ
            for _file in list_files:
                swf_localpath = "%s/%s"%(swf_dir,_file)
                swf_objectPath = "%s/%s"%(objectPath.split(".")[0],_file)
                uploadFileByPath(bucket,swf_localpath,swf_objectPath,headers)
                _url = "%s/%s"%(attachment_hub_url,swf_objectPath)
                swf_urls.append(_url)
                os.remove(swf_localpath)
        except Exception as e:
            traceback.print_exc()
        return swf_urls

    def comsumer_handle(self,_item,result_queue):
        _dict,_set_filemd5 = _item
        if type(_dict)==dict:
            _document = Document(_dict)
        else:
            _document = _dict
        log("process docid:%s"%(str(_document.getProperties().get(document_docid))))
        try:
            if _document.getProperties().get(document_attachment_extract_status)==1:
                return
            attachPaths = json.loads(_document.getProperties().get(document_attachment_path,'[]'))
            list_filemd5 = []
            dict_attach = {}
            for _d in attachPaths:
                _md5 = _d.get(document_attachment_path_filemd5)
                if _md5 is not None:
                    list_filemd5.append(_md5)
                    dict_attach[_md5] = _d

            not_fully_pers = False
            if len(list_filemd5)>0:
                list_attachment = self.getAttachments(list_filemd5)
                if len(list_filemd5)!=len(list_attachment):
                    #限制时间范围以外的数据直接入库
                    log("not fullly upload of docid:%s"%(str(_document.getProperties().get(document_docid))))
                    _document.setValue(document_attachment_extract_status,0,True)
                    _document.update_row(self.ots_client)
                    d_crtime = _document.getProperties().get(document_crtime)
                    if time.mktime(time.localtime())-time.mktime(time.strptime(d_crtime,"%Y-%m-%d %H:%M:%S"))>60*60*12:
                        not_fully_pers = True
                _all_succeed = True
                for attach in list_attachment:
                    #更新docids
                    filemd5 = attach.getProperties().get(attachment_filemd5,"")
                    docids = attach.getProperties().get(attachment_docids,"")
                    if docids!="":
                        set_docids = set(docids.split(","))
                    else:
                        set_docids = set()

                    set_docids.add(str(_document.getProperties().get(document_docid)))
                    docids = ",".join(list(set_docids))
                    attach.setValue(attachment_docids,docids,True)

                    _source_link = dict_attach.get(filemd5,{}).get(attachment_file_link,"")
                    attach.setValue(attachment_file_link,_source_link,True)
                    _status = attach.getProperties().get(attachment_status)

                    if _status in (ATTACHMENT_PROCESSED,ATTACHMENT_TOOLARGE):
                        continue
                    else:
                        _all_succeed = False

                if _all_succeed or not_fully_pers:

                    debug("update docid:%d with all attachment succeed"%(_document.getProperties().get(document_docid)))
                    _document.fix_columns(self.ots_client,[document_dochtmlcon],False)
                    _document_html = _document.getProperties().get(document_dochtmlcon,"")


                    list_html = []

                    list_attachmenttextcon = []
                    debug("update attach of docid:%d"%_document.getProperties().get(document_docid))
                    for attach in list_attachment:

                        filemd5 = attach.getProperties().get(attachment_filemd5,"")

                        _document_html = _document.getProperties().get(document_dochtmlcon,"")

                        _file_title = self.getTitleFromHtml(filemd5,_document_html)
                        filelink = self.getSourceLinkFromHtml(filemd5,_document_html)
                        if _file_title!="":
                            attach.setValue(attachment_file_title,_file_title,True)
                        if filelink!="":
                            attach.setValue(attachment_file_link,filelink,True)

                        for _d in attachPaths:
                            if _d.get(document_attachment_path_filemd5)==filemd5:
                                _d[document_attachment_path_fileTitle] = _file_title
                                _d[document_attachment_path_fileLink] = filelink

                        _html = attach.getProperties().get(attachment_attachmenthtml,"")
                        if len(_html)>5:
                            list_html.append(_html)
                            _soup = BeautifulSoup(_html,"lxml")
                            list_attachmenttextcon.append(_soup.get_text())
                            if _soup.find("table") is not None:
                                attach.setValue(attachment_has_table,1,True)

                        #判断是否是swf,如果是则判断是否已经拆分并且上传到阿里云，完成上传到阿里云后,新增img标签到公告html中
                        if attach.getProperties().get(attachment_filetype)=="swf":
                            log("swf of docid:%d"%_document.getProperties().get(document_docid))
                            swf_urls = json.loads(attach.getProperties().get(attachment_swfUrls,"[]"))
                            if attach.getProperties().get(attachment_swfUrls) is None:
                                objectPath = attach.getProperties().get(attachment_path,"")
                                localpath = os.path.join(self.current_path,"download/%s.swf"%(uuid4().hex))
                                swf_dir = os.path.join(self.current_path,uuid4().hex)
                                if downloadFile(self.bucket,objectPath,localpath):
                                    try:
                                        swf_urls = self.transformSWF(self.bucket,self.attachment_hub_url,objectPath,localpath,swf_dir)
                                        # swf2images(localpath,swf_dir)
                                        # list_files = os.listdir(swf_dir)
                                        # list_files.sort(key=lambda x:x)
                                        # headers = dict()
                                        # headers["x-oss-object-acl"] = oss2.OBJECT_ACL_PUBLIC_READ
                                        # for _file in list_files:
                                        #     swf_localpath = "%s/%s"%(swf_dir,_file)
                                        #     swf_objectPath = "%s/%s"%(objectPath.split(".")[0],_file)
                                        #     uploadFileByPath(self.bucket,swf_localpath,swf_objectPath,headers)
                                        #     _url = "%s/%s"%(self.attachment_hub_url,swf_objectPath)
                                        #     swf_urls.append(_url)
                                        #     os.remove(swf_localpath)
                                    except Exception as e:
                                        traceback.print_exc()
                                # log("localpath exists:%s"%str(os.path.exists(localpath)))
                                if os.path.exists(localpath):
                                    os.remove(localpath)
                                # log("swfdir exists:%s %d"%(str(os.path.exists(swf_dir)),len(os.listdir(swf_dir))))
                                if os.path.exists(swf_dir):

                                    os.rmdir(swf_dir)
                                attach.setValue(attachment_swfUrls,json.dumps(swf_urls,ensure_ascii=False),True)
                            _document.updateSWFImages(swf_urls)



                    debug("attach done of docid:%d"%_document.getProperties().get(document_docid))
                    _document.setValue(document_attachmenttextcon,"".join(list_attachmenttextcon)[:60*10000],True)

                    #重新做要素提取

                    # _document.setValue(document_status,random.randint(40,50),True)
                    if _document.getProperties().get(document_status,0) >= 151 and _document.getProperties().get(document_status,0)<=450:
                        _document.fromEas2Maxcompute()
                    else:
                        _document.fromEasFailed()

                    _document.updateAttachment(list_html)
                    if _all_succeed:
                        _document.setValue(document_attachment_extract_status,1,True)
                    if not_fully_pers:
                        _document.setValue(document_attachment_extract_status,2,True)
                    _document.setValue(document_attachment_path,json.dumps(attachPaths,ensure_ascii=False))
                    _document.setValue(document_opertime,getCurrent_date(format="%Y-%m-%d %H:%M:%S"),True)

                    debug("updating docid:%d"%(_document.getProperties().get(document_docid)))
                    # 更新附件和公告
                    for attach in list_attachment:
                        filemd5 = attach.getProperties().get(attachment_filemd5,"")
                        if filemd5 in _set_filemd5:
                            attach.setValue(attachment_link_status,1,True)
                        attach.update_row(self.ots_client)
                    _document.update_row(self.ots_client)
                    debug("update docid:%d done "%(_document.getProperties().get(document_docid)))
                else:
                    log("not all succeed of docid:%s"%(str(_document.getProperties().get(document_docid))))
                    _document.setValue(document_attachment_extract_status,0,True)
                    _document.update_row(self.ots_client)
        except Exception as e:
            debug("attach process failed of docid:%s of error:%s"%(str(_document.getProperties().get(document_docid)),str(e)))
            traceback.print_exc()
        finally:
            _docid = _document.getProperties().get(document_docid)
            debug("process over of docid:%s"%(str(_docid)))
            # with self.docid_lock:
            if _docid in self.set_docid:
                self.set_docid.remove(_docid)
            log("process done of docid:%s"%(str(_docid)))

    def comsumer(self):
        #对每篇公告的附件处理数据,查询附件是否以及上传完成和处理完成,未完全上传的直接跳过

        #上传完成但没有处理完成的调用接口处理，更新attachment

        #合并所有attachment数据,html部分写入dochtmlcon,对html取text后写入attachtextcon

        #更新数据

        log("comsumer task_queue size :%d"%(self.task_queue.qsize()))
        mt = MultiThreadHandler(self.task_queue,self.comsumer_handle,None,3)
        # mt = MultiHandler(self.task_queue,self.comsumer_handle,None,2,30)
        # mt = MultiProcessHandler(self.task_queue,self.comsumer_handle,None,30)
        mt.run()

    def comsumer_swf(self):
        mt = MultiHandler(self.swf_queue,self.comsumer_handle,None,2,2)
        mt.run()

    def comsumer_his(self):
        mt = MultiThreadHandler(self.task_queue_his,self.comsumer_handle,None,6)
        # mt = MultiHandler(self.task_queue_his,self.comsumer_handle,None,2,2)
        mt.run()

    def generateP(self,func):
        p = Process(target=func)
        p.start()
        p.join()

    def attachment_process(self):
        # self.producer()
        self.producer_attachment_update()
        self.comsumer()

    def schedule(self):
        _scheduler = BlockingScheduler()

        _scheduler.add_job(self.producer,"cron",minute="*/5")
        _scheduler.add_job(self.producer_attachment_update,"cron",minute="*/5")
        _scheduler.add_job(self.comsumer,"cron",minute="*/5")
        # _scheduler.add_job(self.attachment_process,"cron",second="*/1")

        # _scheduler.add_job(self.producer_attachment_update_swf,"cron",minute="*/1")
        # _scheduler.add_job(self.comsumer_swf,"cron",minute="*/1")

        # _scheduler.add_job(self.producer_attachment_his,"cron",minute="*/1")
        # _scheduler.add_job(self.comsumer_his,"cron",minute="*/1")

        _scheduler.start()

class AttachmentRec():

    def __init__(self):

        self.task_queue = queue.Queue(100)
        self.document_table = "document"
        self.document_table_index = "document_index"
        self.ots_client = getConnect_ots()
        self.attachment_table = "attachment"
        self.attachment_table_index = "attachment_index"
        self.attachment_bucket_name = "attachment-hub"
        self.auth = getAuth()
        check_url = "oss-cn-hangzhou-internal.aliyuncs.com"
        if is_internal:
            self.bucket_url = "http://oss-cn-hangzhou-internal.aliyuncs.com"
        else:
            self.bucket_url = "http://oss-cn-hangzhou.aliyuncs.com"
        oss2.defaults.connection_pool_size = 100
        oss2.defaults.multiget_num_threads = 20
        log("bucket_url:%s"%(self.bucket_url))
        self.bucket = oss2.Bucket(self.auth,self.bucket_url,self.attachment_bucket_name)
        self.current_path = os.path.dirname(__file__)
        self.attachment_queue_large = queue.Queue(1000)
        self.attachment_queue_small = queue.Queue(1000)
        self.attachment_queue_storage = queue.Queue(1000)

        self.update_document_queue = queue.Queue(1000)

        self.mc_succeed_from  = ATTACHMENT_MC_SUCCEED_FROM
        self.mc_succeed_to = ATTACHMENT_MC_SUCCEED_TO
        self.mc_failed_from = ATTACHMENT_MC_FAILED_FROM
        self.mc_failed_to = ATTACHMENT_MC_FAILED_TO

    def producer_large(self):

        #处理1.5M-10M的附件
        columns = [attachment_filemd5,attachment_path,attachment_size,attachment_attachmenthtml,attachment_attachmentcon,attachment_filetype,attachment_docids,attachment_status]

        list_sh = BoolQuery(should_queries=[TermQuery("status",ATTACHMENT_INIT),
                                            TermQuery("status",ATTACHMENT_PROCESSED_FAILED)])

        bool_query = BoolQuery(must_queries=[list_sh,
                                             RangeQuery("size",range_from=ATTACHMENT_SMALL_SIZE)])

        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("crtime",SortOrder.ASC)]),limit=100,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))

        list_dict = getRow_ots(rows)
        for _dict in list_dict:
            _attachment = attachment(_dict)
            self.attachment_queue_large.put(_attachment)

        while next_token:

            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=100,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
            list_dict = getRow_ots(rows)
            for _dict in list_dict:
                _attachment = attachment(_dict)
                self.attachment_queue_large.put(_attachment)


    def comsumer_large(self):

        def _handle(attach,result_queue):

            try:
                filemd5 = attach.getProperties().get(attachment_filemd5)
                _status = attach.getProperties().get(attachment_status)
                _filetype = attach.getProperties().get(attachment_filetype)
                _size = attach.getProperties().get(attachment_size)
                _path = attach.getProperties().get(attachment_path)
                _uuid = uuid4()
                objectPath = attach.getProperties().get(attachment_path)
                localpath = os.path.join(self.current_path,"download",_uuid.hex)
                docids = attach.getProperties().get(attachment_docids)

                if _size>ATTACHMENT_LARGESIZE:
                    attach.setValue(attachment_status, ATTACHMENT_TOOLARGE)
                    log("attachment :%s of path:%s to large"%(filemd5,_path))
                else:

                    if downloadFile(self.bucket,objectPath,localpath):
                        _data_base64 = base64.b64encode(open(localpath,"rb").read())
                        #调用接口处理结果
                        start_time = time.time()
                        _success,_html = getAttachDealInterface(_data_base64,_filetype)
                        if _success:
                            log("process filemd5:%s of type:%s with size:%.3fM takes %ds,ret_size:%d"%(filemd5,_filetype,round(_size/1024/1024,4),time.time()-start_time,len(_html)))
                            attach.setValue(attachment_attachmenthtml,_html,True)
                            attach.setValue(attachment_attachmentcon,BeautifulSoup(_html,"lxml").get_text(),True)
                            attach.setValue(attachment_status,ATTACHMENT_PROCESSED,True)
                            attach.setValue(attachment_recsize,len(_html),True)
                            attach.setValue(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"),True)
                        else:
                            attach.setValue(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"),True)
                            attach.setValue(attachment_status,ATTACHMENT_PROCESSED_FAILED)
                            log("attach interface failed of docid:%s filemd5:%s of type:%s size:%.3fM with result:%s"%(str(docids),filemd5,_filetype,round(_size/1024/1024,4),str(_html)))
                            sentMsgToDD("attach interface failed of docid:%s of filemd5:%s of type:%s size:%.3fM with result:%s"%(str(docids),filemd5,_filetype,round(_size/1024/1024,4),str(_html)))


                attach.update_row(self.ots_client)
            except Exception as e:
                traceback.print_exc()
            finally:
                if os.path.exists(localpath):
                    os.remove(localpath)
        mt = MultiThreadHandler(self.attachment_queue_large,_handle,None,5)
        mt.run()


    def producer_small(self):
        columns = [attachment_filemd5,attachment_path,attachment_size,attachment_attachmenthtml,attachment_attachmentcon,attachment_filetype,attachment_docids,attachment_status]

        should_q = BoolQuery(should_queries=[TermQuery("status",0),
                                             TermQuery("status",3)])

        bool_query = BoolQuery(must_queries=[should_q,
                                             RangeQuery("size",range_to=ATTACHMENT_LARGESIZE)
                                             ])

        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("crtime",SortOrder.ASC)]),limit=100,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
        print(total_count)

        list_dict = getRow_ots(rows)
        for _dict in list_dict:
            _attachment = attachment(_dict)
            self.attachment_queue_small.put(_attachment)

        while next_token:

            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=100,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
            list_dict = getRow_ots(rows)
            for _dict in list_dict:
                _attachment = attachment(_dict)
                self.attachment_queue_small.put(_attachment)

    def comsumer_small(self):

        def _handle(attach,result_queue):
            _uuid = uuid4()
            filemd5 = attach.getProperties().get(attachment_filemd5)
            filetype = attach.getProperties().get(attachment_filetype)
            objectPath = attach.getProperties().get(attachment_path)
            localpath = os.path.join(self.current_path,"download","%s.%s"%(_uuid.hex,filetype))
            if downloadFile(self.bucket,objectPath,localpath):

                try:
                    # _data_base64 = base64.b64encode(open(localpath,"rb").read())
                    #
                    # _dict = {}
                    # for k,v in attach.getProperties().items():
                    #     if k in attach.getProperties().get("all_columns",[]):
                    #         _dict[k] = v
                    #
                    # list_attach = splitBase64(_data_base64,_dict)
                    # list_attach.reverse()
                    # for _attach in list_attach:
                    #     _attach.setStatusToMCDeal()
                    #     _attach.update_row(self.ots_client)
                    #
                    # if len(list_attach)==0:
                    #     attach.setValue(attachment_status,ATTACHMENT_TOOLARGE,True)
                    #     attach.update_row(self.ots_client)


                    # attach.setValue(attachment_base64,_data_base64,True)
                    attach.setStatusToMCDeal()
                    attach.update_row(self.ots_client)
                    log("add attach to maxcompute succeed %s"%(filemd5))

                except Exception as e:
                    print("add to maxcompute error:%s %s"%(filemd5,objectPath))
                    traceback.print_exc()

            else:
                log("download failed of %s %s"%(filemd5,objectPath))
            if os.path.exists(localpath):
                os.remove(localpath)



        mt = MultiThreadHandler(self.attachment_queue_small,_handle,None,20)
        mt.run()


    def producer_storage(self):

        columns = [attachment_filemd5,attachment_path,attachment_size,attachment_attachmenthtml,attachment_attachmentcon,attachment_filetype,attachment_docids,attachment_status]


        bool_query = BoolQuery(must_queries=[RangeQuery("status",self.mc_succeed_from,self.mc_failed_to,True,True)])

        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("crtime",SortOrder.ASC)]),limit=100,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))

        list_dict = getRow_ots(rows)
        for _dict in list_dict:
            _attachment = attachment(_dict)
            self.attachment_queue_storage.put(_attachment)

        while next_token:

            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=100,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
            list_dict = getRow_ots(rows)
            for _dict in list_dict:
                _attachment = attachment(_dict)
                self.attachment_queue_storage.put(_attachment)

    def comsumer_storage(self):

        def _handle(attach,result_queue):
            _status = attach.getProperties().get(attachment_status,0)
            new_status = ATTACHMENT_PROCESSED
            if _status>=self.mc_succeed_from and _status<=self.mc_succeed_to:
                new_status = ATTACHMENT_PROCESSED
            elif _status>=self.mc_failed_from and _status<=self.mc_failed_to:
                new_status = random.randint(ATTACHMENT_MC_TODEAL_FROM,ATTACHMENT_MC_TODEAL_TO)
            attach.setValue(attachment_status,new_status,True)
            if _status==ATTACHMENT_PROCESSED:

                _html = attach.getProperties().get(attachment_attachmenthtml,"")
                attach.setValue(attachment_base64,"",True)
                attach.setValue(attachment_attachmentcon,BeautifulSoup(_html,"lxml").get_text(),True)
                attach.setValue(attachment_process_time,getCurrent_date(format="%Y-%m-%d %H:%M:%S"),True)

            attach.update_row(self.ots_client)
        mt = MultiThreadHandler(self.attachment_queue_storage,_handle,None,20)
        mt.run()

    def dealWithFailed(self):

        def changeStatus(list_dict):
            for _dict in list_dict:
                try:
                    _attach = attachment(_dict)
                    _attach.setStatusToMCDeal()
                    _attach.update_row(self.ots_client)
                    log("change status to mc success:%s"%(str(_attach.getPrimaryKey_turple())))
                except Exception as e:
                    log("dealWithFailed change status error %s"%(str(e)))


        columns = ["status","crtime"]

        bool_query = BoolQuery(must_queries=[RangeQuery("status",self.mc_failed_from,self.mc_failed_to,True,True)])


        rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("crtime",SortOrder.ASC)]),limit=100,get_total_count=True),
                                                                            columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))

        list_dict = getRow_ots(rows)
        _count = len(list_dict)

        changeStatus(list_dict)


        min_crtime = ""
        max_crtime = ""

        if _count>0:
            min_crtime = list_dict[0].get("crtime","")
            max_crtime = list_dict[-1].get("crtime","")

        while next_token:
            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.attachment_table,self.attachment_table_index,
                                                                                SearchQuery(bool_query,next_token=next_token,limit=100,get_total_count=True),
                                                                                columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
            list_dict = getRow_ots(rows)
            changeStatus(list_dict)
            if len(list_dict)>0:
                _count += len(list_dict)
                max_crtime = list_dict[-1].get("crtime","")
        if _count>0:

            bool_query = BoolQuery(must_queries=[RangeQuery("crtime",min_crtime,max_crtime,True,True)])
            rows,next_token,total_count,is_all_succeed = self.ots_clent.search(self.attachment_table,self.attachment_table_index,
                                                                               SearchQuery(bool_query,limit=1,get_total_count=True),
                                                                               columns_to_get=ColumnsToGet(columns,ColumnReturnType.SPECIFIED)
                                                                               )

            _msg = "dataworks处理附件失败统计:创建时间为%s到%s之间的附件数是%d,处理失败数是%d,占比%2f,详情看dataworks的attachment_rec_log表"%(min_crtime,max_crtime,total_count,_count,_count/total_count)
            sentMsgToDD(_msg)



    def schedule(self):
        _scheduler = BlockingScheduler()
        _scheduler.add_job(self.producer_small,"cron",minute="*/1")
        _scheduler.add_job(self.comsumer_small,"cron",minute="*/1")
        # _scheduler.add_job(self.producer_large,"cron",minute="*/1")
        # _scheduler.add_job(self.comsumer_large,"cron",minute="*/1")
        _scheduler.add_job(self.dealWithFailed,"cron",minute="*/1")
        _scheduler.start()






def start_sychro_attachProcess():
    ap = AttachProcess()
    ap.schedule()

def start_sychro_attachRec():
    ar = AttachmentRec()
    ar.schedule()

def setSpaceToAttachmenttextcon(docid):
    ots_client = getConnect_ots()
    document_table = "document"
    document_table_index = "document_index"
    bool_query = BoolQuery(must_queries=[TermQuery("docid",docid)])
    columns = [document_docid]
    rows, next_token, total_count, is_all_succeed = ots_client.search(document_table, document_table_index,
                                                                           SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort(document_page_time,SortOrder.ASC)]), limit=100, get_total_count=True),
                                                                           ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
    list_data = getRow_ots(rows)
    for _data in list_data:
        _d = Document(_data)
        _d.setValue(document_attachmenttextcon,"",True)
        _d.update_row(ots_client)
        break


def testDownloadRace():
    ots_client = getConnect_ots()
    auth = getAuth()
    check_url = "oss-cn-hangzhou-internal.aliyuncs.com"
    if check_net(check_url):
        bucket_url = "http://oss-cn-hangzhou-internal.aliyuncs.com"
    else:
        bucket_url = "http://oss-cn-hangzhou.aliyuncs.com"
    oss2.defaults.connection_pool_size = 20
    oss2.defaults.multiget_num_threads = 20
    log("bucket_url:%s"%(bucket_url))
    attachment_bucket_name = "attachment-hub"
    bucket = oss2.Bucket(auth,bucket_url,attachment_bucket_name)

    bool_query = BoolQuery(must_queries=[RangeQuery("size",500000000)])

    rows,next_token,total_count,is_all_succeed = ots_client.search("attachment","attachment_index",
                                                                   SearchQuery(bool_query,limit=100),
                                                                   columns_to_get=ColumnsToGet(["path"],ColumnReturnType.SPECIFIED))
    list_dict = getRow_ots(rows)

    current_path = os.path.dirname(__file__)
    task_queue = queue.Queue()
    for _dict in list_dict:
        task_queue.put(_dict)
    def _handle(item,result_queue):
        path = item.get("path")

        _uuid = uuid4()
        objectPath = path
        localpath = os.path.join(current_path,"download",_uuid.hex)

        start_time = time.time()
        if downloadFile(bucket,objectPath,localpath):
            log("download path %s succeed speed:%2f"%(objectPath,len(bytes(open(localpath,"rb").read()))/(time.time()-start_time)))
        else:
            log("download path %s failed"%(objectPath))

    mt = MultiThreadHandler(task_queue,_handle,None,1)
    mt.run()


if __name__=="__main__":
    pass
    # start_sychro_attachProcess()
    # setSpaceToAttachmenttextcon(156818075)
    # ar = AttachmentRec()
    # ar.schedule()
    # _data_base64 = base64.b64encode(open("download/UbiquitBtree.pdf","rb").read())
    # _dict = {"filemd5":"111a4f91c5d6bd229f580334dc7848ea952","base64":_data_base64,"base641":_data_base64,"base642":_data_base64
    #          ,"base643":_data_base64,"base644":_data_base64,"base645":_data_base64}
    # _attach = attachment(_dict)
    # _attach.update_row(getConnect_ots())
    # print(_data_base64)
    ap = AttachProcess()
    ap.producer_attachment_update()
    # ap.schedule()
    # print(getMDFFromFile("./download/8a9494757a859f17017b041ca05e4444.pdf"))
    print(len(bytes(None,encoding="utf8")))
