
from BaseDataMaintenance.common.multiProcess import MultiHandler,MultiProcessHandler
from BaseDataMaintenance.common.multiThread import MultiThreadHandler
from BaseDataMaintenance.dataSource.pool import ConnectorPool
from BaseDataMaintenance.dataSource.source import *
from BaseDataMaintenance.common.Utils import *
import queue
from tablestore import *
from multiprocessing import RLock
from multiprocessing import Queue
from threading import Thread
from apscheduler.schedulers.blocking import BlockingScheduler

from BaseDataMaintenance.model.mysql.attach_document_richtext import attach_document_richtext
from BaseDataMaintenance.model.mysql.BaseModel import BaseModel
from BaseDataMaintenance.model.ots.document import *
import traceback
from BaseDataMaintenance.dataSource.download import download
import base64
from BaseDataMaintenance.dataSource.interface import getAttachDealInterface,sentMsgToDD
from BaseDataMaintenance.model.ots.attachment import *
from BaseDataMaintenance.common.ossUtils import *
from uuid import uuid4
from bs4 import BeautifulSoup
import random
import platform
from BaseDataMaintenance.common.SWFUtils import swf2images
from multiprocessing import Process

STATUS_TODEAL = 10
STATUS_DEALING = 20
STATUS_DONE = 30
STATUS_FAILED = 40
MAX_DEAL_COUNT = 5

class AttachProcess():

    def __init__(self):

        self.set_docid = set()
        self.docid_lock = RLock()
        self.q_size = 5000
        self.task_queue = Queue()
        self.document_table = "document"
        self.document_table_index = "document_index"
        self.ots_client = getConnect_ots()
        self.ots_capacity = getConnect_ots_capacity()
        self.attachment_table = "attachment"
        self.attachment_table_index = "attachment_index"
        self.attachment_bucket_name = "attachment-hub"
        self.auth = getAuth()
        if is_internal:
            self.bucket_url = "http://oss-cn-hangzhou-internal.aliyuncs.com"
        else:
            self.bucket_url = "http://oss-cn-hangzhou.aliyuncs.com"
        self.attachment_hub_url = "https://attachment-hub.oss-cn-hangzhou.aliyuncs.com/"
        log("bucket_url:%s"%(self.bucket_url))
        self.bucket = oss2.Bucket(self.auth,self.bucket_url,self.attachment_bucket_name)
        self.current_path = os.path.dirname(__file__)
        self.current_docid = 0
        self.doc_file = os.path.join(self.current_path,"attach_docid.txt")
        if os.path.exists(self.doc_file):
            with open(self.doc_file,"r",encoding="utf8") as f:
                line = f.readline()
                if line:
                    self.current_docid = int(line.strip())



    def producer(self):
        #直接从ots_document表查询待处理数据
        #查询有附件内容并且还没有处理的

        try:

            bool_q = BoolQuery(must_queries=[RangeQuery(document_docid,self.current_docid,270747031),
                                             NestedQuery(document_attachment_path,ExistsQuery("%s.%s"%(document_attachment_path,document_attachment_path_filemd5))),
                                             # TermQuery(document_docid,1100)
                                             ])

            rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.document_table,self.document_table_index,
                                                                                SearchQuery(bool_q,sort=Sort(sorters=[FieldSort("docid",SortOrder.ASC)]),get_total_count=False,limit=100),
                                                                                ColumnsToGet([document_attachment_path],ColumnReturnType.SPECIFIED))
            list_data = []
            list_d = getRow_ots(rows)
            list_data.extend(list_d)

            while next_token:
                rows,next_token,total_count,is_all_succeed = self.ots_client.search(self.document_table,self.document_table_index,
                                                                                    SearchQuery(bool_q,next_token=next_token,get_total_count=False,limit=100),
                                                                                    ColumnsToGet([document_attachment_path],ColumnReturnType.SPECIFIED))
                list_d = getRow_ots(rows)
                list_data.extend(list_d)
                if len(list_data)>=self.q_size:
                    break
            for _d in list_data:
                self.task_queue.put(_d)
            if len(list_data)>0:
                self.current_docid = list_data[-1].get(document_docid)
        except Exception as e:
            log("attachProcess producer error %s"%(str(e)))



    def getAttachments(self,list_filemd5,columns_to_get=[attachment_attachmenthtml]):
        list_attachment = []
        rows_to_get = []
        for _md5 in list_filemd5[:50]:
            primary_key = [(attachment_filemd5,_md5)]
            rows_to_get.append(primary_key)
        req = BatchGetRowRequest()
        req.add(TableInBatchGetRowItem(self.attachment_table,rows_to_get,columns_to_get,None,1))
        try:
            result = self.ots_client.batch_get_row(req)
            attach_result = result.get_result_by_table(self.attachment_table)
            for item in attach_result:
                if item.is_ok:
                    _dict = getRow_ots_primary(item.row)
                    if _dict is not None:
                        list_attachment.append(attachment(_dict))

        except Exception as e:
            log(str(list_filemd5))
            log("attachProcess comsumer error %s"%str(e))
        return list_attachment

    def getTitleFromHtml(self,filemd5,_html):
        _soup = BeautifulSoup(_html,"lxml")

        _find = _soup.find("a",attrs={"data":filemd5})
        _title = ""
        if _find is not None:
            _title = _find.get_text()
        return _title

    def getSourceLinkFromHtml(self,filemd5,_html):
        _soup = BeautifulSoup(_html,"lxml")

        _find = _soup.find("a",attrs={"filelink":filemd5})
        filelink = ""
        if _find is None:
            _find = _soup.find("img",attrs={"filelink":filemd5})
            if _find is not None:
                filelink = _find.attrs.get("src","")
        else:
            filelink = _find.attrs.get("href","")
        return filelink

    def transformSWF(self,bucket,attachment_hub_url,objectPath,localpath,swf_dir):
        swf_urls = []
        try:
            swf2images(localpath,swf_dir)
            list_files = os.listdir(swf_dir)
            list_files.sort(key=lambda x:x)
            headers = dict()
            headers["x-oss-object-acl"] = oss2.OBJECT_ACL_PUBLIC_READ
            for _file in list_files:
                swf_localpath = "%s/%s"%(swf_dir,_file)
                swf_objectPath = "%s/%s"%(objectPath.split(".")[0],_file)
                uploadFileByPath(bucket,swf_localpath,swf_objectPath,headers)
                _url = "%s/%s"%(attachment_hub_url,swf_objectPath)
                swf_urls.append(_url)
                os.remove(swf_localpath)
        except Exception as e:
            traceback.print_exc()
        return swf_urls

    def comsumer_handle(self,_item,result_queue):


        _dict = _item
        if type(_dict)==dict:
            _document = Document(_dict)
        else:
            _document = _dict
        log("process docid:%s"%(str(_document.getProperties().get(document_docid))))
        try:
            attachPaths = json.loads(_document.getProperties().get(document_attachment_path,'[]'))
            list_filemd5 = []
            dict_attach = {}
            for _d in attachPaths:
                _md5 = _d.get(document_attachment_path_filemd5)
                if _md5 is not None:
                    list_filemd5.append(_md5)
                    dict_attach[_md5] = _d

            if len(list_filemd5)>0:
                list_attachment = self.getAttachments(list_filemd5)
                _all_succeed = True
                list_html = []
                for attach in list_attachment:
                    #更新docids
                    filemd5 = attach.getProperties().get(attachment_filemd5,"")
                    docids = attach.getProperties().get(attachment_docids,"")
                    _html = attach.getProperties().get(attachment_attachmenthtml)
                    _dict = {"filemd5":filemd5,
                             "html":_html}
                    if len(_html)>10:
                        list_html.append(_dict)

                if len(list_html)>0:

                    html_d = {document_partitionkey:_document.getProperties().get(document_partitionkey),
                               document_docid:_document.getProperties().get(document_docid)}
                    d_html = Document(html_d)
                    for _ in range(3):
                        if d_html.fix_columns(self.ots_capacity,[document_dochtmlcon],True):
                            log("update docid:%d attachments"%(d_html.getProperties().get(document_docid)))
                            d_html.updateAttachment(list_html)
                            d_html.update_row(self.ots_capacity)
                            break

        except Exception as e:
            debug("attach process failed of docid:%s of error:%s"%(str(_document.getProperties().get(document_docid)),str(e)))
            traceback.print_exc()

    def comsumer(self):
        #对每篇公告的附件处理数据,查询附件是否以及上传完成和处理完成,未完全上传的直接跳过

        #上传完成但没有处理完成的调用接口处理，更新attachment

        #合并所有attachment数据,html部分写入dochtmlcon,对html取text后写入attachtextcon

        #更新数据

        log("comsumer task_queue size :%d"%(self.task_queue.qsize()))
        # mt = MultiThreadHandler(self.task_queue,self.comsumer_handle,None,60)
        mt = MultiHandler(self.task_queue,self.comsumer_handle,None,2,30)
        # mt = MultiProcessHandler(self.task_queue,self.comsumer_handle,None,30)
        mt.run()
        log("current docid:%d"%(self.current_docid))
        with open(self.doc_file,"w") as f:
            f.write(str(self.current_docid))

    def attachment_process(self):
        self.producer()
        self.comsumer()

    def schedule(self):
        _scheduler = BlockingScheduler()
        # _scheduler.add_job(self.attachment_process,"cron",second="*/1")
        _scheduler.add_job(self.producer,"cron",second="*/10")
        _scheduler.add_job(self.comsumer,"cron",second="*/10")
        _scheduler.start()


def start_attachAttachment():
    ap = AttachProcess()
    ap.schedule()

if __name__ == '__main__':
    ap = AttachProcess()
    ap.attachment_process()