#encoding:UTF8
from BaseDataMaintenance.dataSource.pool import ConnectorPool
from BaseDataMaintenance.dataSource.source import *
from BaseDataMaintenance.common.Utils import *
import queue
from tablestore import *
from multiprocessing import RLock
from threading import Thread
from apscheduler.schedulers.blocking import BlockingScheduler
from BaseDataMaintenance.common.multiThread import MultiThreadHandler
from BaseDataMaintenance.model.ots.proposedBuilding_tmp import proposedBuilding_tmp
from BaseDataMaintenance.model.ots.designed_project import designed_project
import traceback

class DataSynchronization():

    def __init__(self):
        self.done_lock = RLock()
        self.isDone = False
        self.proposedBuilding_table = "proposedBuilding_tmp"
        self.proposedBuilding_table_index = "proposedBuilding_tmp_index"

        self.designed_project_table = "designed_project"
        self.designed_project_table_index = "designed_project_index"
        self.ots_client = getConnect_ots()

        self.task_queue = queue.Queue()

    def producer(self,task_queue):
        '''
        :return:生产数据
        '''
        ots_client = self.ots_client

        bool_query = BoolQuery(must_queries=[ExistsQuery("crtime")])

        columns = ["uuid","crtime","json_list_group"]

        rows, next_token, total_count, is_all_succeed = ots_client.search(self.proposedBuilding_table, self.proposedBuilding_table_index,
                                                                          SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort("crtime",SortOrder.ASC)]), limit=100, get_total_count=True),
                                                                          ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
        list_data = getRow_ots(rows)
        for _data in list_data:
            _proposed = proposedBuilding_tmp(_data)
            task_queue.put(_proposed,True)
        _count = len(list_data)
        while next_token:
            rows, next_token, total_count, is_all_succeed = ots_client.search(self.proposedBuilding_table, self.proposedBuilding_table_index,
                                                                              SearchQuery(bool_query ,next_token=next_token, limit=100, get_total_count=True),
                                                                              ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
            list_data = getRow_ots(rows)
            for _data in list_data:
                _proposed = proposedBuilding_tmp(_data)
                task_queue.put(_proposed,True)
            _count += len(list_data)
            if _count>3000:
                break


    def comsumer_handle(self,_proposed,result_queue):


        #修改designed_project
        _time = time.time()

        try:
            _project_dict = _proposed.toDesigned_project(self.ots_client)
            log("toDesigned_project takes %.2fs"%(time.time()-_time))
            _time = time.time()
            if _project_dict is not None:
                #更新数据
                # log("project_dict:"+json.dumps(_project_dict))
                _designed_project = designed_project(_project_dict)
                _designed_project.update_project(self.ots_client)
                log("update desined takes %.2fs"%(time.time()-_time))

            #删除tmp
            _proposed.delete_row(self.ots_client)
            log("update designed takes %.2fs"%(time.time()-_time))
        except Exception as e:
            log("comsumer failed cause of %s"%(str(e)))
            log("proposed:%s"%(str(_proposed)))
            log(traceback.format_exc())

    def comsumer(self,task_queue):
        mt = MultiThreadHandler(task_queue,self.comsumer_handle,None,thread_count=30)
        mt.run()


    def waitTask(self,task_queue):
        for i in range(60):
            if task_queue.qsize()>0:
                return True
            else:
                time.sleep(1)
        return False

    def maxcompute2ots(self):


        self.producer(self.task_queue)

        # _dict = {"uuid":"12313","crtime":123,
        #                 "json_list_group":'''
        #                 [{"docid": 218170163, "shenpi_id": null, "type_id": "218170163", "page_time": "2022-01-26", "province": "广东", "city": "深圳", "district": "罗湖", "tenderee": "深圳市城市建设开发（集团）有限公司", "tenderee_contact": "孙工", "tenderee_phone": "18998998087", "agency": "", "project_code": null, "project_name": "成都市白鹭湾(住宅用地)项目可行性研究", "doctitle": "成都市白鹭湾（住宅用地）项目可行性研究服务招标公告", "docchannel": "52", "stage": "可研阶段", "proportion": "建筑面积为32388.83㎡", "projectDigest": "招标信息)，六、开标地点：深圳市罗湖区桂园街道滨河东路1011号鹿丹大厦12层会议室；(鉴于疫情原因，投标人自行决定是否到开标现场开标；如投标人需要到开标现场，请投标人关注、执行深圳市关于疫情的相关规定，并提前2天与招标人进行沟通协商。)，七、投标人资格标准等详细内容详见招标文件。招标联系人：孙工；联系电话：18998998087；邮箱：sundh@szcjjt.com；张工；联系电话：13928429353；邮箱：zli@szcjjt.com", "projectAddress": null, "begin_time": null, "end_time": null, "project_name_refind": "成都市白鹭湾(住宅用地)项目可行性研究", "industry": "办公楼", "location": null, "section_num": "-1", "new_enough": 1, "follow_enough": 1}]
        #                 '''}
        # task_queue.put(proposedBuilding_tmp(_dict))

        self.comsumer(self.task_queue)

    def turn_stage(self):
        '''
        根据latest_service_time更新stage
        :return:
        '''

        ots_client = self.ots_client
        current_date = getCurrent_date("%Y-%m-%d")

        bool_query = BoolQuery(must_queries=[RangeQuery("latest_service_time",range_from="2000-01-01",range_to=current_date)],
                               must_not_queries=[TermQuery("progress","竣工阶段")])

        columns = ["progress"]

        rows, next_token, total_count, is_all_succeed = ots_client.search(self.designed_project_table, self.designed_project_table_index,
                                                                          SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort("crtime",SortOrder.DESC)]), limit=100, get_total_count=True),
                                                                          ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
        list_data = getRow_ots(rows)
        print("total_count",total_count)
        for _data in list_data:
            _proposed = designed_project(_data)
            _proposed.setValue("progress","竣工阶段",True)
            _proposed.update_row(ots_client)
        _count = len(list_data)
        while next_token:
            rows, next_token, total_count, is_all_succeed = ots_client.search(self.designed_project_table, self.designed_project_table_index,
                                                                              SearchQuery(bool_query ,next_token=next_token, limit=100, get_total_count=True),
                                                                              ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
            list_data = getRow_ots(rows)
            for _data in list_data:
                _proposed = designed_project(_data)
                _proposed.setValue("progress","竣工阶段",True)
                _proposed.update_row(ots_client)
            _count += len(list_data)

    def fix_progress(self):
        ots_client = self.ots_client
        current_date = getCurrent_date("%Y-%m-%d")

        bool_query = BoolQuery(must_queries=[RangeQuery("latest_service_time",range_to="1001"),
                                             TermQuery("progress","竣工阶段")])
        columns = ["latest_service_time","json_list_group"]
        task_queue = queue.Queue()
        rows, next_token, total_count, is_all_succeed = ots_client.search(self.designed_project_table, self.designed_project_table_index,
                                                                          SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort("crtime",SortOrder.DESC)]), limit=100, get_total_count=True),
                                                                          ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))


        list_data = getRow_ots(rows)
        for _data in list_data:
            if _data.get("latest_service_time") is not None:
                _proposed = proposedBuilding_tmp(_data)
                task_queue.put(_proposed,True)
                break
        print(list_data)
        print("total_count",total_count)
        while next_token:
            rows, next_token, total_count, is_all_succeed = ots_client.search(self.designed_project_table, self.designed_project_table_index,
                                                                              SearchQuery(bool_query ,next_token=next_token, limit=100, get_total_count=True),
                                                                              ColumnsToGet(columns,return_type=ColumnReturnType.SPECIFIED))
            list_data = getRow_ots(rows)
            for _data in list_data:
                if _data.get("latest_service_time") is not None:
                    _proposed = proposedBuilding_tmp(_data)
                    task_queue.put(_proposed,True)
        print("task_queue size %d"%(task_queue.qsize()))

        def _handle(_proposed,result_queue):
            try:
                _time = time.time()
                _project_dict = _proposed.toDesigned_project(ots_client)
                log("toDesigned_project takes %.2fs"%(time.time()-_time))
                _time = time.time()
                if _project_dict is not None:
                    #更新数据
                    log("project_dict:"+json.dumps(_project_dict,ensure_ascii=False))
                    _designed_project = designed_project(_project_dict)
                    _designed_project.update_project(ots_client)
                    log("update desined takes %.2fs"%(time.time()-_time))

            except Exception as e:
                log("comsumer failed cause of %s"%(str(e)))
                log("proposed:%s"%(str(_proposed)))
                log(traceback.format_exc())

        mt=MultiThreadHandler(task_queue,_handle,None,30)
        mt.run()

    def drop_data(self):
        ots_client = getConnect_ots()

        bool_query = BoolQuery(must_queries=[ExistsQuery("crtime")])

        task_queue = queue.Queue()

        rows, next_token, total_count, is_all_succeed = ots_client.search(self.proposedBuilding_table, self.proposedBuilding_table_index,
                                                                          SearchQuery(bool_query ,sort=Sort(sorters=[FieldSort("crtime",SortOrder.ASC)]), limit=100, get_total_count=True),
                                                                          ColumnsToGet(return_type=ColumnReturnType.SPECIFIED))
        list_data = getRow_ots(rows)
        for _data in list_data:
            _proposed = proposedBuilding_tmp(_data)
            task_queue.put(_proposed,True)
            print(total_count,task_queue.qsize())
        _count = len(list_data)
        while next_token:
            rows, next_token, total_count, is_all_succeed = ots_client.search(self.proposedBuilding_table, self.proposedBuilding_table_index,
                                                                              SearchQuery(bool_query ,next_token=next_token, limit=100, get_total_count=True),
                                                                              ColumnsToGet(return_type=ColumnReturnType.SPECIFIED))
            list_data = getRow_ots(rows)
            for _data in list_data:
                _proposed = proposedBuilding_tmp(_data)
                task_queue.put(_proposed,True)
                print(total_count,task_queue.qsize())

        def _handle(item,result_queue):
            item.delete_row(ots_client)

        mt = MultiThreadHandler(task_queue,_handle,None,30)
        mt.run()

    def scheduler(self):
        from BaseDataMaintenance.maintenance.major_project.unionDocument import MajorUnion
        mu = MajorUnion()
        _scheduler = BlockingScheduler()
        _scheduler.add_job(self.maxcompute2ots,"cron",minute="*/1")
        _scheduler.add_job(self.turn_stage,"cron",hour="*/5")
        _scheduler.add_job(mu.comsumer,"cron",minute="*/8")
        _scheduler.start()

def startSychro():
    ds = DataSynchronization()
    ds.scheduler()




if __name__=="__main__":
    ds = DataSynchronization()
    # # ds.scheduler()
    # ds.maxcompute2ots()
    # ds.turn_stage()
    # ds.fix_progress()
    ds.drop_data()


