

from BaseDataMaintenance.common.multiThread import MultiThreadHandler
from apscheduler.schedulers.blocking import BlockingScheduler
from BaseDataMaintenance.dataSource.source import *
from BaseDataMaintenance.common.Utils import *
from queue import Queue
from tablestore import *
from BaseDataMaintenance.model.ots.Preproject import *
from uuid import uuid4

class PreprojectFill():

    def __init__(self):
        self.ots_client = getConnect_ots()
        self.task_queue = Queue(3000)
        self.fill_concat_queue = Queue(10000)

    def fill_comsumer(self):

        def comsumer_handle(_row,result_queue):
            if _row.get(preproject_uuid) is None:
                _preproject = Preproject(_row)
                # 删除无uuid数据
                _preproject.delete_row(self.ots_client)
                return
            if _row.get(preproject_has_bidfile) is None:
                json_docids = _row.get(preproject_json_docids)
                if json_docids is not None:
                    docids = json.loads(json_docids)
                    list_docids = []
                    for a in docids[:30]:
                        for b in a.split(","):
                            list_docids.append(b)
                    atta_query = BoolQuery(should_queries=[TermQuery("docids",_d) for _d in list_docids[:100]])
                    atta_b_q = BoolQuery(must_queries=[TermQuery("classification","招标文件"),atta_query])
                    atta_rows,atta_next_token,atta_total_count,_ = self.ots_client.search("attachment","attachment_index",
                                                                                          SearchQuery(atta_b_q,get_total_count=True,limit=1),
                                                                                          ColumnsToGet(["docids"],ColumnReturnType.SPECIFIED))
                    if atta_total_count>0:
                        _row[preproject_has_bidfile] = 1
                    else:
                        _row[preproject_has_bidfile] = 0
            _preproject = Preproject(_row)
            _preproject.update_row(self.ots_client)


        _mul = MultiThreadHandler(self.task_queue,comsumer_handle,None,10)
        _mul.run()



    def fill_producer(self):
        # 存在uuid数据，补充'has_bidfile'字段
        q1 = BoolQuery(must_queries=[
            ExistsQuery("uuid"),
            BoolQuery(must_not_queries=[
                ExistsQuery("has_bidfile")
            ])
        ])
        # 无uuid数据，用于删除行数据
        q2 = BoolQuery(must_not_queries=[
                ExistsQuery("uuid")
            ])
        columns = ["uuid","has_bidfile","json_docids"]
        query = BoolQuery(should_queries=[q1,
                                          q2])
        rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
                                                                            SearchQuery(query,sort=Sort(sorters=[FieldSort("uuid")]),get_total_count=True,limit=100),
                                                                            ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
        while True or self.task_queue.full():
            if len(rows)>0:
                dict_rows = getRow_ots(rows)
                for _row in dict_rows:
                    self.task_queue.put(_row)
            else:
                break
            rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
                                                                                SearchQuery(query,next_token=next_token,get_total_count=True,limit=100),
                                                                                ColumnsToGet(columns,ColumnReturnType.SPECIFIED))

    def fill_contact_producer(self):
        q1 = BoolQuery(must_queries=[
            TermQuery("status",1),
            ExistsQuery("uuid")
                                       ])
        columns = ["status",preproject_tenderee,preproject_last_tenderee_contact,preproject_last_tenderee_phone,preproject_last_win_tenderer,preproject_last_win_tenderer_contact,preproject_last_win_tenderer_phone]
        query = q1
        rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
                                                                            SearchQuery(query,sort=Sort(sorters=[FieldSort("crtime",SortOrder.DESC)]),get_total_count=True,limit=100),
                                                                            ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
        dict_rows = getRow_ots(rows)
        for _row in dict_rows:
            self.fill_concat_queue.put(_row)
        while next_token:
            rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
                                                                                SearchQuery(query,next_token=next_token,get_total_count=True,limit=100),
                                                                                ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
            dict_rows = getRow_ots(rows)
            for _row in dict_rows:
                self.fill_concat_queue.put(_row)

    def fill_contact_comsumer(self):

        def comsumer_handle(_row,result_queue):

            product = _row.get(preproject_product)

            tenderee = _row.get(preproject_tenderee)
            tenderee_concat = _row.get(preproject_last_tenderee_contact)
            tenderee_phone = _row.get(preproject_last_tenderee_phone)

            win_tenderer = _row.get(preproject_last_win_tenderer)
            win_tenderer_concat = _row.get(preproject_last_win_tenderer_contact)
            win_tenderer_phone = _row.get(preproject_last_win_tenderer_phone)


            if tenderee is not None and tenderee!="":
                # if (tenderee_concat is None or tenderee_concat=="") and (tenderee_phone is None or tenderee_phone==""):
                if tenderee_phone is None or tenderee_phone=="":
                    #fill tenderee concat and phone
                    bool_query = BoolQuery(must_queries=[
                        TermQuery(preproject_tenderee,tenderee),
                        BoolQuery(should_queries=[
                            MatchPhraseQuery("doctextcon",product),
                            MatchPhraseQuery("attachmenttextcon",product)
                        ])
                    ])

                    rows,next_token,total_count,is_all_succeed = self.ots_client.search("project2","project2_index",
                                                                                        SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("page_time",SortOrder.DESC)]),limit=100),
                                                                                        ColumnsToGet(["tenderee_contact","tenderee_phone"],return_type=ColumnReturnType.SPECIFIED))
                    list_data = getRow_ots(rows)
                    _find = False
                    for _data in list_data:
                        tenderee_contact = _data.get("tenderee_contact")
                        tenderee_phone = _data.get("tenderee_phone")
                        # if (tenderee_contact is not None and tenderee_contact!="") or (tenderee_phone is not None and tenderee_phone!=""):
                        if tenderee_phone is not None and tenderee_phone!="":
                            _find = True
                            _row[preproject_last_tenderee_contact] = tenderee_contact
                            _row[preproject_last_tenderee_phone] = tenderee_phone
                            break
                    if not _find:
                        bool_query = BoolQuery(must_queries=[
                            TermQuery("status", 1),
                            TermQuery("enterprise_name",tenderee)
                        ])
                        rows,next_token,total_count,is_all_succeed = self.ots_client.search("enterprise_contact","enterprise_contact_index",
                                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("score",SortOrder.DESC)]),limit=5),
                                                                                            ColumnsToGet(["contact_person","phone_no"],return_type=ColumnReturnType.SPECIFIED))
                        list_data = getRow_ots(rows)
                        if len(list_data)>0:
                            # _data = list_data[0]
                            for _data in list_data:
                                tenderee_contact = _data.get("contact_person")
                                tenderee_phone = _data.get("phone_no")
                                if tenderee_phone is not None and tenderee_phone!="":
                                    _row[preproject_last_tenderee_contact] = tenderee_contact
                                    _row[preproject_last_tenderee_phone] = tenderee_phone
                                    break


            if win_tenderer is not None and win_tenderer!="":
                # if (win_tenderer_concat is None or win_tenderer_concat!="") and (win_tenderer_phone is None or win_tenderer_phone==""):
                if win_tenderer_phone is None or win_tenderer_phone=="":
                    # fill win_tenderer concat and phone
                    bool_query = BoolQuery(must_queries=[
                        TermQuery("win_tenderer",win_tenderer),
                        BoolQuery(should_queries=[
                            MatchPhraseQuery("doctextcon",product),
                            MatchPhraseQuery("attachmenttextcon",product)
                        ])
                    ])

                    rows,next_token,total_count,is_all_succeed = self.ots_client.search("project2","project2_index",
                                                                                        SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("page_time",SortOrder.DESC)]),limit=100),
                                                                                        ColumnsToGet(["win_tenderer_manager","win_tenderer_phone"],return_type=ColumnReturnType.SPECIFIED))
                    list_data = getRow_ots(rows)
                    _find = False
                    for _data in list_data:
                        _contact = _data.get("win_tenderer_manager")
                        _phone = _data.get("win_tenderer_phone")
                        # if (_contact is not None and _contact!="") or (_phone is not None and _phone!=""):
                        if _phone is not None and _phone!="" and _phone!=_row.get(preproject_last_tenderee_phone):
                            _find = True
                            _row[preproject_last_win_tenderer_contact] = _contact
                            _row[preproject_last_win_tenderer_phone] = _phone
                            break
                    if not _find:
                        bool_query = BoolQuery(must_queries=[
                            TermQuery("status", 1),
                            TermQuery("enterprise_name",win_tenderer)
                        ])
                        rows,next_token,total_count,is_all_succeed = self.ots_client.search("enterprise_contact","enterprise_contact_index",
                                                                                            SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("score",SortOrder.DESC)]),limit=5),
                                                                                            ColumnsToGet(["contact_person","phone_no"],return_type=ColumnReturnType.SPECIFIED))
                        list_data = getRow_ots(rows)
                        if len(list_data)>0:
                            # _data = list_data[0]
                            for _data in list_data:
                                _contact = _data.get("contact_person")
                                _phone = _data.get("phone_no")
                                # _phone不为空，且不等于招标人电话
                                if _phone is not None and _phone!="" and _phone!=_row.get(preproject_last_tenderee_phone):
                                    _row[preproject_last_win_tenderer_contact] = _contact
                                    _row[preproject_last_win_tenderer_phone] = _phone
                                    break

            _row["status"] = 0
            _preproject = Preproject(_row)
            _preproject.setValue("status",2,True)
            _preproject.update_row(self.ots_client)

        _mul = MultiThreadHandler(self.fill_concat_queue,comsumer_handle,None,20)
        _mul.run()

    def schedule(self):
        _scheduler = BlockingScheduler()
        _scheduler.add_job(self.fill_producer,"cron",minute="*/10")
        _scheduler.add_job(self.fill_comsumer,"cron",minute="*/10")
        _scheduler.add_job(self.fill_contact_producer,"cron",minute="*/1")
        _scheduler.add_job(self.fill_contact_comsumer,"cron",minute="*/1")
        _scheduler.start()

def start_fill_preproject():
    preprojectFill = PreprojectFill()
    preprojectFill.schedule()

def delete_wrong_data():
    import pandas as pd
    list_data = []
    task_queue = Queue()
    ots_client = getConnect_ots()
    q1 = BoolQuery(must_queries=[
        WildcardQuery("may_begin","000*"),
        # RangeQuery("crtime",'2022-05-24')
    ])
    rows,next_token,total_count,is_all_succeed = ots_client.search("preproject","preproject_index",
                                                                        SearchQuery(q1,sort=Sort(sorters=[FieldSort("uuid")]),get_total_count=True,limit=100),
                                                                        ColumnsToGet(return_type=ColumnReturnType.ALL))
    dict_rows = getRow_ots(rows)
    list_data.extend(dict_rows)
    _count = len(dict_rows)
    print("total_count",total_count)
    while next_token:
        rows,next_token,total_count,is_all_succeed = ots_client.search("preproject","preproject_index",
                                                                       SearchQuery(q1,next_token=next_token,get_total_count=True,limit=100),
                                                                       ColumnsToGet(return_type=ColumnReturnType.ALL))
        dict_rows = getRow_ots(rows)
        list_data.extend(dict_rows)
        _count += len(dict_rows)
        print("%d/%d"%(_count,total_count))
        # if _count>10000:
        #     break
    # df = pd.read_csv("fa45de36-6e47-4817-b2d6-e79ea8c154601.csv")
    # for tenderee,product,may_begin,may_end in zip(df["tenderee"],df["product"],df["may_begin"],df["may_end"]):
    #     _data = {"tenderee":tenderee,
    #              "product":product,
    #              "may_begin":str(may_begin),
    #              "may_end":str(may_end)}
    #     list_data.append(_data)
    for _data in list_data:
        # may_begin = _data.get("may_begin")
        # may_end = _data.get("may_end")
        # if len(may_begin)!=10 or len(may_end)!=10:
        #     task_queue.put(_data)
        task_queue.put(_data)
    def _handle(item,result_queue):
        _preproject = Preproject(item)
        _preproject.delete_row(ots_client)
        print(item)
        item["may_begin"] = item["may_begin"].replace("0001","2022")
        item["may_end"] = item["may_end"].replace("0001","2022")
        _preproject = Preproject(item)
        _preproject.update_row(ots_client)
        print(item)
    log("====%d"%task_queue.qsize())
    mt = MultiThreadHandler(task_queue,_handle,None,30)
    mt.run()


def fixProductName():
    from BaseDataMaintenance.dataSource.source import getConnect_ots
    ots_client = getConnect_ots()
    bool_query = BoolQuery(must_queries=[MatchPhraseQuery("product","酒水降尘")])
    rows,next_token,total_count,is_all_succeed = ots_client.search("preproject","preproject_index",
                                                                   SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("uuid")]),get_total_count=True,limit=100),
                                                                   ColumnsToGet(return_type=ColumnReturnType.ALL))
    dict_rows = getRow_ots(rows)
    for _row in dict_rows:
        preproject1 = Preproject(_row)
        preproject1.delete_row(ots_client)
        _row["product"] = _row["product"].replace("酒水降尘","洒水降尘")
        preproject2 = Preproject(_row)
        preproject2.update_row(ots_client)
    print(dict_rows)

if __name__=="__main__":
    # preprojectFill = PreprojectFill()
    # preprojectFill.schedule()
    # delete_wrong_data()
    # fixProductName()
    start_fill_preproject()
    pass