
from BaseDataMaintenance.model.ots.BaseModel import BaseModel
from BaseDataMaintenance.common.Utils import *
import json
from BaseDataMaintenance.dataSource.source import getConnect_ots
from tablestore import *

ots_client = getConnect_ots()

class t_shen_pi_xiang_mu(BaseModel):

    def __init__(self,_dict):

        for k,v in _dict.items():
            self.setValue(k,v,True)
        self.table_name = "t_shen_pi_xiang_mu"

    def getPrimary_keys(self):
        return ["id"]



def delete_dumplicate():
    from BaseDataMaintenance.common.multiThread import MultiThreadHandler

    def produce(task_queue,ots_client):
        bool_query = BoolQuery(must_queries=[TermQuery("web_source_no","to_delete")])
        rows,next_token,total_count,is_all_succeed = ots_client.search("t_shen_pi_xiang_mu","t_shen_pi_xiang_mu_index",
                                                                       SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("id")]),get_total_count=True,limit=100),
                                                                       columns_to_get=ColumnsToGet(return_type=ColumnReturnType.NONE))
        list_data = getRow_ots(rows)
        _count = len(list_data)
        for _data in list_data:
            task_queue.put(_data)
        log("total_count:%d"%(total_count))
        while next_token:
            rows,next_token,total_count,is_all_succeed = ots_client.search("t_shen_pi_xiang_mu","t_shen_pi_xiang_mu_index",
                                                                           SearchQuery(bool_query,next_token=next_token,get_total_count=True,limit=100),
                                                                           columns_to_get=ColumnsToGet(return_type=ColumnReturnType.NONE))
            list_data = getRow_ots(rows)
            _count += len(list_data)
            for _data in list_data:
                task_queue.put(_data)
            if _count>=10000:
                break

    def comsume(item,result_queue):
        a = t_shen_pi_xiang_mu(item)
        a.delete_row(ots_client)


    from queue import Queue
    task_queue= Queue()

    while 1:

        produce(task_queue,ots_client)
        if task_queue.qsize()>0:
            mt = MultiThreadHandler(task_queue,comsume,None,30)
            mt.run()


if __name__=="__main__":

    delete_dumplicate()

