

from tablestore import *
from BaseDataMaintenance.dataSource.source import getConnect_ots,getConnect_ots_capacity,getConnection_oracle

from BaseDataMaintenance.model.ots.document import Document
from BaseDataMaintenance.model.ots.document_tmp import Document_tmp
from BaseDataMaintenance.model.ots.document_html import Document_html

from BaseDataMaintenance.common.Utils import *
from BaseDataMaintenance.common.multiThread import MultiThreadHandler

from queue import Queue

def fixDoc():
    bool_query = BoolQuery(must_queries=[RangeQuery("crtime","2022-08-02 17:00:00","2022-08-02 18:00:00")])
    ots_client =getConnect_ots()
    capacity_client =getConnect_ots_capacity()
    rows,next_token,total_count,is_all_succeed = ots_client.search("document_tmp","document_tmp_index",
                                                                   SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("docid")]),get_total_count=True,limit=100),
                                                                   columns_to_get=ColumnsToGet(return_type=ColumnReturnType.ALL))
    task_queue = Queue()
    list_data = getRow_ots(rows)

    for _data in list_data:
        task_queue.put(_data)
    while next_token:
        rows,next_token,total_count,is_all_succeed = ots_client.search("document_tmp","document_tmp_index",
                                                                       SearchQuery(bool_query,next_token=next_token,get_total_count=True,limit=100),
                                                                       columns_to_get=ColumnsToGet(return_type=ColumnReturnType.ALL))
        list_data = getRow_ots(rows)

        for _data in list_data:
            task_queue.put(_data)
        # if task_queue.qsize()>4000:
        #     break
        log("%d/%d"%(task_queue.qsize(),total_count))

    def _handle(item,result_queue):
        docid = item.get("docid")
        if "all_columns" in item:
            item.pop("all_columns")
        _d = Document(item)
        _exists = _d.exists_row(ots_client)

        if not _exists:
            _d.setValue("status",1,True)
            _d_html = Document_html({"partitionkey":_d.getProperties().get("partitionkey"),
                                "docid":_d.getProperties().get("docid")})
            _d_html.fix_columns(ots_client,["dochtmlcon"],True)

            _dict_d = _d.getProperties()
            listk = []
            for k,v in _dict_d.items():
                if k in ("COLUMN_MAX_SIZE","all_columns","table_name","prefixs"):
                    listk.append(k)
            for k in listk:
                _dict_d.pop(k)

            listk = []
            _dict_html = _d_html.getProperties()
            for k,v in _dict_html.items():
                if k not in ("partitionkey","docid","dochtmlcon"):
                    listk.append(k)
            for k in listk:
                _dict_html.pop(k)

            _d1 = Document(_dict_d)
            _d1.update_row(ots_client)
            _d2 = Document(_dict_html)
            _d2.update_row(capacity_client)

            log("docid:%s not exists"%(str(docid)))

    mt = MultiThreadHandler(task_queue,_handle,None,30)
    mt.run()


def fix_html_from_oracle():
    ots_client = getConnect_ots()
    ots_capacity = getConnect_ots_capacity()
    conn_oracle = getConnection_oracle()
    cursor = conn_oracle.cursor()
    from BaseDataMaintenance.model.oracle.GongGaoTemp import dict_channel_table

    _query = BoolQuery(must_queries=[TermQuery("docid",233591284)])
    rows,next_token,total_count,is_all_succeed = ots_client.search("document_extract2","document_extract2_index",
                                                                   SearchQuery(_query,limit=1),
                                                                   ColumnsToGet(column_names=["uuid","original_docchannel"],return_type=ColumnReturnType.SPECIFIED))
    list_data = getRow_ots(rows)
    for _data in list_data:
        d = Document(_data)
        d.fix_columns(ots_client,["uuid","original_docchannel"],True)
        _channel = d.getProperties().get("original_docchannel")
        _uuid = d.getProperties().get("uuid")
        for _channel,_table_name in dict_channel_table.items():
            if _channel==527:
                _channel = 52
            _table_name = dict_channel_table[str(_channel)]
            _table_name = _table_name.replace("_TEMP","")

            sql = "select page_content from %s where id='%s'"%(_table_name,_uuid)
            cursor.execute(sql)
            rows = cursor.fetchall()
            print(_table_name)
            print(rows)
    print(d.getProperties())
    print(list_data)

if __name__ == '__main__':
    # fixDoc()
    fix_html_from_oracle()