

from BaseDataMaintenance.common.multiThread import MultiThreadHandler
from BaseDataMaintenance.dataSource.source import *
from BaseDataMaintenance.model.mysql.documentRedownload import DocumentRedownload
from queue import Queue
from BaseDataMaintenance.common.Utils import *
from tablestore import *
from BaseDataMaintenance.dataSource.pool import ConnectorPool
from bs4 import BeautifulSoup
import traceback

def docRedownload():
    task_queue = Queue()

    ots_client = getConnect_ots()

    bool_query = BoolQuery(must_queries=[
        # RangeQuery("crtime",'2021-08-10 00:00:00','2021-10-10 00:00:00'),
        # RangeQuery("docid",174453795)
        WildcardQuery("web_source_no","17147*"),
        # should_docid
    ]
        ,must_not_queries=[NestedQuery("page_attachments",WildcardQuery("page_attachments.fileMd5","*"))]
    )

    bool_query1 = BoolQuery(must_queries=[
        WildcardQuery("web_source_no","17147*"),
        NestedQuery('page_attachments',TermQuery("page_attachments.fileMd5",'d41d8cd98f00b204e9800998ecf8427e')),
    ])

    query = BoolQuery(
        # must_queries=[TermQuery("docid",143977755)],
                      should_queries=[bool_query,bool_query1])

    rows,next_token,total_count,is_all_succeed = ots_client.search("document","document_index",
                                                                        SearchQuery(query,sort=Sort(sorters=[FieldSort("docid",SortOrder.ASC)]),get_total_count=True,limit=100),
                                                                        ColumnsToGet(["uuid","dochtmlcon","original_docchannel"],ColumnReturnType.SPECIFIED))
    log("total_count:%d"%total_count)
    counts = 0
    list_dict = getRow_ots(rows)
    for _dict in list_dict:
        task_queue.put(_dict)
    counts += len(list_dict)
    while next_token:
        rows,next_token,total_count,is_all_succeed = ots_client.search("document","document_index",
                                                                       SearchQuery(query,next_token=next_token,get_total_count=True,limit=100),
                                                                       ColumnsToGet(["uuid","dochtmlcon","original_docchannel"],ColumnReturnType.SPECIFIED))
        list_dict = getRow_ots(rows)
        for _dict in list_dict:
            task_queue.put(_dict)
        counts += len(list_dict)
        log("%d/%d"%(counts,total_count))

    def get_detail_link(dochtmlcon):
        _soup = BeautifulSoup(dochtmlcon,"lxml")
        list_a = _soup.find_all("a")
        for _a in list_a:
            _link = _a.attrs.get("href")
            if re.search("http.+\.html",_link) is not None:
                return _link

    def _handle(_dict,result_queue,pool_oracle):
        try:
            conn = pool_oracle.getConnector()
            _dict["docchannel"] = _dict.get("original_docchannel")
            _detail_link = get_detail_link(_dict.get("dochtmlcon"))
            if _detail_link is not None:
                _dict["detail_link"] = _detail_link
                _d = DocumentRedownload(_dict)
                _d.insert_row(conn)
            print(_dict)

        except Exception as e:
            traceback.print_exc()
        finally:
            pool_oracle.putConnector(conn)

    pool_oracle = ConnectorPool(10,30,getConnection_oracle)
    mt = MultiThreadHandler(task_queue,_handle,None,30,pool_oracle=pool_oracle)
    mt.run()
    pool_oracle.destory()



if __name__ == '__main__':
    docRedownload()