BIDI_EXTRACTION_AI/BiddingKG/dl/fingerprint/documentFingerprint.py

import hashlib
import codecs
from bs4 import BeautifulSoup
import re


def getHtmlText(sourceHtml):
    _soup = BeautifulSoup(sourceHtml,"lxml")
    list_a = _soup.find_all("a")
    for _a in list_a:
        _href = _a.attrs.get("href","")
        if _href.find("www.bidizhaobiao.com")>0:
            _a.decompose()
    # richText = _soup.find("div",attrs={"class":"richTextFetch"})
    # if richText is not None:
    #     richText.decompose()
    _text = _soup.get_text()

    _text = re.sub("\s*",'',_text)
    # 删除“阅读量|点击量”等描述,避免同一站源重复爬取去重失败
    re_page_views = re.compile("(阅读|点击|浏览|访问|访客|阅览|查看|曝光)(量|数|次数|人数|人次)[:：]?\d{1,8}")
    _text = re.sub(re_page_views,"",_text[:500],count=1) + _text[500:]
    if len(_text)==0:
        _text = str(_soup)
    return _text

def getMD5(sourceHtml):
    if sourceHtml is not None and len(sourceHtml)>0:
        _text = getHtmlText(sourceHtml)
        if isinstance(_text,str):
            bs = _text.encode()
        elif isinstance(_text,bytes):
            bs = _text
        else:
            return ""
        md5 = hashlib.md5()
        md5.update(bs)
        return md5.hexdigest()
    return ""


def getFingerprint(sourceHtml):
    md5 = getMD5(sourceHtml)
    if md5!="":
        _fingerprint = "md5=%s"%(md5)
    else:
        _fingerprint = ""
    return _fingerprint

if __name__=="__main__":
    sourceHtml = codecs.open("C:\\Users\\\Administrator\\Desktop\\2.html","rb",encoding="utf8").read()
    # sourceHtml = "abcddafafffffffffffffffffffffffff你"
    print(getFingerprint(sourceHtml))