import os from selenium import webdriver from selenium.webdriver import DesiredCapabilities from bs4 import BeautifulSoup import platform import requests from lxml import html import re import signal from threading import Thread, RLock from module.Utils import * from queue import Queue import time header={ "Accept": "text/html, application/xhtml+xml, image/jxr, */*", # "Referer": "http://uia.hnist.cn/sso/login?service=http%3A%2F%2Fportal.hnist.\ # cn%2Fuser%2FsimpleSSOLogin", "Accept-Language": "zh-Hans-CN,zh-Hans;q=0.8,en-US;q=0.5,en;q=0.3", "Content-Type": "application/x-www-form-urlencoded", "Connection": "Keep-Alive", "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36", #"Accept-Encoding": "gzip, deflate", # "Origin": "http://uia.hnist.cn", "Upgrade-Insecure-Requests": "1", } TYPE = "phantomjs" # TYPE = "chrome" current_path = os.path.abspath("/".join(__file__.split("\\")[:-1])) driver_paths = {"phantomjs_linux":current_path+"/../driver/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs", "phantomjs_window":current_path+"/../driver/phantomjs/phantomjs-2.1.1-windows/bin/phantomjs.exe", "chrome_linux":current_path+"/../driver/chromedriver/chromedriver_linux64/chromedriver", "chrome_window":current_path+"/../driver/chromedriver/chromedriver_win32/chromedriver.exe"} print(driver_paths) def getBrowser_phantomJS(platform="linux",straight=False): if platform=="linux": executable_path = driver_paths["phantomjs_linux"] else: executable_path = driver_paths["phantomjs_window"] desired_capabilities= DesiredCapabilities.PHANTOMJS.copy() # print('os.path.exists executable_path', executable_path, os.path.exists(executable_path)) for key, value in header.items(): desired_capabilities['phantomjs.page.customHeaders.{}'.format(key)] = value desired_capabilities['phantomjs.page.customHeaders.User-Agent'] ='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.366' desired_capabilities["phantomjs.page.settings.loadImages"] = False desired_capabilities["phantomjs.page.settings.disk-cache"] = False browser_phantomjs = webdriver.PhantomJS(executable_path=executable_path,desired_capabilities=desired_capabilities,service_args=['--ignore-ssl-errors=true','--ssl-protocol=TLSv1']) browser_phantomjs.implicitly_wait(10) browser_phantomjs.set_script_timeout(20) browser_phantomjs.set_page_load_timeout(10) return browser_phantomjs def getBrowser_chrome(platform="linux",straight=False): if platform=="linux": executable_path = driver_paths["chrome_linux"] else: executable_path = driver_paths["chrome_window"] chrome_options = webdriver.ChromeOptions() prefs = {"profile.managed_default_content_settings.images":2} chrome_options.add_experimental_option("prefs",prefs) chrome_options.add_argument('--headless') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.36') desired_capabilities= DesiredCapabilities.CHROME.copy() desired_capabilities['loggingPrefs'] = { 'performance':'ALL' } browser_chrome = webdriver.Chrome(desired_capabilities=desired_capabilities,executable_path=executable_path, chrome_options=chrome_options,service_args=['--ignore-ssl-errors=true','--ssl-protocol=any']) # '--ssl-protocol=any' TLSv1 browser_chrome.implicitly_wait(15) browser_chrome.set_page_load_timeout(15) # browser_chrome = webdriver.Chrome(executable_path=executable_path) return browser_chrome def getBrowser(type=TYPE,straight=False): if platform.system()=="Windows": if type=="phantomjs": _browser = getBrowser_phantomJS(platform="window",straight=straight) else: _browser = getBrowser_chrome(platform="window",straight=straight) else: if type=="phantomjs": _browser = getBrowser_phantomJS(straight=straight) else: _browser = getBrowser_chrome(straight=straight) _browser.maximize_window() return _browser def getStatus(url): try: r = requests.get(url, headers=header, allow_redirects = False,timeout=15) except Exception as e: log('requests.get error :%s'%e) return 404 return r.status_code _queue = Queue(4) lock = RLock()#用于获取池中的driver _get_count = 0#记录调用次数 lock_kill = RLock()#用于杀掉进程，如果已经有一个kill线程启动了，则其他线程不启动 def releaseAllDriver(): ''' @summary: 通过线程释放所有driver,并重新初始化 ''' def _method(): if lock_kill.acquire(blocking=True, timeout=0): try: lock.acquire() wait_count = 0 t0 = time.time() while(True): if _queue.full(): break elif time.time()-t0>60: log('等待放回浏览器超时，强制释放所有driver') break else: wait_count += 1 log("waitting for drivers all back..."+str(wait_count)+"qsize:"+str(_queue.qsize())) time.sleep(1) initQueue() finally: lock.release() lock_kill.release() t = Thread(target=_method) t.start() t.join(100) def killAllDriver(): ''' @summary: 干掉所有webdriver进程 ''' try: log("killing all webdrivers") if platform.system()=="Windows": if TYPE=="phantomjs": os.system('taskkill /im phantomjsdriver.exe /F') os.system('taskkill /im phantomjs.exe /F') else: os.system('taskkill /im chromedriver.exe /F') os.system('taskkill /im chrome.exe /F') else: if TYPE=="phantomjs": os.system('killall phantomjsdriver') os.system('killall phantomjs') else: os.system('killall chromedriver ') os.system('killall chrome') except Exception as e: error(str(e)) def adddriver(browser): _queue.put(browser) def getdriver(): ''' @summary: 获取driver，要先获得锁 ''' global _get_count _get_count += 1 if _get_count>1000: log("get_driver 达到调用次数，重新进行初始化") releaseAllDriver() _get_count = 0 lock.acquire() browser = _queue.get() lock.release() return browser def initQueue(): ''' @summary: 初始化webdriver队列，每调用一定的次数或者卡死的时候进行初始化，将所有webdriver进程干掉，然后初始化这个队列 ''' #释放资源 while(not _queue.empty()): _driver = _queue.get() _driver = None #杀死进程 killAllDriver() #初始化 while(not _queue.full()): browser = getBrowser() adddriver(browser) initQueue() def executeMethod(_method,args): ''' @summary: 执行所有需要browser的方法，确保driver拿到之后会放回队列中 ''' try: browser = getdriver(_queue) args["browser"] = browser _method(args) finally: adddriver(browser) def hasDrew(url,list_rule): ''' @summary: 根据规则判断是否渲染 @param: url:网页链接，list_rule: xpath规则数组 ''' def hasdrew(url,list_rule): try: r = requests.get(url, headers=header, allow_redirects = False, timeout=10) _encoding = r.encoding if _encoding is None: _encoding = "utf8" dom = html.fromstring(r.content.decode(_encoding)) for item in list_rule: if item["type"]=="xpath": if item["rule"] is not None: list_nodes = dom.xpath(item["rule"]) if len(list_nodes)==0: return True except Exception as e: error(str(e)) return False rs = thread_run(hasdrew, url,list_rule) if rs != None: return rs else: return False def loadPage(browser,url,timeout=30): ''' @summary: 解决selenium加载网页不返回的问题，设置线程进行加载，对线程设置超时时间 ''' def _thread_load(browser,url): try: debug("load "+url) browser.get(url) debug("load "+url+" done") except Exception as e: error(str(e)) log('加载页面抛出异常：'+str(e)) if re.search("由于目标计算机积极拒绝",str(e)) is not None: log('log page exception') releaseAllDriver() t = Thread(target=_thread_load,args=(browser,url)) t.start() debug("load_thread:"+str(t.ident)+" "+str(t.name)) t.join(timeout) if t.isAlive(): #browser = None ''' browser.service.process.send_signal(signal.SIGTERM) ''' #执行释放资源的线程 error("driver get方法卡住，强制释放所有资源") stop_thread(t) log('stop_loadpage thread return false') adddriver(browser) debug("release driver") releaseAllDriver() return False # raise NameError("超时加载"+str(url)) return True def getSource(url): ''' #import chardet sess = requests.Session() sess.headers = header data=sess.get(url) encoding = data.encoding #print(encoding) data = data.text.encode(data.encoding) content = data.decode(encoding) #print(content) browser = webdriver.PhantomJS() browser.get(url) iframe = browser.find_elements_by_tag_name("iframe")[0] browser.switch_to_frame(iframe) print(browser.page_source) content = browser.page_source print(content) #browser.close() ''' #browser = getBrowser_chrome() browser = getBrowser() browser.get(url) element = browser.find_elements_by_xpath('//*') for ele in element: print(ele.rect) #print(element.value_of_css_property("width")) #print(element.get_attribute("innerHTML")) content = browser.page_source soup = BeautifulSoup(content,"lxml") iframes = soup.find_all("iframe",recursive=True) if len(iframes)>0: browser_iframes = browser.find_elements_by_tag_name("iframe") for iframe,browser_iframe in zip(iframes,browser_iframes): browser.switch_to_default_content() browser.switch_to_frame(browser_iframe) iframe.replace_with(BeautifulSoup(browser.page_source,"lxml")) return soup if __name__=="__main__": url = "http://yl.km.gov.cn/tzgg/" #source = getSource(url) hasDrew(url,[{"rule":'//*[@class="li_c"]',"type":"xpath"}])