python爬虫——selenium

pip install selenium

​
from selenium import webdriver
import time
driver=webdriver.Chrome('./chromedriver.exe')#初始化WebDriver对象
driver.get('https://www.baidu.com')#访问百度
time.sleep(20)

​

from selenium import webdriver
from  selenium.webdriver.common.by import By
import time
driver=webdriver.Chrome('./chromedriver.exe')
driver.get('http://www.baidu.com')
#css选择器进行筛选
#如果有多个元素被匹配，find_element只会获取第一个
result=driver.find_element(by=By.CSS_SELECTOR,value='.mnav.c-font-normal.c-color-t')
print(result,'\n','---------------')
#匹配多个find_elements
result_list=driver.find_elements(by=By.CSS_SELECTOR,value='.mnav.c-font-normal.c-color-t')
for item in result_list:
    print(item)
print('---------------')
result_byid=driver.find_element(by=By.NAME,value='tj_more')
print(result_byid)

<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="bd760d4c-a045-4316-b398-7cb4ae7b6f6e")> 
 ---------------
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="bd760d4c-a045-4316-b398-7cb4ae7b6f6e")>
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="4c25b06b-8dd5-4da7-b362-43965f389102")>
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="43406d63-edea-4a7b-b2d2-307a7ad16714")>
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="c3434610-4579-4f43-9d42-42837dc8b151")>
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="fb6d11ff-f87b-4993-bad0-b35876e85c46")>
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="d8388d32-f732-488b-8b15-92a003c2a217")>
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="e8b99605-50d0-4679-9ef6-4a708d4d8ec8")>
---------------
<selenium.webdriver.remote.webelement.WebElement (session="695c760a0bf340402875429f56903178", element="e448b794-6c02-4683-8225-25a2d647d28d")>

driver=webdriver.Chrome('./chromedriver.exe')
driver.get('http://www.baidu.com')
result=driver.find_element(by=By.CSS_SELECTOR,value='.mnav.c-font-normal.c-color-t')
print(result.text,result.get_attribute('href'),'\n','------------------------------------')
result_list=driver.find_elements(by=By.CSS_SELECTOR,value='.mnav.c-font-normal.c-color-t')
for item in result_list:
    print(item.text,item.get_attribute('href'))
print('---------------------------------')
#webdriver只与可见元素交互，如果该元素被隐藏，将不会被获取文本值，但是属性仍可获取到，可通过is_displayed()进行查看
result_byid=driver.find_element(by=By.NAME,value='tj_more')
print({'text':result_byid.text,'href':result_byid.get_attribute('href')})
print(result_byid.is_displayed())

新闻 http://news.baidu.com/ 
 ------------------------------------
新闻 http://news.baidu.com/
hao123 https://www.hao123.com/?src=from_pc
地图 http://map.baidu.com/
贴吧 http://tieba.baidu.com/
视频 https://haokan.baidu.com/?sfrom=baidu-top
图片 http://image.baidu.com/
网盘 https://pan.baidu.com/?from=1026962h
---------------------------------
{'text': '', 'href': 'http://www.baidu.com/more/'}
False

from selenium import webdriver
import time
driver=webdriver.Chrome('./chromedriver.exe')
driver.get('https://www.baidu.com')
web_element=driver.find_element(By.CSS_SELECTOR,'.s_ipt')
web_element.send_keys('张三')
driver.find_element(By.CSS_SELECTOR,'[id="su"]').click()
time.sleep(100)

from  selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
driver=webdriver.Chrome('./chromedriver.exe')
driver.get('http://www.baidu.com')
wait=WebDriverWait(driver,10)#传入webdriver对象，设置等待时间
#等待节点，返回值是WebElement
result_waitelement=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'.mnav.c-font-normal.c-color-t')))
#等待节点并判断某个节点是否满足条件，返回值是bool类型
result_waittext=wait.until(EC.text_to_be_present_in_element((By.CLASS_NAME, 'mnav'), '新闻'))#等待节点包含的文本值
result_waitattribute=wait.until(EC.text_to_be_present_in_element_attribute((By.CLASS_NAME,'mnav'),'href','http://'))#等待节点的属性值
print(result_waitelement.text,result_waittext,result_waitattribute)

新闻 True True

driver=webdriver.Chrome('./chromedriver.exe')
driver.get('https://login.taobao.com/member/login.jhtml')
time.sleep(40)#手动登录获取cookies
cookies=driver.get_cookies()
print(cookies,type(cookies))
with open('cookies.txt','w',encoding='utf-8') as f:
        json.dump(cookies,f)

driver=webdriver.Chrome('chromedriver.exe')
driver.get('https://www.taobao.com/')
with open(r'./cookies.txt','r',encoding='utf-8') as f:
   cookie_data=json.load(f)
   for item in cookie_data:
       print(item)
       driver.add_cookie(item)
driver.get('https://www.taobao.com/')
time.sleep(30)

driver=webdriver.Chrome('./chromedriver.exe')
driver.get('https://www.baidu.com')
driver.get_screenshot_as_file('./screenshot_png/1.png')#默认存储位置在当前同级目录下
driver.find_element(By.ID,'lg').screenshot('./screenshot_png/2.png')#截取选定的节点图片

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from  selenium.webdriver.support import expected_conditions as EC
option=ChromeOptions()
#设置无头模式
# option.add_argument('--headless')
#反屏蔽处理
#以开发者模式启动调试chrome，可以去掉提示受到自动软件控制,隐藏提示条
option.add_experimental_option('excludeSwitches',['enable-automation'])
option.add_experimental_option('useAutomationExtension',False)#去掉提示以开发者模式调用，自动拓展信息

driver = webdriver.Chrome('chromedriver.exe',options=option)
# 将webdriver属性制为空
driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument',{
    'source':'Object.defineProperty(navigator,"webdriver",{get:()=>undefined})'
})

from selenium import webdriver
from selenium.webdriver import ChromeOptions
#设置无头模式
option=ChromeOptions()
option.add_argument('--headless')
driver=webdriver.Chrome('chromedriver.exe',options=option)
driver.get('https://www.baidu.com')
driver.get_screenshot_as_png('baidu.png')

from selenium import webdriver
driver=webdriver.Chrome('./chromedriver.exe')

from  selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
driver=webdriver.Chrome('./chromedriver.exe')
driver.get('http://www.baidu.com')
wait=WebDriverWait(driver,10)#传入webdriver对象，设置等待时间
#等待节点，返回值是WebElement
result_waitelement=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'.mnav.c-font-normal.c-color-t')))

  def __init__(
        self,
        driver,
        timeout: float,
        poll_frequency: float = POLL_FREQUENCY,
        ignored_exceptions: typing.Optional[WaitExcTypes] = None,
    ):

        :Args:
         - driver - Instance of WebDriver (Ie, Firefox, Chrome or Remote)
         - timeout - Number of seconds before timing out
         - poll_frequency - sleep interval between calls
           By default, it is 0.5 second.
         - ignored_exceptions - iterable structure of exception classes ignored during 
        self._driver = driver
        self._timeout = float(timeout)
  def until(self, method, message: str = ""):
        """Calls the method provided with the driver as an argument until the \
        return value does not evaluate to ``False``.

        :param method: callable(WebDriver)
        :param message: optional message for :exc:`TimeoutException`
        :returns: the result of the last call to `method`
        :raises: :exc:`selenium.common.exceptions.TimeoutException` if timeout occurs
        """
        screen = None
        stacktrace = None

        end_time = time.monotonic() + self._timeout
        while True:
            try:
                value = method(self._driver)
                if value:
                    return value
            except self._ignored_exceptions as exc:
                screen = getattr(exc, "screen", None)
                stacktrace = getattr(exc, "stacktrace", None)
            time.sleep(self._poll)
            if time.monotonic() > end_time:
                break
        raise TimeoutException(message, screen, stacktrace)

def presence_of_element_located(locator):
    """An expectation for checking that an element is present on the DOM
    of a page. This does not necessarily mean that the element is visible.
    locator - used to find the element
    returns the WebElement once it is located
    """

    def _predicate(driver):
        return driver.find_element(*locator)

    return _predicate

def text_to_be_present_in_element(locator, text_):
    """An expectation for checking if the given text is present in the
    specified element.
    locator, text
    """

    def _predicate(driver):
        try:
            element_text = driver.find_element(*locator).text
            return text_ in element_text
        except StaleElementReferenceException:
            return False

    return _predicate