Search on blog:

Python: How to scrape with selenium

It is example code to scrape it:

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from import By
from import WebDriverWait
from import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import time

    #driver = webdriver.Chrome()
    driver = webdriver.Firefox()
    url = driver.get("")

    search_text_box = driver.find_element_by_id("search_query_top")

    time.sleep(1) # page display (and update) autocompletion when you make little longer delay 

    # --- select using arrow key ---

    # move selection on list and accept it

    # OR

    # --- select using tag `<li>` and `text()` in autocompletion --- 

    # click on first matching item on list
    #one_option = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//li[contains(text(),'Dress')]")))
    one_option = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='ac_results']//li[contains(text(),'Dress')]")))

    print(' tag:', one_option.tag_name)
    print('text:', one_option.text)
    print('bold:', one_option.find_element_by_xpath('.//strong').text)

    # OR

    # --- get all elements in autocompletion using `<li>` tag --- 

    # get many matching items and use [index] to click on some item on list
    #one_option = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//li[contains(text(),'Dress')]")))
    #all_options = driver.find_elements_by_xpath("//li[contains(text(),'Dress')]")
    #for option in all_options:
    #    print(option.tag_name, ':', option.text)

    print(' current:', driver.current_url)
    print('expected:', "")
    print('the same:', driver.current_url == "")

    assert "" == driver.current_url, "This Test case is fallied"
    #assertEqual("", self.driver.current_url, "This Test case is fallied")

except NoSuchElementException as e:
     print('NoSuchElementException:', e)
except TimeoutException as e:
     print('TimeoutException:', e)
except AssertionError as e:
     print('AssertionError:', e)
If you like it
Buy a Coffee