Python selenium.webdriver() Examples

def test_dataset_file_browser(driver: selenium.webdriver, *args, **kwargs):
    """
    Test that a file can be dragged and dropped into data in a dataset.

    Args:
        driver
    """
    username = testutils.log_in(driver)
    testutils.GuideElements(driver).remove_guide()
    dataset_elts = testutils.DatasetElements(driver)
    dataset_title = dataset_elts.create_dataset()

    logging.info(f"Navigating to data for dataset {dataset_title}")
    driver.get(f"{os.environ['GIGANTUM_HOST']}/datasets/{username}/{dataset_title}/data")
    file_browser_elts = testutils.FileBrowserElements(driver)
    file_browser_elts.file_browser_area.wait_to_appear()
    logging.info(f"Dragging and dropping file into data for dataset {dataset_title}")
    file_browser_elts.drag_drop_file_in_drop_zone()

    assert file_browser_elts.file_information.find().text == 'sample-upload.txt', \
        "Expected sample-upload.txt to be the first file in Data" 

def test_import_projects_via_zip_file(driver: selenium.webdriver, *args, **kwargs):
    """
    Test that projects can be imported via zip file and that they build successfully.

    Args:
        driver
    """
    project_zip_file = "sample-prj-93f4b4.zip"
    file_path = os.path.abspath(os.path.join(__file__, f"../../../resources/{project_zip_file}"))
    username = testutils.log_in(driver)
    testutils.GuideElements(driver).remove_guide()
    try:
        import_project_elts = testutils.ImportProjectElements(driver)
        import_project_elts.import_project_via_zip_file_drag_and_drop(file_path)
        project_control = testutils.ProjectControlElements(driver)

        assert project_control.container_status_stopped.find().is_displayed(), \
            f"Project {project_zip_file} was not imported successfully via zip file"
    finally:
        try:
            delete_local_project(username, "sample-prj")
        except Exception as e:
            logging.error(f"Failed to delete project {username}/'sample-prj': {e}") 

def testPOSTXSSDriver(url: str, cookies: Mapping[str, str], data: Mapping[str, str], driver: webdriver) -> \
        Optional[str]:
    """ If the given URL pops an alert box when accessed with the given cookies, return the contents of the alert box,
        otherwise return None """
    driver.setCookies(url, cookies)

    try:
        driver.post(url, data)

        WebDriverWait(driver, config.timeout).until(expected_conditions.alert_is_present())
        # Note that despite the name switch_to_alert also handles prompt:
        #   - http://selenium-python.readthedocs.io/navigating.html#popup-dialogs
        alert = driver.switch_to_alert()
        text = alert.text
        driver.reset()
        return text
    except (TimeoutException, URLError):
        driver.reset()
        return None 

def testGETXSSDriver(url: str, cookies: Mapping[str, str], driver: webdriver) -> Optional[str]:
    """ If the given URL pops an alert box when accessed with the given cookies, return the contents of the alert box,
        otherwise return None """
    driver.setCookies(url, cookies)

    try:
        driver.get(url)

        WebDriverWait(driver, config.timeout).until(expected_conditions.alert_is_present())
        # Note that despite the name switch_to_alert also handles prompt:
        #   - http://selenium-python.readthedocs.io/navigating.html#popup-dialogs
        alert = driver.switch_to_alert()
        text = alert.text
        driver.reset()
        return text
    except (TimeoutException, URLError):
        driver.reset()
        return None 

def __init__(self,domain_name,page_results,exts,download_dir,webdriver):
        """Everything that should be initiated with a new object goes here.

        Parameters:
        domain_name     The domain name to search under for files
        page_results    A limit on the number of search page results to request and parse
        exts            A list of file type extensions for the searches
        download_dir    The directory to use for the downloaded files
        webdriver       A Selenium webdriver object to use for the web browsing
        """
        self.exts = exts
        self.container = list()
        self.browser = webdriver
        self.domain_name = domain_name
        self.page_results = page_results
        self.download_dir = download_dir + "file_downloads/" 

def delete_dataset_cloud(driver: selenium.webdriver, dataset_title):
    """
    Delete a dataset from cloud.

    Args:
        driver
        dataset

    """
    logging.info(f"Removing dataset {dataset_title} from cloud")
    driver.find_element_by_xpath("//a[contains(text(), 'Datasets')]").click()
    driver.find_element_by_css_selector(".Datasets__nav-item--cloud").click()
    time.sleep(2)
    driver.find_element_by_css_selector(".RemoteDatasets__icon--delete").click()
    driver.find_element_by_css_selector("#deleteInput").send_keys(dataset_title)
    time.sleep(2)
    driver.find_element_by_css_selector(".ButtonLoader").click()
    time.sleep(5)
    wait = WebDriverWait(driver, 200)
    wait.until(EC.invisibility_of_element_located((By.CSS_SELECTOR, ".DeleteDataset"))) 

def create_project_without_base(driver: selenium.webdriver) -> str:
    """
    Create a project without a base.

    Args:
        driver

    Returns:
        Name of project just created
    """
    unique_project_name = testutils.unique_project_name()
    logging.info(f"Creating a new project: {unique_project_name}")
    project_elts = elements.AddProjectElements(driver)
    project_elts.create_new_button.click()
    project_elts.project_title_input.click()
    project_elts.project_title_input.find().send_keys(unique_project_name)
    project_elts.project_description_input.click()
    project_elts.project_description_input.find().send_keys(testutils.unique_project_description())
    project_elts.project_continue_button.click()
    return unique_project_name 

def prep_base(driver, base_button_check, skip_login=False):
    """Create a new project from the UI and wait until it builds successfully

    Args:
        driver: Selenium webdriver
        base_button_check: Lambda which gives identifier to element in selection menu.
        skip_login: If true, assume you are already logged in
    """
    username = None
    if skip_login is False:
        username = log_in(driver)
        elements.GuideElements(driver).remove_guide()
    else:
        time.sleep(3)
    proj_name = create_project_without_base(driver)
    time.sleep(7)
    select_project_base(driver, base_button_check())

    # assert container status is stopped
    project_elts = elements.ProjectControlElements(driver)
    # This will throw an exception on time-out
    project_elts.container_status_stopped.wait_to_appear(300)

    return ProjectPrepResponse(username=username, project_name=proj_name) 

def test_valid_custom_docker(driver: selenium.webdriver, *args, **kwargs):
    """
    Test valid custom Docker instructions.

    Args:
        driver
    """
    r = testutils.prep_py3_minimal_base(driver)
    username, project_title = r.username, r.project_name

    env_elts = testutils.EnvironmentElements(driver)
    env_elts.add_custom_docker_instructions(username, project_title,
                                            "RUN cd /tmp && "
                                            "git clone https://github.com/gigantum/confhttpproxy && "
                                            "cd /tmp/confhttpproxy && pip install -e.")
    project_control_elts = testutils.ProjectControlElements(driver)
    project_control_elts.container_status_stopped.wait_to_appear(60)

    container_status = project_control_elts.container_status_stopped.find().is_displayed()
    assert container_status, "Expected stopped container status" 

def install(self, patch_selenium=True):
        """
        Initialize the patch

        This will:
         download chromedriver if not present
         patch the downloaded chromedriver
         patch selenium package if <patch_selenium> is True (default)

        :param patch_selenium: patch selenium webdriver classes for Chrome and ChromeDriver (for current python session)
        :return:
        """
        if not os.path.exists(self.executable_path):
            self.fetch_chromedriver()
            self.patch_binary()
            self.__class__.installed = True

        if patch_selenium:
            self.patch_selenium_webdriver() 

def test_edge_build_versions(driver: selenium.webdriver, *args, **kwargs):
    """
    Test that the requests edge build version matches the selenium edge build version.

    Args:
        driver
    """
    host = f"{os.environ['GIGANTUM_HOST']}/api/ping"
    logging.info("Getting requests edge build version")
    r = requests.get(host)
    if r.status_code != 200:
        logging.error(f"Gigantum is not found at {host}")
        sys.exit(1)
    requests_edge_build_version = json.loads(r.text)
    logging.info("Getting selenium edge build version")
    driver.get(host)
    time.sleep(2)
    if driver.name == "firefox":
       driver.find_element_by_css_selector("#rawdata-tab").click()
    selenium_edge_build_version = json.loads(driver.find_element_by_css_selector("pre").text)

    assert requests_edge_build_version == selenium_edge_build_version, \
        "requests edge build version does not match selenium edge build version" 

def test_create_local_branch(driver: selenium.webdriver, *args, **kwargs):
    """
    Test the creation of a local branch.

    Args:
        driver
    """
    r = testutils.prep_py3_minimal_base(driver)
    username, project_name = r.username, r.project_name

    branch_elts = testutils.BranchElements(driver)
    branch_elts.create_local_branch("test-branch")

    logging.info("Checking that you are on the new branch and that the new branch is local only")

    assert branch_elts.upper_left_branch_name.find().text == "test-branch", \
        "Expected to be on test-branch, upper left"
    assert branch_elts.upper_left_branch_local_only.find(), "Expected test-branch to be local only, upper left"

    branch_elts.manage_branches_button.wait_to_appear().click()
    branch_elts.manage_branches_modal.wait_to_appear()

    assert branch_elts.manage_branches_branch_name.find().text == "test-branch", \
        "Expected to be on test-branch, manage branches"
    assert branch_elts.manage_branches_local_only.find(), "Expected test-branch to be local only, manage branches" 

def __init__(self, configuration):
        self.timeout = 10

        '''
        # NOTE : selenium.webdriver methods/properties
        add_cookie, application_cache, back, binary, capabilities, close,
        command_executor, create_web_element, current_url, current_window_handle, delete_all_cookies,
        delete_cookie, desired_capabilities, error_handler, execute, execute_async_script, execute_script,
        file_detector, file_detector_context, find_element, find_element_by_class_name, find_element_by_css_selector,
        find_element_by_id, find_element_by_link_text, find_element_by_name, find_element_by_partial_link_text,
        find_element_by_tag_name, find_element_by_xpath, find_elements, find_elements_by_class_name,
        find_elements_by_css_selector, find_elements_by_id, find_elements_by_link_text, find_elements_by_name,
        find_elements_by_partial_link_text, find_elements_by_tag_name, find_elements_by_xpath, firefox_profile,
        forward, get, get_cookie, get_cookies, get_log, get_screenshot_as_base64, get_screenshot_as_file,
        get_screenshot_as_png, get_window_position, get_window_size, implicitly_wait, log_types, maximize_window,
        mobile, name, options, orientation, page_source, profile, quit, refresh, save_screenshot, session_id,
        set_context, set_page_load_timeout, set_script_timeout, set_window_position, set_window_size, start_client,
        start_session, stop_client, switch_to, switch_to_active_element, switch_to_alert, switch_to_default_content,
        switch_to_frame, switch_to_window, title, w3c, window_handles
        ''' 

def go_to_specific_results_page(driver, delay, results_page):
    """
    go to a specific results page in case of an error, can restart 
    the webdriver where the error occurred.
    """
    if results_page < 2:
        return
    current_page = 1
    for i in range(results_page):
        current_page += 1
        time.sleep(5)
        try:
            next_results_page(driver, delay)
            print("\n**************************************************")
            print("\n\n\nNavigating to results page {}" \
                  "\n\n\n".format(current_page))
        except ValueError:
            print("**************************************************")
            print("\n\n\n\n\nSearch results exhausted\n\n\n\n\n") 

def test_base_python3_min(driver: selenium.webdriver, *args, **kwargs):
    """
    Test the creation a project with a python 3 minimal base.

    Args:
        driver
    """
    b = lambda : \
        testutils.elements.AddProjectBaseElements(driver).py3_minimal_base_button.find()
    testutils.prep_base(driver, b) 

def get_html(self):
        """ Downloads HTML content of page given the page_url"""

        if self.use_ghost:
            self.url = urljoin("http://", self.url)
            import selenium
            import selenium.webdriver
            driver = selenium.webdriver.PhantomJS(
                service_log_path=os.path.devnull)
            driver.get(self.url)
            page_html = driver.page_source
            page_url = driver.current_url
            driver.quit()
        else:
            if self.proxy_url:
                print("Using proxy: " + self.proxy_url + "\n")
            try:
                page = requests.get(self.url, proxies=self.proxies)
                if page.status_code != 200:
                    raise PageLoadError(page.status_code)
            except requests.exceptions.MissingSchema:
                self.url = "http://" + self.url
                page = requests.get(self.url, proxies=self.proxies)
                if page.status_code != 200:
                    raise PageLoadError(page.status_code)
            except requests.exceptions.ConnectionError:
                raise PageLoadError(None)
            try:
                page_html = page.text
                page_url = page.url
            except UnboundLocalError:
                raise PageLoadError(None)

        self.page_html = page_html
        self.page_url = page_url
        return (self.page_html, self.page_url) 

def __init__(self,webdriver):
        """Everything that should be initiated with a new object goes here.

        Parameters:
        webdriver   A Selenium webdriver object to use for automated web browsing
        """
        self.browser = webdriver
        self.browser.set_page_load_timeout(self.browser_timeout) 

def process(self, profile, state, vertex):
        if 'type' not in vertex:
            return { 'error' : "No vertex type defined!" }

        properties = dict()

        if vertex['type'] in [ 'url', 'domain' ]:

            url = vertex['id'].lower()
            if not url.startswith('http://') and not url.startswith('https://'):
                url = "http://" + url

            driver = selenium.webdriver.Firefox()
            driver.delete_all_cookies()
            driver.set_page_load_timeout(self.timeout)
            
            driver.set_window_size(800, 600)
            driver.set_window_position(0, 0)

            try:
                driver.get(url)
                properties['page_source'] = driver.page_source
                properties['title'] = driver.title
                properties['cookies'] = driver.get_cookies()
                properties['screenshot'] = driver.get_screenshot_as_base64()
            except:
                pass

            driver.close()

        return { "properties": properties, "neighbors" : [] } 

def _create_browser_instance(cls):
        driver_name = cls.get_driver_name()
        kwargs = cls.get_webdriver_options()
        driver = getattr(webdriver, driver_name)(**kwargs)
        driver.set_page_load_timeout(cls.get_page_load_timeout())
        return driver 

def __init__(self, driver: selenium.webdriver):
        self.driver = driver 

def __init__(self, cookie=None, scraperInstance=None, driver=selenium.webdriver.Chrome, driver_options={}, scroll_pause=0.1, scroll_increment=300, timeout=10):
        if type(self) is Scraper:
            raise Exception(
                'Scraper is an abstract class and cannot be instantiated directly')

        if scraperInstance:
            self.was_passed_instance = True
            self.driver = scraperInstance.driver
            self.scroll_increment = scraperInstance.scroll_increment
            self.timeout = scraperInstance.timeout
            self.scroll_pause = scraperInstance.scroll_pause
            return

        self.was_passed_instance = False
        self.driver = driver(**driver_options)
        self.scroll_pause = scroll_pause
        self.scroll_increment = scroll_increment
        self.timeout = timeout
        self.driver.get('http://www.linkedin.com')
        self.driver.set_window_size(1920, 1080)

        if 'LI_EMAIL' in environ and 'LI_PASS' in environ:
            self.login(environ['LI_EMAIL'], environ['LI_PASS'])
        else:
            if not cookie and 'LI_AT' not in environ:
                raise ValueError(
                    'Must either define LI_AT environment variable, or pass a cookie string to the Scraper')
            elif not cookie:
                cookie = environ['LI_AT']
            self.driver.add_cookie({
                'name': 'li_at',
                'value': cookie,
                'domain': '.linkedin.com'
            }) 

def __new__(cls, *args, **kwargs):

        if not ChromeDriverManager.installed:
            ChromeDriverManager(*args, **kwargs).install()
        if not ChromeDriverManager.selenium_patched:
            ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()
        if not kwargs.get('executable_path'):
            kwargs['executable_path'] = './{}'.format(ChromeDriverManager(*args, **kwargs).executable_path)
        if not kwargs.get('options'):
            kwargs['options'] = ChromeOptions() 
        instance = object.__new__(_Chrome)
        instance.__init__(*args, **kwargs)
        instance.execute_cdp_cmd(
            "Page.addScriptToEvaluateOnNewDocument",
            {
                "source": """
        Object.defineProperty(window, 'navigator', {
            value: new Proxy(navigator, {
              has: (target, key) => (key === 'webdriver' ? false : key in target),
              get: (target, key) =>
                key === 'webdriver'
                  ? undefined
                  : typeof target[key] === 'function'
                  ? target[key].bind(target)
                  : target[key]
            })
        })
                  """
            },
        )
        original_user_agent_string = instance.execute_script(
            "return navigator.userAgent"
        )
        instance.execute_cdp_cmd(
            "Network.setUserAgentOverride",
            {
                "userAgent": original_user_agent_string.replace("Headless", ""),
            },
        )
        logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})")
        return instance 

def create_driver(cls):
        assert cls.DRIVER_PATH, "必须设置 DRIVER_PATH"

        driver_type = cls.DRIVER_TYPE or cls.DEFAULT_DRIVER_TYPE

        assert driver_type in cls.SUPPORT_DRIVER_TYPE, "DRIVER_TYPE 必须为: {}"\
            .format(",".join(cls.SUPPORT_DRIVER_TYPE))

        from selenium import webdriver
        driver_cls = getattr(webdriver, driver_type)
        return driver_cls(cls.DRIVER_PATH) 

def test_base_rstudio(driver: selenium.webdriver, *args, **kwargs):
    """
    Test the creation of a project with a RStudio base.

    Args:
        driver
    """
    b = lambda: \
        testutils.elements.AddProjectBaseElements(driver).rstudio_base_button.find()
    testutils.prep_base(driver, b) 

def test_base_python3_ds(driver: selenium.webdriver, *args, **kwargs):
    """
    Test the creation of a project with a python 3 data science base.

    Args:
        driver
    """
    b = lambda : \
        testutils.elements.AddProjectBaseElements(driver).py3_data_science_base_button.find()
    testutils.prep_base(driver, b) 

def test_delete_file_project_file_browser(driver: selenium.webdriver, *args, **kwargs):
    """
    Test that a file can be dragged and dropped into input data and deleted properly.

    Args:
        driver
    """
    r = testutils.prep_py3_minimal_base(driver)
    username, project_title = r.username, r.project_name

    file_browser_elts = testutils.FileBrowserElements(driver)
    logging.info(f"Navigating to input data for project: {project_title}")
    driver.get(f"{os.environ['GIGANTUM_HOST']}/projects/{username}/{project_title}/inputData")
    file_browser_elts.file_browser_area.wait_to_appear()
    logging.info(f"Dragging and dropping file into input data for project {project_title}")
    file_browser_elts.drag_drop_file_in_drop_zone()

    assert file_browser_elts.file_information.find().text == "sample-upload.txt", \
        "Expected sample-upload.txt to be the first file in input data"

    file_browser_elts.trash_can_button().click()
    file_browser_elts.confirm_delete_file_button().click()
    driver.refresh()
    file_browser_elts.file_browser_area.wait_to_appear()

    # Expect to be empty
    file_contents = file_browser_elts.input_file_browser_contents_list
    assert len(file_contents) == 1, "Expected file browser to be empty but more than 1 file/directory exists"
    assert file_contents[0] == 'untracked', "Expected file browser to be empty, but still have untracked folder" 

def test_base_python2_min(driver: selenium.webdriver, *args, **kwargs):
    """
    Test the creation of a project with a python 2 minimal base.

    Args:
        driver
    """
    b = lambda : \
        testutils.elements.AddProjectBaseElements(driver).py2_minimal_base_button.find()
    testutils.prep_base(driver, b) 

def test_use_mine_merge_conflict_project(driver: selenium.webdriver, *args, **kwargs):
    """
    Test that a merge conflict is properly resolved when the user decides to "use mine."

    Args:
        driver
    """
    # Prepare merge conflict
    username, project_title, collaborator = prep_merge_conflict(driver)

    # Owner resolves the merge conflict with 'Use Mine'
    cloud_project_elts = testutils.CloudProjectElements(driver)
    cloud_project_elts.merge_conflict_modal.wait_to_appear(20)
    time.sleep(2)
    cloud_project_elts.merge_conflict_use_mine_button.click()
    cloud_project_elts.merge_conflict_modal.wait_to_disappear(nsec=20)
    project_control_elts = testutils.ProjectControlElements(driver)
    waiting_start = time.time()
    while "Sync complete" not in project_control_elts.footer_notification_message.find().text:
        time.sleep(0.5)
        if time.time() - waiting_start > 35:
            raise ValueError(f'Timed out waiting for sync to complete')

    # Check that merge conflict resolves to "use mine"
    file_path = os.path.join(os.environ['GIGANTUM_HOME'], username, username, 'labbooks',
                             project_title, 'input', 'sample-upload.txt')
    with open(file_path, "r") as resolve_merge_conflict_file:
        resolve_merge_conflict_file = resolve_merge_conflict_file.read()

    assert resolve_merge_conflict_file == "owner", \
        f"Merge did not resolve to 'use mine,' expected to see 'owner' in file, " \
        f"but instead got {resolve_merge_conflict_file}" 

def patch_selenium_webdriver(self_):
        """
        Patches selenium package Chrome, ChromeOptions classes for current session

        :return:
        """
        import selenium.webdriver.chrome.service
        import selenium.webdriver
        selenium.webdriver.Chrome = Chrome
        selenium.webdriver.ChromeOptions = ChromeOptions
        logger.warning(
            "Selenium patched. Safe to import Chrome / ChromeOptions"
        )
        self_.__class__.selenium_patched = True 

def test_packages_requirements_file(driver: selenium.webdriver, *args, **kwargs):
    """
    Test that packages can be installed successfully using a requirements.txt file.

    Args:
        driver
    """
    r = testutils.prep_py3_minimal_base(driver)
    username, project_title = r.username, r.project_name

    # Open add packages modal
    env_elts = testutils.EnvironmentElements(driver)
    env_elts.open_add_packages_modal(username, project_title)
    env_elts.add_requirements_file_button.wait_to_appear().click()

    # Drag and drop requirements.txt
    env_elts.drag_drop_requirements_file_in_drop_zone()
    env_elts.install_queued_packages()
    project_control_elts = testutils.ProjectControlElements(driver)

    assert env_elts.package_info_table_version_one.find().text == "0.19", \
        "gigantum==0.19 package was not installed successfully since it is not visible in the UI"

    # Check that pip package was installed inside the container
    project_control_elts.start_stop_container_button.wait_to_appear().click()
    project_control_elts.container_status_running.wait_to_appear(30)
    container_pip_packages = env_elts.obtain_container_pip_packages(username, project_title)

    assert "gigantum==0.19" in container_pip_packages, \
        "gigantum==0.19 package was not installed successfully since it is not installed inside the container"