Python selenium.webdriver.common.by.By.CSS_SELECTOR Examples

def index_page(page):
    try:
        print('正在爬取第： %s 页' % page)
        wait.until(
            EC.presence_of_element_located((By.ID, "dt_1")))
        # 判断是否是第1页，如果大于1就输入跳转，否则等待加载完成。
        if page > 1:
            # 确定页数输入框
            input = wait.until(EC.presence_of_element_located(
                (By.XPATH, '//*[@id="PageContgopage"]')))
            input.click()
            input.clear()
            input.send_keys(page)
            submit = wait.until(EC.element_to_be_clickable(
                (By.CSS_SELECTOR, '#PageCont > a.btn_link')))
            submit.click()
            time.sleep(2)
        # 确认成功跳转到输入框中的指定页
        wait.until(EC.text_to_be_present_in_element(
            (By.CSS_SELECTOR, '#PageCont > span.at'), str(page)))
    except Exception:
        return None 

def _find_web_element(self):
        """Find WebElement using element locator and save it in _web_element attribute"""
        if not self._web_element or not self.config.getboolean_optional('Driver', 'save_web_element'):
            # If the element is encapsulated we use the shadowroot tag in yaml (eg. Shadowroot: root_element_name)
            if self.shadowroot:
                if self.locator[0] != By.CSS_SELECTOR:
                    raise Exception('Locator type should be CSS_SELECTOR using shadowroot but found: '
                                    '%s' % self.locator[0])
                # querySelector only support CSS SELECTOR locator
                self._web_element = self.driver.execute_script('return document.querySelector("%s").shadowRoot.'
                                                               'querySelector("%s")' % (self.shadowroot,
                                                                                        self.locator[1]))
            else:
                # Element will be finded from parent element or from driver
                base = self.utils.get_web_element(self.parent) if self.parent else self.driver
                # Find elements and get the correct index or find a single element
                self._web_element = base.find_elements(*self.locator)[self.order] if self.order else base.find_element(
                    *self.locator) 

def get_web_value(self, element):
        """
		Gets the informations of field based in the ID
        """

        if element.tag_name == "div":
            element_children = element.find_element(By.CSS_SELECTOR, "div > * ")
            if element_children is not None:
                element = element_children

        if element.tag_name == "label":
            web_value = element.get_attribute("text")
        elif element.tag_name == "select":
            current_select = int(element.get_attribute('value'))
            selected_element = element.find_elements(By.CSS_SELECTOR, "option")[current_select]
            web_value = selected_element.text
        else:
            web_value = element.get_attribute("value")

        return web_value 

def get_first_connections(self):
        try:
            see_connections_link = WebDriverWait(self.driver, self.timeout).until(EC.presence_of_element_located((
                By.CSS_SELECTOR,
                '.pv-top-card-v2-section__link--connections'
            )))
        except TimeoutException as e:
            print("""Took too long to load connections link. This usually indicates you were trying to
scrape the connections of someone you aren't connected to.""")
            return []

        see_connections_link.click()
        try:
            self.configure_connection_type()
        except TimeoutException:
            return []
        all_conns = [] 

def open(self):
        try:
            self.worker.first(By.CSS_SELECTOR, '#TABLE_DATA_fileList')
        except NoSuchElementException:
            self.print_letter_status('Opening table...', '')

            self.worker.get('/mng/action/home.do?mode=ajax')
            # Open Alma configuration
            self.worker.wait_for_and_click(
                By.CSS_SELECTOR, '#ALMA_MENU_TOP_NAV_configuration')
            # text() = "General"
            self.worker.click(By.XPATH, '//*[@href="#CONF_MENU6"]')
            self.worker.click(By.XPATH, '//*[text() = "Customize Letters"]')
            self.worker.wait_for(By.CSS_SELECTOR, '#TABLE_DATA_fileList')

        return self 

def read(self):

        # Identify the indices of the column headers we're interested in
        elems = self.worker.all(By.CSS_SELECTOR, '#TABLE_DATA_fileList tr > th')
        column_headers = [el.get_attribute('id') for el in elems]
        filename_col = column_headers.index('SELENIUM_ID_fileList_HEADER_cfgFilefilename') + 1
        updatedate_col = column_headers.index('SELENIUM_ID_fileList_HEADER_updateDate') + 1

        # Read the filename column
        elems = self.worker.all(By.CSS_SELECTOR,
                                '#TABLE_DATA_fileList tr > td:nth-child(%d) > a' % filename_col)
        self.filenames = [el.text.replace('../', '') for el in elems]

        # Read the modification date column
        elems = self.worker.all(By.CSS_SELECTOR,
                                '#TABLE_DATA_fileList tr > td:nth-child(%d) > span' % updatedate_col)
        self.update_dates = [el.text for el in elems]

        # return [{x[0]:2 {'modified': x[1], 'index': n}} for n, x in enumerate(zip(filenames, update_dates))] 

def load_initial(self, company):
        url = 'https://www.linkedin.com/company/{}'.format(company)

        self.driver.get(url)
        try:
            myElem = WebDriverWait(self.driver, self.timeout).until(AnyEC(
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, '.organization-outlet')),
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, '.error-container'))
            ))
        except TimeoutException as e:
            raise ValueError(
                """Took too long to load company.  Common problems/solutions:
                1. Invalid LI_AT value: ensure that yours is correct (they
                   update frequently)
                2. Slow Internet: increase the timeout parameter in the Scraper constructor""")
        try:
            self.driver.find_element_by_css_selector('.organization-outlet')
        except:
            raise ValueError(
                'Company Unavailable: Company link does not match any companies on LinkedIn') 

def regular_login(url, course_folder, driver):
    driver.get("https://www.lynda.com/signin/")          # launch lynda.com/signin

    # enter username
    email = driver.find_element_by_css_selector("#email-address")
    email.clear()
    email.send_keys(read.username)
    driver.find_element_by_css_selector('#username-submit').click()
    print('\nusername successfully entered ....')
    # wait for password field to appear
    WebDriverWait(driver, 15).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "#password-input")))
    # enter password
    passwrd = driver.find_element_by_css_selector('#password-input')
    passwrd.send_keys(read.password)
    driver.find_element_by_css_selector('#password-submit').click()
    print('password successfully entered ....') 

def send_anon_message(self, phone, text):
        payload = urlencode({"phone": phone, "text": text, "source": "", "data": ""})
        self.browser.get("https://api.whatsapp.com/send?"+payload)
        try:
            Alert(self.browser).accept()
        except:
            print("No alert Found")
        WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, "#action-button")))
        send_message = self.browser.find_element_by_css_selector("#action-button")
        send_message.click()
        confirm = WebDriverWait(self.browser, self.timeout+5).until(EC.presence_of_element_located(
                (By.XPATH, "/html/body/div/div/div/div[4]/div/footer/div[1]/div[2]/div/div[2]")))
        confirm.clear()
        confirm.send_keys(text+Keys.ENTER)

    # Check if the message is present in an user chat 

def __init__(self, wait, screenshot=None, session=None):
        chrome_options = Options()
        if session:
            chrome_options.add_argument("--user-data-dir={}".format(session))
            self.browser = webdriver.Chrome(options=chrome_options)  # we are using chrome as our webbrowser
        else:
            self.browser = webdriver.Chrome()
        self.browser.get("https://web.whatsapp.com/")
        # emoji.json is a json file which contains all the emojis
        with open("emoji.json") as emojies:
            self.emoji = json.load(emojies)  # This will load the emojies present in the json file into the dict
        WebDriverWait(self.browser,wait).until(EC.presence_of_element_located(
            (By.CSS_SELECTOR, '._3FRCZ')))
        if screenshot is not None:
            self.browser.save_screenshot(screenshot)  # This will save the screenshot to the specified file location

    # This method is used to send the message to the individual person or a group
    # will return true if the message has been sent, false else 

def get_invite_link_for_group(self, groupname):
        search = self.browser.find_element_by_css_selector("._3FRCZ")
        search.send_keys(groupname+Keys.ENTER)
        self.browser.find_element_by_css_selector("#main > header > div._5SiUq > div._16vzP > div > span").click()
        try:
            #time.sleep(3)
            WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located(
                    (By.CSS_SELECTOR, "#app > div > div > div.MZIyP > div._3q4NP._2yeJ5 > span > div > span > div > div > div > div:nth-child(5) > div:nth-child(3) > div._3j7s9 > div > div")))
            invite_link = self.browser.find_element_by_css_selector("#app > div > div > div.MZIyP > div._3q4NP._2yeJ5 > span > div > span > div > div > div > div:nth-child(5) > div:nth-child(3) > div._3j7s9 > div > div")
            invite_link.click()
            WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located(
                    (By.ID, "group-invite-link-anchor")))
            link = self.browser.find_element_by_id("group-invite-link-anchor")
            return link.text
        except:
            print("Cannot get the link")

    # This method is used to exit a group 

def log_in_user(self):
        driver = self.webdriver
        self.log('You need to login to access this profile.',
                 'Redirecting you to the login page in the browser.',
                 level=logging.WARN)
        driver.get(self.get_url('accounts/login/'))

        # Wait until user has been successfully logged in and redirceted
        # to his/her feed.
        WebDriverWait(driver, 60).until(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, '.-cx-PRIVATE-FeedPage__feed'),
            )
        )

        self.log('User successfully logged in.', level=logging.INFO)
        self.set_num_posts()  # Have to set this again
        driver.get(self.profile_url) 

def next_page(page_num):
    try:
        print('获取下一页数据')
        next_btn = WAIT.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#server-search-app > div.contain > div.body-contain > div > div.page-wrap > div > ul > li.page-item.next > button')))
        next_btn.click()
        WAIT.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, '#server-search-app > div.contain > div.body-contain > div > div.page-wrap > div > ul > li.page-item.active > button'),str(page_num)))
        get_source()
    except TimeoutException:
        browser.refresh()
        return next_page(page_num) 

def test_TC001_py3_doc_button(self):
        self.driver.get("https://www.python.org")
        element = WebDriverWait(self.driver, 10).until(EC.visibility_of_element_located((By.ID, "documentation")))
        ActionChains(self.driver).move_to_element(element).perform()
        locator = "#documentation > ul > li.tier-2.super-navigation > p.download-buttons > a:nth-child(1)"
        py3docbutton = WebDriverWait(self.driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, locator)))
        assert py3docbutton.text == 'Python 3.x Docs'
        py3docbutton.click()
        assert self.driver.current_url == 'https://docs.python.org/3/' 

def extract_transform_load(driver, delay, selector, date, 
                           keyword, location, filename):
    """
    using the next job posting selector on a given results page, wait for
    the link to become clickable, then navigate to it. Wait for the job 
    posting page to load, then scrape the page and write the data to file.
    Finally, go back to the search results page
    """
    # wait for the job post to load then navigate to it
    try:
        wait_for_clickable_element(driver, delay, selector)
        robust_click(driver, delay, selector)
    except Exception as e:
        print("error navigating to job post page")
        print(e)
    try:
        # wait for the premium applicant insights to load
        # wait_for_clickable_element(driver, delay, "div.premium-insights")
        WebDriverWait(driver, delay).until(
            EC.presence_of_element_located(
                (By.CSS_SELECTOR, "div.premium-insights")
                )
            )
    except Exception as e:
        print(e)

    try:
        # scrape page and prepare to write the data to file
        data = scrape_page(driver, keyword=keyword, location=location, dt=date)
    except Exception as e:
        print("\nSearch results may have been exhausted. An error was " \
                "encountered while attempting to scrape page data")
        print(e)
    else:
        # write data to file
        write_line_to_file(filename, data)
    finally:
        if not ("Search | LinkedIn" in driver.title):
            driver.execute_script("window.history.go(-1)") 

def wait_for_clickable_element_css(driver, delay, selector):
    """use WebDriverWait to wait for an element to become clickable"""
    obj = WebDriverWait(driver, delay).until(
            EC.element_to_be_clickable(
                (By.CSS_SELECTOR, selector)
            )
        )
    return obj 

def tixcraft_ticket_main(url, is_verifyCode_editing):
    form_select = None
    try:
        #form_select = driver.find_element(By.TAG_NAME, 'select')
        form_select = driver.find_element(By.CSS_SELECTOR, '.mobile-select')

        if form_select is not None:
            try:
                #print("get select ticket value:" + Select(form_select).first_selected_option.text)
                if Select(form_select).first_selected_option.text=="0":
                    is_verifyCode_editing = False
            except Exception as exc:
                print("query selected option fail")
                print(exc)
                pass

            if is_verifyCode_editing == False:
                ticket_number_auto_fill(url, form_select)

                # start to input verify code.
                try:
                    #driver.execute_script("$('#TicketForm_verifyCode').focus();")
                    driver.execute_script("document.getElementById(\"TicketForm_verifyCode\").focus();")

                    is_verifyCode_editing = True
                    print("goto is_verifyCode_editing== True")
                except Exception as exc:
                    print(exc)
                    pass
            else:
                #print("is_verifyCode_editing")
                # do nothing here.
                pass

    except Exception as exc:
        print("find select fail")
        pass

    return is_verifyCode_editing 

def click_on_more_button(self):
        more = self.get_page_header(
                "//div[contains(@class, 'btn-more')]")
        self.click_through(
            more, (By.CSS_SELECTOR, "div.dropdown.pull-right.btn-more.open")) 

def test_TC003_verify_upcoming_events_section_present(self):
        self.driver.get("https://www.python.org/about/")
        css_locator = '#content > div > section > div.list-widgets.row > div.medium-widget.event-widget.last > div > h2'
        elemnt = WebDriverWait(self.driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, css_locator)))
        assert elemnt.text == 'Upcoming Events' 

def get_status(self, name):
        search = self.browser.find_element_by_css_selector("._3FRCZ")
        search.send_keys(name+Keys.ENTER)  # we will send the name to the input key box
        try:
            group_xpath = "/html/body/div/div/div/div[3]/header/div[1]/div/span/img"
            click_menu = WebDriverWait(self.browser,self.timeout).until(EC.presence_of_element_located(
                (By.XPATH, group_xpath)))
            click_menu.click()
        except TimeoutException:
            raise TimeoutError("Your request has been timed out! Try overriding timeout!")
        except NoSuchElementException:
            return "None"
        except Exception:
            return "None"
        try:
            status_css_selector = ".drawer-section-body > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > span:nth-child(1) > span:nth-child(1)"   # This is the css selector for status
            WebDriverWait(self.browser,self.timeout).until(EC.presence_of_element_located(
                (By.CSS_SELECTOR, status_css_selector)))
            status = self.browser.find_element_by_css_selector(status_css_selector).text
            # We will try for 100 times to get the status
            for i in range(10):
                if len(status) > 0:
                    return status
                else:
                    time.sleep(1) # we need some delay
            return "None"
        except TimeoutException:
            raise TimeoutError("Your request has been timed out! Try overriding timeout!")
        except NoSuchElementException:
            return "None"
        except Exception:
            return "None"

    # to get the last seen of the person 

def get_source():
    WAIT.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#server-search-app > div.contain > div.body-contain > div > div.result-wrap.clearfix')))
    html = browser.page_source
    soup = BeautifulSoup(html,'lxml')
    save_to_excel(soup) 

def load_profile_page(self, url='', user=None):
        """Load profile page and all async content

        Params:
            - url {str}: url of the profile to be loaded
        Raises:
            ValueError: If link doesn't match a typical profile url
        """
        if user:
            url = 'http://www.linkedin.com/in/' + user
        if 'com/in/' not in url:
            raise ValueError("Url must look like ...linkedin.com/in/NAME")
        self.current_profile = url.split(r'com/in/')[1]
        self.driver.get(url)
        # Wait for page to load dynamically via javascript
        try:
            myElem = WebDriverWait(self.driver, self.timeout).until(AnyEC(
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, '.pv-top-card-section')),
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, '.profile-unavailable'))
            ))
        except TimeoutException as e:
            raise Exception(
                """Took too long to load profile.  Common problems/solutions:
                1. Invalid LI_AT value: ensure that yours is correct (they
                   update frequently)
                2. Slow Internet: increase the timeout parameter in the Scraper constructor""")

        # Check if we got the 'profile unavailable' page
        try:
            self.driver.find_element_by_css_selector('.pv-top-card-section')
        except:
            raise ValueError(
                'Profile Unavailable: Profile link does not match any current Linkedin Profiles') 

def wait_for_el(self, selector):
        return self.wait(EC.presence_of_element_located((
            By.CSS_SELECTOR, selector
        ))) 

def load_profile_page(self, url='', user=None):
        """Load profile page and all async content

        Params:
            - url {str}: url of the profile to be loaded
        Raises:
            ValueError: If link doesn't match a typical profile url
        """
        if user:
            url = 'http://www.linkedin.com/in/' + user
        if 'com/in/' not in url and 'sales/gmail/profile/proxy/' not in url:
            raise ValueError(
                "Url must look like... .com/in/NAME or... '.com/sales/gmail/profile/proxy/EMAIL")
        self.driver.get(url)
        # Wait for page to load dynamically via javascript
        try:
            myElem = WebDriverWait(self.driver, self.timeout).until(AnyEC(
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, self.MAIN_SELECTOR)),
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, self.ERROR_SELECTOR))
            ))
        except TimeoutException as e:
            raise ValueError(
                """Took too long to load profile.  Common problems/solutions:
                1. Invalid LI_AT value: ensure that yours is correct (they
                   update frequently)
                2. Slow Internet: increase the time out parameter in the Scraper
                   constructor
                3. Invalid e-mail address (or user does not allow e-mail scrapes) on scrape_by_email call
                """)

        # Check if we got the 'profile unavailable' page
        try:
            self.driver.find_element_by_css_selector(self.MAIN_SELECTOR)
        except:
            raise ValueError(
                'Profile Unavailable: Profile link does not match any current Linkedin Profiles')
        # Scroll to the bottom of the page incrementally to load any lazy-loaded content
        self.scroll_to_bottom() 

def wait_for_presence_css(self, selector):
        WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, selector))) 

def test_submit(self):
        driver = self.driver

        driver.get(self.base_url + "/course/test/helloworld")
        print("-----------------Trying to find textarea-----------------")
        for i in range(5):
            try:
                if self.is_element_present(By.CSS_SELECTOR, "div.CodeMirror textarea"):
                    break
            except:
                pass
            time.sleep(1)
        else:
            self.fail("time out")
        print("-----------------Calling script-----------------")
        driver.execute_script("""codeEditors[0].setValue('print "Hello World!"')""")
        time.sleep(2)
        self.assertEqual("""print "Hello World!\"""", driver.find_element_by_css_selector('textarea.code-editor').get_attribute('value'))
        print("-----------------Clicking-----------------")
        driver.find_element_by_id("task-submit").click()
        print("-----------------Trying to find task alert-----------------")
        for i in range(5):
            try:
                if self.is_element_present(By.XPATH, "//div[@id='task_alert']/div/p"):
                    break
            except:
                pass
            time.sleep(1)
        else:
            self.fail("time out")
        print("-----------------Done-----------------")
        print(driver.find_element_by_xpath("//div[@id='task_alert']/div").text)
        self.assertEqual("You solved this difficult task!", driver.find_element_by_xpath("//div[@id='task_alert']/div/p").text)
        self.assertEqual("100.0", driver.find_element_by_id("task_grade").text)
        self.assertEqual("Succeeded", driver.find_element_by_id("task_status").text)
        self.assertEqual("1", driver.find_element_by_id("task_tries").text) 

def get_starred_messages(self, delay=10):
        starred_messages = []
        self.browser.find_element_by_css_selector("div.rAUz7:nth-child(3) > div:nth-child(1) > span:nth-child(1)").click()
        chains = ActionChains(self.browser)
        time.sleep(2)
        for i in range(4):
            chains.send_keys(Keys.ARROW_DOWN)
        chains.send_keys(Keys.ENTER)
        chains.perform()
        time.sleep(delay)
        messages = self.browser.find_elements_by_class_name("MS-DH")
        for message in messages:
            try:
                message_html = message.get_attribute("innerHTML")
                soup = BeautifulSoup(message_html, "html.parser")
                _from = soup.find("span", class_="_1qUQi")["title"]
                to = soup.find("div", class_="copyable-text")["data-pre-plain-text"]
                message_text = soup.find("span", class_="selectable-text invisible-space copyable-text").text
                message.click()
                selector = self.browser.find_element_by_css_selector("#main > header > div._5SiUq > div._16vzP > div > span")
                title = selector.text
                selector.click()
                time.sleep(2)
                WebDriverWait(self.browser, 5).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div._14oqx:nth-child(3) > div:nth-child(1) > div:nth-child(1) > span:nth-child(1) > span:nth-child(1)")))
                phone = self.browser.find_element_by_css_selector("div._14oqx:nth-child(3) > div:nth-child(1) > div:nth-child(1) > span:nth-child(1) > span:nth-child(1)").text
                if title in _from:
                    _from = _from.replace(title, phone)
                else:
                    to = to.replace(title, phone)
                starred_messages.append([_from, to, message_text])
            except Exception as e:
                print("Handled: ", e)
        return starred_messages

    # Getting usernames which has unread messages 

def is_message_present(self, username, message):
        search = self.browser.find_element_by_css_selector("._3FRCZ")
        search.send_keys(username+Keys.ENTER)
        search_bar = WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, "._1i0-u > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > span:nth-child(1)")))
        search_bar.click()
        message_search = self.browser.find_element_by_css_selector("._1iopp > div:nth-child(1) > label:nth-child(4) > input:nth-child(1)")
        message_search.clear()
        message_search.send_keys(message+Keys.ENTER)
        try:
            WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div/div/div[2]/div[3]/span/div/div/div[2]/div/div/div/div/div[1]/div/div/div/div[2]/div[1]/span/span/span")))
            return True
        except TimeoutException:
            return False

    # Get all starred messages 

def exit_group(self, group_name):
        search = self.browser.find_element_by_css_selector("._3FRCZ")
        search.send_keys(group_name+Keys.ENTER)
        self.browser.find_element_by_css_selector("._2zCDG > span:nth-child(1)").click()
        WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div._1CRb5:nth-child(6) > div:nth-child(1) > div:nth-child(2) > div:nth-child(1) > span:nth-child(1)")))
        time.sleep(3)
        _exit = self.browser.find_element_by_css_selector("div._1CRb5:nth-child(6) > div:nth-child(1) > div:nth-child(2) > div:nth-child(1) > span:nth-child(1)")
        _exit.click()
        WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div._1WZqU:nth-child(2)")))
        confirm_exit = self.browser.find_element_by_css_selector("div._1WZqU:nth-child(2)")
        confirm_exit.click()

    # Send Anonymous message 

def goto_main(self):
        try:
            self.browser.refresh()
            Alert(self.browser).accept()
        except Exception as e:
            print(e)
        WebDriverWait(self.browser, self.timeout).until(EC.presence_of_element_located(
            (By.CSS_SELECTOR, '._3FRCZ')))



    # get the status message of a person
    # TimeOut is approximately set to 10 seconds