I have written some code in python in combination with selenium to parse all the names from facebook friend list. It was hard to manage the pop up notification and the process of scrolling to the end of that page. However, my scraper can do that successfully. I tried to do the whole thing very carefully. There are always rooms for improvement, though. here is the working code:from selenium import webdriverfrom selenium.webdriver.chrome.options import Optionsimport timechrome_options = webdriver.ChromeOptions()prefs = {profile.default_content_setting_values.notifications : 2}chrome_options.add_experimental_option(prefs,prefs)def Facebook_Crawler(driver):    driver.get('https://www.facebook.com/login.php?login_attempt=1&lwv=111')    driver.find_element_by_xpath('//*[@id=email]').send_keys('SomeEmail')    driver.find_element_by_xpath('//*[@id=pass]').send_keys('Password')    driver.find_element_by_xpath('//*[@id=loginbutton]').click()    time.sleep(5)    driver.find_element_by_xpath(//a[@id='findFriendsNav']).click()    time.sleep(5)    num_Of_links = -1    num = 0    while num != num_Of_links:        num_Of_links = num        driver.execute_script(window.scrollTo(0, document.body.scrollHeight);)        time.sleep(8)        items = [post.text for post in driver.find_elements_by_xpath(//div[contains(@class,'friendBrowserNameTitle')]/a)]        num = len(items)    for item in items:        print(item)if __name__ == '__main__':    driver = webdriver.Chrome(chrome_options = chrome_options)    try:        Facebook_Crawler(driver)    finally:        driver.quit()
Parser for Facebook friend list
python;python 3.x;web scraping;selenium
I would improve the following things:implement a crawler class that will store the driver instance as a variable and will also have a better separation of concerns - a separate method to log in, a separate method to get friends etcstop using time.sleep() for waits with selenium - hardcoded time delays are seriously unreliable - use Explicit Waits insteadstop using XPaths - they are the least readable and the least reliable - use the fastest by id or the more concise by css selector locatorsImproved code:from selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.wait import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECclass FacebookCrawler:    LOGIN_URL = 'https://www.facebook.com/login.php?login_attempt=1&lwv=111'    def __init__(self, login, password):        chrome_options = webdriver.ChromeOptions()        prefs = {profile.default_content_setting_values.notifications: 2}        chrome_options.add_experimental_option(prefs, prefs)        self.driver = webdriver.Chrome(chrome_options=chrome_options)        self.wait = WebDriverWait(self.driver, 10)        self.login(login, password)    def login(self, login, password):        self.driver.get(self.LOGIN_URL)        # wait for the login page to load        self.wait.until(EC.visibility_of_element_located((By.ID, email)))        self.driver.find_element_by_id('email').send_keys(login)        self.driver.find_element_by_id('pass').send_keys(password)        self.driver.find_element_by_id('loginbutton').click()        # wait for the main page to load        self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, a#findFriendsNav)))    def _get_friends_list(self):        return self.driver.find_elements_by_css_selector(.friendBrowserNameTitle > a)    def get_friends(self):        # navigate to friends page        self.driver.find_element_by_css_selector(a#findFriendsNav).click()        # continuous scroll until no more new friends loaded        num_of_loaded_friends = len(self._get_friends_list())        while True:            self.driver.execute_script(window.scrollTo(0, document.body.scrollHeight);)            try:                self.wait.until(lambda driver: len(self._get_friends_list()) > num_of_loaded_friends)                num_of_loaded_friends = len(self._get_friends_list())            except TimeoutException:                break  # no more friends loaded        return [friend.text for friend in self._get_friends_list()]if __name__ == '__main__':    crawler = FacebookCrawler(login='login', password='password')    for friend in crawler.get_friends():        print(friend)(not tested)