A script that scrapes all YouTube's videos closed captions(subtitles) of a single creator's video library. It scrapes either auto-generated or manually written subtitles. The script accepts a link to the creators video library, such as, https://www.youtube.com/user/creator/videos and outputs subtitles of each video to it own respective '.txt' file. I use Selenium to trigger the events and BeautifulSoup to scrape the content. import timefrom bs4 import BeautifulSoupimport urllib.requestfrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.common.by import Byimport reimport sys# Clicks on Load More button to display all users videos. def display_all_videos(driver):    while(True):        try:            element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, yt-uix-load-more)))            element.click()        except:            break# Creates a list of tuples (video_title, video_link) of all # videos displayed on page def video_list(driver):    videos = driver.find_elements_by_class_name(yt-uix-tile-link)    video_links = []    for vid in videos:        video_links.append((vid.text,vid.get_attribute(href)))    return video_links# Clicks on CC(Closed Caption) button in YouTube videodef enable_subtitles(driver):    elem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, ytp-subtitles-button)))    elem.click()def subtitles_link(driver):    time.sleep(1)    timings = driver.execute_script(return window.performance.getEntries();)    # Find string in timings that contains the substring 'srv3'    # which is the subtitles link.    link =     for t in timings:         for v in t.values():             if srv3 in str(v):                 link = v    return linkdef create_file(title, link,subtitles):    # remove illegal chars for file name    title = .join([c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()    try:        file = open(title + '.txt', 'w')            file.write('LINK: ' + link + '\n')        file.write(subtitles)        file.close()    except:        print(Can't create file for:  + title +  :  + link)def scrape_subtitles(subtitle_link):    r = urllib.request.urlopen(subtitle_link).read()    soup = BeautifulSoup(r)    # Remove tags (<*>), \n, and unecessary whitespace     s = re.sub(r'<.+?>', '', soup.prettify())       s = re.sub(r'\n', '', s)                        s = re.sub( '\s+', ' ', s ).strip()             return sdef main(argv):    driver = webdriver.Chrome('C:\Program Files\ChromeDriver\chromedriver.exe')    # Visit page and load all videos to create a list of    # tuples(video_name,video_link) of the videos     driver.get(argv[1])    display_all_videos(driver)    videos = video_list(driver)    # Visit video's page and enable 'CC' to scrape the subtitles and     # save subtitles to '.txt' file.     for v in videos:        driver.get(v[1])        try:            enable_subtitles(driver)            link = subtitles_link(driver)            subtitles = scrape_subtitles(link)        except:            subtitles = No Closed Caption        create_file(v[0],v[1],subtitles)if __name__ == __main__:    main(sys.argv)Readable?Follows Python's best practices?
Scraping all closed captions (subtitles) of a YouTube's creator's video library
python;web scraping;beautifulsoup;selenium;youtube
BugThis was probably a posting or a copy/paste bug - in create_file() you meant to name the first parameter to be filename instead of title.Code Style and PEP8 notesuse more descriptive variable names - r, s, c, v are not good variable namesremove unused TimeoutException import, organize imports in groups and alphabetically (PEP8 reference)move the comments before the functions into proper docstringsavoid catching broad exceptions via bare exceptfollow PEP8 whitespace and newline usage guidelines - specifically, whitespaces around the operators, two newlines between the function definitions Third-party Library Usage notesBeautifulSoup constructor accepts a file-like object as well - you don't have to explicitly call .read() methodit is a good idea to specify BeautifulSoup's parser explicitly:soup = BeautifulSoup(r, html.parser)  # or html5lib, or lxmlOtherwise, BeautifulSoup would pick a parser automatically depending on what is available. This would mean that on your machine it may pick, say, lxml - on someone else's - html5lib, on other's html.parser. And, there might be different results since different parser behave differently.a bigger problem is that you are not actually using BeautifulSoup capabilities at all - you are post-processing the soup with regular expressions. It looks like you can achieve the same by calling the .get_text() method on the soup objectfor better reusability I would probably assume chromedriver is in PATH instead of hardcoding the path to it explicitlyyou can reuse WebDriverWait instance instead of re-instantiating it multiple timesyou never close the driver instancetime.sleep() is generally unreliable when used for waiting for something with selenium - recheck if you actually need it Code Organization notesYou are passing driver to multiple methods - it may be a good idea to have a class with a driver and wait attributes for better reusability.Some Syntax Sugaryou can construct a list of videos with a list comprehension:[(video.text, video.get_attribute(href)) for video in driver.find_elements_by_class_name(yt-uix-tile-link)]you can unpack the list of videos into filename and a link:for filename, link in videos:Modified Code:import sysimport timeimport urllib.requestfrom bs4 import BeautifulSoupfrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.support.ui import WebDriverWaitclass YoutubeSubtitlesScraper:    def __enter__(self):        self.driver = webdriver.Chrome()        self.wait = WebDriverWait(self.driver, 10)        self.driver.get(start_url)        self.display_all_videos()    def __exit__(self, exc_type, exc_val, exc_tb):        self.driver.close()    def display_all_videos(self):        Clicks on Load More button to display all users videos.        while True:            try:                element = self.wait.until(EC.element_to_be_clickable((By.CLASS_NAME, yt-uix-load-more)))                element.click()            except TimeoutException:                break    def subtitles(self):        Visits video's page, enables 'CC' to scrape the subtitles and generates filename, link and the subtitles content.        videos = [(video.text, video.get_attribute(href))                  for video in self.driver.find_elements_by_class_name(yt-uix-tile-link)]        for filename, link in videos:            self.driver.get(link)            self.enable_subtitles()            link = self.get_subtitles_link()            yield filename, link, self.scrape_subtitles(link) if link else No Closed Caption    def enable_subtitles(self):        Clicks on CC(Closed Caption) button in YouTube video.        show_subtitles_button = self.wait.until(EC.element_to_be_clickable((By.CLASS_NAME, ytp-subtitles-button)))        show_subtitles_button.click()    def get_subtitles_link(self):        Finds string in performance timings that contains the substring 'srv3' which is the subtitles link.        time.sleep(1)        timings = self.driver.execute_script(return window.performance.getEntries();)        for timing in timings:            for value in timing.values():                if srv3 in str(value):                    return value        return     def scrape_subtitles(self, subtitle_link):        HTML parses subtitles.        response = urllib.request.urlopen(subtitle_link)        soup = BeautifulSoup(response, lxml)        return soup.get_text(strip=True)def create_file(filename, link, subtitles):    Creates file for the subtitle.    title = .join([c for c in filename if c.isalpha() or c.isdigit() or c == ' ']).rstrip()    with open(title + '.txt', 'w') as subtitles_file:        subtitles_file.write('LINK: ' + link + '\n')        subtitles_file.write(subtitles)if __name__ == __main__:    start_url = sys.argv[1]    with YoutubeSubtitlesScraper(start_url) as scraper:        for filename, link, content in scraper.subtitles():            try:                create_file(filename, link, content)            except:                print(Can't create file for:  + filename +  :  + link)(not tested)Note how we made the class a context manager by defining the __enter__() and __exit__() magic methods.