<jupyter_start><jupyter_text>Gillian's test notebook 11/12/19 in class work to pull movie list.<jupyter_code>#import dependencies 
import pandas as pd
import time
import os
from bs4 import BeautifulSoup as bs
from splinter import Browser
# initialize path /browser
executable_path = {"executable_path": "chromedriver"}
browser = Browser("chrome", **executable_path, headless=False)
# URL of page to be scraped 
url = "https://www.imsdb.com/all%20scripts/"
browser.visit(url)
time.sleep(1)
# beautiful soup object
html = browser.html
soup = bs(html, 'html.parser')
#test to see if the code works- scraping the first movie title 
titles = soup.find('p', title_='').text
print(titles)
#list of movies in IMSDB
movies = soup.find_all('a', class_='')

#create list
movie_title = []
movie_links= []

for movie in movies:
    title = movie.text
    movie_title.append(title)
#printing list to check 
movie_title
#counting how many values are in the list 
len(movies)
# scraping URL list 
movieurls = soup.find_all('a', class_='')
for x in movieurls:
    links = x.get('href')
    movie_links.append(links)
movie_links
#checking to see if the length matches the movie titles 
len(movie_links)
#list to dataframe. setting index for merge
title_df = pd.DataFrame(movie_title)
title_df.reset_index(level=0, inplace=True)
title_df.head()
#list to dataframe. setting index for merge
link_df = pd.DataFrame(movie_links)
link_df.reset_index(level=0, inplace=True)
link_df.head()
#merging dataframes , renaming the columns 
movies_df = pd.merge(title_df, link_df, on='index')
movies_df = movies_df.rename(columns ={'0_x':"Movie_Title", '0_y': "Movie_Links"})
movies_df.head()
#output datafram to csv file for easier clean up 
movies_df.to_csv("../output/movie_list.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11378923766816143, 'fraction_numerical': 0.007847533632286996, 'mean_word_length': 5.611111111111111, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '338437', 'license': 'MIT', 'n_tokens_mistral': 636, 'n_tokens_neox': 608, 'n_words': 181, 'path': 'Resources/gillian_notebook.ipynb', 'repo_name': 'gilliangadiano/ETL-Project', 'size': '1784'}