<jupyter_start><jupyter_text>[1] Mount Google Drive<jupyter_code>#@markdown <br><center><img src='https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/Google_Drive_logo.png/600px-Google_Drive_logo.png' height="50" alt="Gdrive-logo"/></center>
#@markdown <center><h3>Mount GDrive to /content/drive</h3></center><br>
MODE = "MOUNT" #@param ["MOUNT", "UNMOUNT"]
#Mount your Gdrive! 
from google.colab import drive
drive.mount._DEBUG = False
if MODE == "MOUNT":
  drive.mount('/content/drive', force_remount=True)
elif MODE == "UNMOUNT":
  try:
    drive.flush_and_unmount()
  except ValueError:
    pass
  get_ipython().system_raw("rm -rf /root/.config/Google/DriveFS")<jupyter_output><empty_output><jupyter_text>[2] Setup Working Environment<jupyter_code>import os
import shutil
import pandas as pd
import numpy as np
from bs4 import BeautifulSoup as bs
import re


path_datasets ='/content/drive/MyDrive/Github/Synopsis/Datasets'<jupyter_output><empty_output><jupyter_text>[3] Merge Scriptbase and TRIPOD Databases to SSGD  Define Common Utility Functions<jupyter_code>import nltk
from nltk.tokenize import RegexpTokenizer


def str_clean(s)-> str:
  # Clean strings
  if s:
    s = re.sub('Ã¢Â\x80Â\x94', '—', s)
    s = re.sub('\x94', '—', s)
    s = re.sub('Ã\x83Â¡', 'á', s)
    s = re.sub('Ã\x82Â\xa0', ' ', s)
    s = re.sub('$333\xa0', ' ', s)
    s = re.sub('Ã¢Â\x80Â\x99', '’', s)
    s = re.sub('\xa0', '', s)
    return s
  else:
    return

def ls_clean(ls)-> str:
  # Clean strings in a list
  ret = []
  if ls:
    for s in ls:
        ret.append(str_clean(s))
    return ret
  else:
    return


tokenizer = RegexpTokenizer(r'\w+')
def str_word_count(s: str) -> int:
  # count number of tokens in a string
  if s:
    tokenizer = RegexpTokenizer(r'\w+')
    tokens = tokenizer.tokenize(s)
    return len(tokens)
  else:
    return 0

def ls_word_count(ls):
  # count number of tokens for each string in a list
  counts = []
  tokenizer = RegexpTokenizer(r'\w+')
  if ls:
    for s in ls:
      counts.append(str_word_count(s))
    return counts
  else:
    return []<jupyter_output><empty_output><jupyter_text>Process scriptbase  from 'scriptbase_j' extract '[title]_script.txt'<jupyter_code>### Get all the cleaned screenplay txt from scriptbase_j

# Set Path to scriptbase_j
# Download scriptbase-master, extract, and set path to scriptbase_j
# scriptbase-master  -> scriptbase_j
path_scriptbase = path_data + '/' + 'scriptbase_j'

os.chdir(path_scriptbase)
for root, dirs, files in os.walk(path_scriptbase):
  #print('r', root, 'd', dirs, 'f', files)
  if root == path_scriptbase:
    fztars = files
    break

path_script_clean = path_scriptbase + '/script_clean' # target storage on drive
for ztar in fztars:
  fn = re.sub('.tar.gz','', ztar)
  fn = re.sub('_', ':', fn)
  path_script = './' + fn + '/processed/script_clean.txt' # processed script path within tar file
  path_tmp = path_scriptbase + '/' + ztar
  !tar -zxvf "$path_tmp" -C "$path_script_clean" "$path_script" --strip-components 3
  orig_name = path_script_clean + '/' + 'script_clean.txt'
  target_name = path_script_clean + '/' + fn + '_script.txt' 
  os.rename(orig_name, target_name)<jupyter_output><empty_output><jupyter_text>from 'scriptbase_alpha' extract summaries, taglines and loglines and wikiplots to 'alpha_sums.json'wikiplots, summaries, taglines and loglines can be view as summary to the screenplay with varying compression ratios.<jupyter_code># Extract Summaries, taglines and loglines from scriptbase_alpha and save to json
path_scriptbase_alpha = path_data + '/' + 'scriptbase_alpha'
os.chdir(path_scriptbase_alpha)
for root, dirs, files in os.walk(path_scriptbase_alpha):
  if root == path_scriptbase_alpha:
    fztars = files
    break
path_sum = path_scriptbase_alpha + '/sumtmp' # target storage on drive
#print(fztars)
summs = []
for ztar in fztars:
  fn = re.sub('.tar.gz', '', ztar)
  fn = re.sub('_', ':', fn)
  summ = {'name': fn}
  path_tmp = path_scriptbase_alpha + '/' + ztar
  path_summ = './' + fn + '/processed/summaries/*'
  path_wikiplot = './' + fn + '/processed/wikiplot.txt'
  !tar -zxvf "$path_tmp" -C "$path_sum" "$path_wikiplot" --strip-components 3
  !tar -zxvf "$path_tmp" -C "$path_sum" --wildcards  "$path_summ" --strip-components 4
  for root, dirs, files in os.walk(path_sum):
    if root == path_sum:
      sumfiles = files
      break
  sms = []
  for file in sumfiles:
    s = open(path_sum + '/' + file)
    if re.match('wiki*', file):
      summ['wikiplot'] = s.read()
    else:
      sms.append(s.read())
    s.close()
  summ['sums'] = sms
  summs.append(summ)
  shutil.rmtree(path_sum)
  os.mkdir(path_sum)
df_summs = pd.DataFrame(summs)
df_summs.to_json(path_scriptbase_alpha + '/' + 'alpha_sums.json')
path_datasets = '/content/drive/MyDrive/Github/Synopsis/Datasets'

# sums
df_sums = pd.read_json(path_datasets + '/' + 'alpha_sums.json')
df_sums['wikiplot'] = df_sums['wikiplot'].apply(str_clean)
df_sums['sums'] = df_sums['sums'].apply(ls_clean)

#
path_Elements = path_datasets + '/' + 'Element Report'
for root, dirs, files in os.walk(path_Elements):
  if root == path_Elements:
    fn_scriptbase_j = files
    break
print(len(fn_scriptbase_j), fn_scriptbase_j)

df_sums.to_json(path_datasets + '/' + 'alpha_sums.json')<jupyter_output><empty_output><jupyter_text>Process TRIPOD  from TRIPOD extract summaries to ''tripod_sums.json'<jupyter_code>path_tripod = '/content/drive/MyDrive/Github/Synopsis/Datasets/Tripod'
path_tripod_exceptions = path_tripod + '/' + 'exceptions'
for root, dirs, files in os.walk(path_tripod_exceptions):
  if root == path_tripod_exceptions:
    fn_tripod_exceptions = files
    break
print(len(fn_tripod_exceptions), ' ', fn_tripod_exceptions)

path_tripod_elements = path_tripod + '/' + 'Element_Report'
for root, dirs, files in os.walk(path_tripod_elements):
  if root == path_tripod_elements:
    fn_rotripod = files
    break
print(len(fn_rotripod), ' ', fn_rotripod)
# Tripod Summaries
file_train = '/content/drive/MyDrive/Github/Synopsis/Datasets/Tripod/TRIPOD_synopses_train.csv'
file_test = '/content/drive/MyDrive/Github/Synopsis/Datasets/Tripod/TRIPOD_synopses_test.csv'
file_test_sc = '/content/drive/MyDrive/Github/Synopsis/Datasets/Tripod/TRIPOD_screenplays_test.csv'

df_tripod_synopses_train = pd.read_csv(file_train, header=0, encoding='ISO-8859-1')
df_tripod_synopses_test = pd.read_csv(file_test, header=0, encoding='ISO-8859-1')
df_tripod_sc_tps = pd.read_csv(file_test_sc, header=0, encoding='ISO-8859-1')

df_tripod_synopses_train['synopsis_raw'] = df_tripod_synopses_train['synopsis_raw'].apply(str_clean)
df_tripod_synopses_train['synopsis_raw_wc'] = df_tripod_synopses_train['synopsis_raw'].apply(str_word_count)
df_tripod_synopses_train['tmp_name'] = df_tripod_synopses_train['movie_name'].apply(lambda x: x[:-2])
df_tripod_sums = df_tripod_synopses_train.loc[:, ['tmp_name', 'synopsis_raw', 'synopsis_raw_wc']]
df_tripod_sums.drop_duplicates(keep='first', inplace=True)
df_tripod_sums.rename(columns={'tmp_name': 'title'}, inplace=True)
df_tripod_sums.reset_index(drop=True, inplace=True)
df_tripod_sums['usage'] = 'train'

df_tripod_synopses_test['synopsis_raw'] = df_tripod_synopses_test['synopsis_raw'].apply(str_clean)
df_tripod_synopses_test['synopsis_raw_wc'] = df_tripod_synopses_test['synopsis_raw'].apply(str_word_count)
df_tripod_sums_test = df_tripod_synopses_test.loc[:, ['movie_name', 'synopsis_raw', 'synopsis_raw_wc']]

df_tripod_sums_test.drop_duplicates(keep='first', inplace=True)
df_tripod_sums_test.rename(columns={'movie_name': 'title'}, inplace=True)
df_tripod_sums_test.reset_index(drop=True, inplace=True)
df_tripod_sums_test.reset_index(drop=True, inplace=True)
df_tripod_sums_test['usage'] = 'test'

df_tripod_sums = pd.concat([df_tripod_sums, df_tripod_sums_test], axis=0, ignore_index=True)
df_tripod_sums.to_json(path_datasets + '/' + 'tripod_sums.json')<jupyter_output><empty_output><jupyter_text>Combine Tripod and ScriptBase  Preparing ScriptBase for Incoming Merges<jupyter_code># scriptbase
df_scriptbase_sums = pd.read_json(path_datasets + '/' + 'alpha_sums.json', encoding='utf-8')
df_scriptbase_sums = df_scriptbase_sums.rename(columns={'name': 'title'})
df_scriptbase_sums = df_scriptbase_sums.sort_values(by=['title'], ascending=True).reset_index(drop=True)

# tripod
df_tripod_sums = pd.read_json(path_datasets + '/' + 'tripod_sums.json', encoding='utf-8')
df_tripod_sums = df_tripod_sums.sort_values(by=['title'], ascending=True).reset_index(drop=True)
df_tripod_sums['sum_200_400'] = np.empty((df_tripod_sums.shape[0], 0)).tolist()
df_tripod_sums['sum_400_800'] = np.empty((df_tripod_sums.shape[0], 0)).tolist()
df_tripod_sums['sum_800_1600'] = np.empty((df_tripod_sums.shape[0], 0)).tolist()

df_tripod_sums.loc[(df_tripod_sums['synopsis_raw_wc'] >= 200) & (df_tripod_sums['synopsis_raw_wc'] < 400), 'sum_200_400'] = \
df_tripod_sums.loc[(df_tripod_sums['synopsis_raw_wc'] >= 200) & (df_tripod_sums['synopsis_raw_wc'] < 400), 'synopsis_raw'].apply(lambda x: [x])

df_tripod_sums.loc[(df_tripod_sums['synopsis_raw_wc'] >= 400) & (df_tripod_sums['synopsis_raw_wc'] < 800), 'sum_400_800'] = \
df_tripod_sums.loc[(df_tripod_sums['synopsis_raw_wc'] >= 400) & (df_tripod_sums['synopsis_raw_wc'] < 800), 'synopsis_raw'].apply(lambda x: [x])

df_tripod_sums.loc[(df_tripod_sums['synopsis_raw_wc'] >= 800) & (df_tripod_sums['synopsis_raw_wc'] < 1600), 'sum_800_1600'] = \
df_tripod_sums.loc[(df_tripod_sums['synopsis_raw_wc'] >= 800) & (df_tripod_sums['synopsis_raw_wc'] < 1600), 'synopsis_raw'].apply(lambda x: [x])

df_tripod_sums.drop(columns=['synopsis_raw', 'synopsis_raw_wc'], inplace=True)
df_tripod_sums
# process wikiplot
df_scriptbase_sums['wiki_wc'] = df_scriptbase_sums['wikiplot'].fillna('').apply(str_word_count)
df_scriptbase_sums['sums_wc'] = df_scriptbase_sums['sums'].apply(ls_word_count)
df_scriptbase_sums['wiki_wc'].dropna().describe()
# setup sum columns by wc interval
df_scriptbase_sums['sum_50_minus'] = np.empty((df_scriptbase_sums.shape[0], 0)).tolist()
df_scriptbase_sums['sum_50_200'] = np.empty((df_scriptbase_sums.shape[0], 0)).tolist()
df_scriptbase_sums['sum_200_400'] = np.empty((df_scriptbase_sums.shape[0], 0)).tolist()
df_scriptbase_sums['sum_400_800'] = np.empty((df_scriptbase_sums.shape[0], 0)).tolist()
df_scriptbase_sums['sum_800_1600'] = np.empty((df_scriptbase_sums.shape[0], 0)).tolist()
df_scriptbase_sums['sum_1600_plus'] = np.empty((df_scriptbase_sums.shape[0], 0)).tolist()

df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] < 50), 'sum_50_minus'] = \
df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] < 50), 'wikiplot'].apply(lambda x: [x])

df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 50) & (df_scriptbase_sums['wiki_wc'] < 200), 'sum_50_200'] = \
df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 50) & (df_scriptbase_sums['wiki_wc'] < 200), 'wikiplot'].apply(lambda x: [x])

df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 200) & (df_scriptbase_sums['wiki_wc'] < 400), 'sum_200_400'] = \
df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 200) & (df_scriptbase_sums['wiki_wc'] < 400), 'wikiplot'].apply(lambda x: [x])

df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 400) & (df_scriptbase_sums['wiki_wc'] < 400), 'sum_400_800'] = \
df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 800) & (df_scriptbase_sums['wiki_wc'] < 800), 'wikiplot'].apply(lambda x: [x])

df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 800) & (df_scriptbase_sums['wiki_wc'] < 1600), 'sum_800_1600'] = \
df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 800) & (df_scriptbase_sums['wiki_wc'] < 1600), 'wikiplot'].apply(lambda x: [x])

df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 1600), 'sum_1600_plus'] = \
df_scriptbase_sums.loc[(df_scriptbase_sums['wiki_wc'] >= 1600), 'wikiplot'].apply(lambda x: [x])

# process sums
for idx, row in df_scriptbase_sums[['sums', 'sums_wc']].iterrows():
  for i, n in enumerate(row['sums_wc']):
    if n < 50:
      df_scriptbase_sums.loc[idx, 'sum_50_minus'].append(row['sums'][i])
    elif n >=50 and n < 200:
      df_scriptbase_sums.loc[idx, 'sum_50_200'].append(row['sums'][i])
    elif n >=200 and n < 400:
      df_scriptbase_sums.loc[idx, 'sum_200_400'].append(row['sums'][i])

df_scriptbase_sums.tail(10)<jupyter_output><empty_output><jupyter_text>Merge TRIPOD summaries to Scriptbase and remove duplicates<jupyter_code># merge summaries from tripod to scriptbase for titles in both
df_scriptbase_sums['source'] = [{'scriptbase'}]*df_scriptbase_sums.shape[0]
df_tripod_sums['source'] = [{'tripod'}]*df_tripod_sums.shape[0]

df_scriptbase_sums.loc[df_scriptbase_sums['title'].isin(df_tripod_sums['title']), 'source'].apply(lambda x: x.add('tripod'))
df_tripod_sums.loc[df_tripod_sums['title'].isin(df_scriptbase_sums['title']), 'source'].apply(lambda x: x.add('scriptbase'))

for idx, row in df_scriptbase_sums.loc[df_scriptbase_sums['title'].isin(df_tripod_sums['title']), :].iterrows():
  df_scriptbase_sums.loc[idx, 'sum_200_400'].extend(df_tripod_sums.loc[df_tripod_sums['title'] == row['title'], 'sum_200_400'].iloc[0])
  df_scriptbase_sums.loc[idx, 'sum_400_800'].extend(df_tripod_sums.loc[df_tripod_sums['title'] == row['title'], 'sum_400_800'].iloc[0])
  df_scriptbase_sums.loc[idx, 'sum_800_1600'].extend(df_tripod_sums.loc[df_tripod_sums['title'] == row['title'], 'sum_800_1600'].iloc[0])

df_scriptbase_sums1 = df_scriptbase_sums[['title', 'sum_50_minus', 'sum_50_200', 'sum_200_400', 'sum_400_800', 'sum_800_1600', 'sum_1600_plus']]
df_scriptbase_sums1.sort_values('title', ascending=True, inplace=True)
df_tripod_not_in_scriptbase = df_tripod_sums.loc[~df_tripod_sums['title'].isin(df_scriptbase_sums1['title']), :]
df_combined = pd.concat([df_scriptbase_sums1, df_tripod_not_in_scriptbase], axis=0, ignore_index=True)
df_combined.sort_values(by='title', ascending=True).reset_index(drop=True)
#Index(['title', 'wikiplot', 'sums', 'wiki_wc', 'sums_wc', 'sum_50_minus',
#       'sum_50_200', 'sum_200_400', 'sum_400_800', 'sum_800_1600',
#       'sum_1600_plus', 'source'],
#      dtype='object')

#df_combined['sum_200_minus'] = df_combined['sum_200_minus'].fillna('').apply(list)
#df_combined['sum_200_minus'].apply(lambda x: list(set(x)))

# removed duplicated summaries in df_combined
df_combined.loc[df_combined['sum_50_minus'].str.len() > 1, 'sum_50_minus'] = \
  df_combined.loc[df_combined['sum_50_minus'].str.len() > 1, 'sum_50_minus'].apply(lambda x: list(set(x)))

df_combined.loc[df_combined['sum_50_200'].str.len() > 1, 'sum_50_200'] = \
  df_combined.loc[df_combined['sum_50_200'].str.len() > 1, 'sum_50_200'].apply(lambda x: list(set(x)))

df_combined.loc[df_combined['sum_200_400'].str.len() > 1, 'sum_200_400'] = \
  df_combined.loc[df_combined['sum_200_400'].str.len() > 1, 'sum_200_400'].apply(lambda x: list(set(x)))

df_combined.loc[df_combined['sum_400_800'].str.len() > 1, 'sum_400_800'] = \
  df_combined.loc[df_combined['sum_400_800'].str.len() > 1, 'sum_400_800'].apply(lambda x: list(set(x))[:1])

df_combined.loc[df_combined['sum_800_1600'].str.len() > 1, 'sum_800_1600'] = \
  df_combined.loc[df_combined['sum_800_1600'].str.len() > 1, 'sum_800_1600'].apply(lambda x: list(set(x))[:1])

df_combined.loc[df_combined['sum_1600_plus'].str.len() > 1, 'sum_1600_plus'] = \
  df_combined.loc[df_combined['sum_1600_plus'].str.len() > 1, 'sum_1600_plus'].apply(lambda x: list(set(x))[:1])
# fillna
df_combined['sum_500_minus'] = df_combined['sum_50_minus'].fillna('').apply(list)
df_combined['sum_50_200'] = df_combined['sum_50_200'].fillna('').apply(list)
df_combined['sum_200_400'] = df_combined['sum_200_400'].fillna('').apply(list)
df_combined['sum_400_800'] = df_combined['sum_400_800'].fillna('').apply(list)
df_combined['sum_800_1600'] = df_combined['sum_800_1600'].fillna('').apply(list)
df_combined['sum_1600_plus'] = df_combined['sum_1600_plus'].fillna('').apply(list)<jupyter_output><empty_output><jupyter_text>Save Merged Files to 'SSGD_1280.json'<jupyter_code>df_combined
df_combined.to_json(path_datasets + '/' + 'SSGD_1280.json')<jupyter_output><empty_output><jupyter_text>[4] Process Element Report, Scene Report and Stats ReportScriptBase - scriptbase_j offers 3423 cleaned screenplay in txt format. Only few films in TRIPOD are not in ScriptBase, and the screenplay textfiles in TRIPOD are not processed for element extraction. Therefore, SSGD only extracts Element Report, Scene Report and Stats Report using cleaned screenplays from scriptbase_j. SSGD uses FadeIn Screenwriting Software to generate reports in html and then process these htmls into json format.Element Report*   List item*   List item  Import Lib and Load Functions<jupyter_code>import os
import shutil
import pandas as pd
import numpy as np
from bs4 import BeautifulSoup as bs
import re

path_datasets ='/content/drive/MyDrive/Github/Synopsis/Datasets'<jupyter_output><empty_output><jupyter_text>Load summary df<jupyter_code>df_sums = pd.read_json(path_datasets + '/' + 'sums1280.json', encoding='utf-8')
df_sums.head(2)<jupyter_output><empty_output><jupyter_text>Element Report<jupyter_code>path_element = path_datasets + '/' + 'Element Report'
for root, dirs, files in os.walk(path_element):
  if root == path_element:
    fns = files
    break
print(fns)

for fn in fns:
  f = open(path_element + '/' + fn, "r")
  bs_element = bs(f.read(), 'html5lib').body
  record = {'title': re.split('_script - ', fn)[0]}
  df_single = pd.read_html(str(bs_element), header=0)[0]
  df_single.drop('Unnamed: 2', axis=1, inplace=True)
  df_single.to_json(path_element + '/processed/' + record['title'] + '_element.json')<jupyter_output>['One Eight Seven_script - Element Report.html', 'Prometheus (2012 film)_script - Element Report.html', 'Psycho (1960 film)_script - Element Report.html', 'Pulp Fiction_script - Element Report.html', 'Punch-Drunk Love_script - Element Report.html', 'Purple Rain (film)_script - Element Report.html', 'Queen of the Damned_script - Element Report.html', 'Quills_script - Element Report.html', 'Rabid_script - Element Report.html', 'Rachel Getting Married_script - Element Report.html', 'Pretty Woman_script - Element Report.html', 'Raging Bull_script - Element Report.html', 'Rambling Rose (film)_script - Element Report.html', 'Rambo_ First Blood Part II_script - Element Report.html', 'Rapture (1965 film)_script - Element Report.html', 'Real Genius_script - Element Report.html', 'Rear Window_script - Element Report.html', 'Reindeer Games_script - Element Report.html', 'Remember Me (2010 film)_script - Element Report.html', 'Requiem for a Dream_script - Element Report.html', 'Reservoir Dogs_scri[...]<jupyter_text>Statistics Report<jupyter_code>def html2df_stats(fns, path_dir: str) -> pd.DataFrame:

  # Read Stat Records to df_stats
  records = []

  for fn in fns:
    f = open(path_dir + '/' + fn, "r")
    bs_stats = bs(f.read(), 'html5lib').body
    record = {'title': re.split('_script - ', fn)[0]}

    # get page, paragraph and word counts
    df_tb_wc =  pd.read_html(str(bs_stats.table))[0].set_index(0)
    record.update({'page_count': df_tb_wc.loc['Pages:', 1]})
    record.update({'para_count': df_tb_wc.loc['Paragraphs:', 1]})
    record.update({'word_count': df_tb_wc.loc['Words:', 1]})

    # get %Action, %Dialog
    loc_ADratio =  bs_stats.find(string=re.compile('The ratio of Action to Dialogue on the page is'))
    if loc_ADratio:
      if re.search('([0-9\.]*\%)', loc_ADratio.next_element.string):
        record.update({'%Action': re.search('([0-9\.]*\%)', loc_ADratio.next_element.string)[0]})
      if re.search('([0-9\.]*\%)', loc_ADratio.next_element.next_element.next_element.next_element.string):
        record.update({'%Dialog': re.search('([0-9\.]*\%)', loc_ADratio.next_element.next_element.next_element.next_element.string)[0]})
    else:
      print('NO RATIO: ', record['title'])


    # get character list
    loc_charlist = bs_stats.find('p', string=re.compile('Character Names by First Letter'))
    if loc_charlist:
      for sib in loc_charlist.next_siblings:
        if re.search('table', repr(sib)):
          df_char = pd.read_html(repr(sib), header=0)[0]
          record.update({'characters': np.array(df_char['Characters'].str.split(', ').sum())})
          break
    else:
      print('NO CHAR: ', record['title'])

    # frequent word list
    loc_frequent_words = bs_stats.find('p', string=re.compile('Word Occurrence'))
    if loc_frequent_words:
      for sib in loc_frequent_words.next_siblings:
        if re.search('table', repr(sib)):
          df_frequent_words = pd.read_html(repr(sib), header=None)[0]
          record.update({'frequent_words':df_frequent_words.to_numpy().flatten()})
          break
    else:
      print('NO FREQ WORDS: ', record['title'])

    # profanity
    loc_profanity = bs_stats.find('p', string=re.compile('Profanity'))
    if loc_profanity:
      for sib in loc_profanity.next_siblings:
        if re.search('table', repr(sib)):
          df_profanity = pd.read_html(repr(sib), header=None)[0]
          record.update({'profanity': df_profanity.apply(lambda x: str(x[0]) + ' (' + str(x[1]) + ')', axis=1).to_numpy()})
          break
    else:
      print('NO PROFANITY: ', record['title'])

    # append to record
    records.append(record)
  return pd.DataFrame(records)
path_stats = path_datasets + '/' + 'Statistics Report'

for root, dirs, files in os.walk(path_stats):
  if root == path_stats:
    fns = files
    break
df_stats_scriptbase = html2df_stats(fns, path_stats)

# Flag records with errors for removal
df_stats_scriptbase['remove'] = False
df_stats_scriptbase.loc[df_stats_scriptbase['title'] == 'The Goonies', 'remove'] = True
df_stats_scriptbase.loc[df_stats_scriptbase['title'] == 'Taking Lives (film)', 'remove'] = True
df_stats_scriptbase.to_json(path_datasets + '/' + 'stats_scriptbase.json')
### TOO MESSY TO CLEAN!!!

path_stats_tripod = path_datasets + '/Tripod/' + 'Statistics Report'
for root, dirs, files in os.walk(path_stats_tripod):
  if root == path_stats_tripod:
    fns_tripod = files
    break
tmp_fns_tripod = [re.sub('_', ':', t[:-32]) + t[len(t)-32:] for t in fns_tripod]
fns_tripod_unique = list((filter(lambda x: x not in fns, fns_tripod)))

#df_stats_tripod = html2df_stats(fns_tripod_unique[:5], path_stats_tripod)
fns_tripod_unique<jupyter_output><empty_output><jupyter_text>Create Scene Report '[title]_scene.json'<jupyter_code>path_scene = path_datasets + '/' + 'Scene Report'
for root, dirs, files in os.walk(path_scene):
  if root == path_scene:
    fns = files
    break
print(fns)
#df_scene_scriptbase = html2df_stats(fns, path_stats)

for fn in fns:
  f = open(path_scene + '/' + fn, "r")
  bs_scene = bs(f.read(), 'html5lib').body
  record = {'title': re.split('_script - ', fn)[0]}
  df_single = pd.read_html(str(bs_scene), header=0)[0]
  df_single['Scene'].fillna(method='ffill', inplace=True)
  df_single['Scene'].fillna(-1, inplace=True)

  # get scene length
  df_processed = df_single[['Scene', 'Length']].dropna().reset_index(drop=True)

  # get scene heading
  dftmp = df_single[['Scene', 'Scene Heading']].dropna()
  tmp = dftmp.loc[~dftmp['Scene Heading'].str.contains('([0-9])'), :].reset_index(drop=True)
  df_processed = df_processed.merge(tmp, on='Scene', how='right')
  tmp = dftmp.loc[dftmp['Scene Heading'].str.contains('([0-9])'), :].reset_index(drop=True)
  tmp.columns = ['Scene', 'Char']
  df_processed = df_processed.merge(tmp, on='Scene', how='left')
  df_processed.to_json(path_scene + '/processed/' + record['title'] + '_scene.json')<jupyter_output><empty_output><jupyter_text>[5] Create SSGD Database 1.0, with 779 Screenplays in Element Format  Load df_sums and match to screenplays<jupyter_code># load cases with sums
df_sums = pd.read_json(path_datasets + '/' + 'SSGD_1280.json', encoding='utf-8')


# load cases with screenplays in with element labels
path_elements = path_datasets + '/Element Report/processed'
for root, dirs, files in os.walk(path_elements):
  if root == path_elements:
    fns = [fn.split('_element')[0] for fn in files]
    break

# find titles in df_sums with element scripts
fns_formatted = [re.sub('_', ':', fn) for fn in fns] #filename -> title in record
formatted2fn = dict(zip(fns_formatted, fns))
df_wscript = df_sums.loc[df_sums['title'].isin(fns_formatted),:].copy()


# Groups summaries of different length to a list, sorted from longesst to shortest word count
df_wscript['ls_sums_sorted'] = df_wscript[['sum_50_minus',
       'sum_50_200', 'sum_200_400', 'sum_400_800', 'sum_800_1600',
       'sum_1600_plus']].agg(sum, axis=1).apply(lambda x: sorted(x, key=lambda s: str_word_count(s), reverse=True))
df_wscript = df_wscript[['title', 'ls_sums_sorted']].copy()
#df_wscript['ls_sums_sorted'].apply(ls_word_count)<jupyter_output><empty_output><jupyter_text>Merge Element Screenplays to Summary Dataframe<jupyter_code># Merge screenplay to df cases
pdtables = []
for title in df_wscript['title']:
  pdtables.append(pd.read_json(path_elements + '/' + formatted2fn[title] + '_element.json'))
  #print(path_elements + '/' + formatted2fn[title] + '_element.json')
df_wscript['dfsc'] = pdtables
## Save to 'SSGD-2021-07-05-779SC.json'
df_wscript.to_json(path_datasets + '/SSGD-2021-07-05-779SC.json')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15808008863564418, 'fraction_numerical': 0.030231085786641344, 'mean_word_length': 6.049651324965133, 'pattern_counts': {'":': 2, '<': 92, '<?xml version=': 0, '>': 109, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '260884', 'license': 'MIT', 'n_tokens_mistral': 10365, 'n_tokens_neox': 9656, 'n_words': 1633, 'path': 'data_cleaning.ipynb', 'repo_name': 'Herais/Synopsis', 'size': '25272'}