<jupyter_start><jupyter_text>Name_basics<jupyter_code>name_basics = imdb.name_basics_df()

name_basics.info(memory_usage='deep')
name_basics.head()
name_basics.describe(include='all')
name_basics['nconst'] = transform.nconst_to_float(name_basics['nconst'])

name_basics.info(memory_usage='deep')<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 9004834 entries, 0 to 9004833
Data columns (total 6 columns):
nconst               uint32
primaryName          object
birthYear            float64
deathYear            float64
primaryProfession    object
knownForTitles       object
dtypes: float64(2), object(3), uint32(1)
memory usage: 1.9 GB<jupyter_text>title_basics<jupyter_code>title_basics = imdb.title_basics_df()
title_basics.head()
title_basics_pre = title_basics.copy()
title_basics_pre['tconst'] = transform.tconst_to_float(title_basics_pre['tconst'])

title_basics_pre.info()

title_basics_pre.head()
title_basics_pre.to_csv('./title_basics.data', index=False, sep='\t', header=False)<jupyter_output><empty_output><jupyter_text>title_akas<jupyter_code>title_akas = imdb.title_akas_df()
title_akas.head()
title_akas_pre = title_akas.copy()
title_akas_pre['titleId'] = transform.tconst_to_float(title_akas_pre['titleId'])
print('Shape', title_akas_pre.shape)

# Remove title_akas for non-existing# title_basic
title_akas_pre = title_akas_pre[title_akas_pre['titleId'].isin(title_basics_pre['tconst'])]

print('Shape', title_akas_pre.shape)
title_akas_pre.head()
%%timeit -n 1 -r 1

df_to_mysql(title_akas_pre, 'title_akas')

#save_csv(title_akas_pre, 'title_akas.csv')<jupyter_output>Table title_akas deleted
3min 12s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)<jupyter_text>title_crew<jupyter_code>title_crew = imdb.title_crew_df()
print("Shape", title_crew.shape)
title_crew.head(10)
title_crew_pre = title_crew.copy()

# Expand rows based on directors and writers list
title_crew_pre['directors'] = title_crew_pre['directors'].astype('str')
title_crew_pre = transform.expand_rows_using_repeat(title_crew_pre, 'directors', ',')

title_crew_pre['writers'] = title_crew_pre['writers'].astype('str')
title_crew_pre = transform.expand_rows_using_repeat(title_crew_pre, 'writers', ',')

title_crew_pre.rename(index=str, columns={"directors": "director", "writers": "writer"}, inplace=True)

# Transform identifiers
title_crew_pre['tconst'] = transform.tconst_to_float(title_crew_pre['tconst'])
title_crew_pre['director'] = transform.nconst_to_float(title_crew_pre['director'])
title_crew_pre['writer'] = transform.nconst_to_float(title_crew_pre['writer'])

print('Shape', title_crew_pre.shape)
title_crew_pre.head(10)
%%timeit -n 1 -r 1

df_to_mysql(title_crew_pre, 'title_crew')

#save_csv(title_crew_pre, 'title_crew.csv')<jupyter_output>Table title_crew deleted<jupyter_text>title_episode<jupyter_code>title_episode = title_episode_df()
print('Shape', title_episode.shape)
title_episode.head()
title_episode_pre = title_episode.copy()

# Transform identifiers
title_episode_pre['tconst'] = tconst_transform(title_episode_pre['tconst'])
title_episode_pre['parentTconst'] = tconst_transform(title_episode_pre['parentTconst'])

print('Shape', title_episode_pre.shape)
title_episode_pre.head()
save_csv(title_episode_pre, 'title_episode.csv')<jupyter_output><empty_output><jupyter_text>title_principals<jupyter_code>title_principals = imdb.title_principals_df()

title_principals.info(memory_usage='deep')
title_principals.head()
title_principals.describe(include='all')
for dtype in ['float','int','object']:
    selected_dtype = title_principals.select_dtypes(include=[dtype])
    mean_usage_b = selected_dtype.memory_usage(deep=True).mean()
    mean_usage_mb = mean_usage_b / 1024 ** 2
    print("Average memory usage for {} columns: {:03.2f} MB".format(dtype,mean_usage_mb))

del title_principals
gc.collect()<jupyter_output><empty_output><jupyter_text>title_ratings<jupyter_code>title_ratings = title_ratings_df()

print('Shape', title_ratings.shape)
title_ratings.head()
title_ratings_pre = title_ratings.copy()

# Transform identifiers
title_ratings_pre['tconst'] = transform.tconst_to_float(title_ratings_pre['tconst'])

print('Shape', title_ratings_pre.shape)
title_ratings_pre.head()
save_csv(title_ratings_pre, 'title_ratings.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12831241283124128, 'fraction_numerical': 0.012319851231985124, 'mean_word_length': 9.36867469879518, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '68474', 'license': 'MIT', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1623, 'n_words': 208, 'path': 'IMDb/IMDB Exploration.ipynb', 'repo_name': 'ingJSNA/ADB', 'size': '4302'}