<jupyter_start><jupyter_text>Read datasetread all the data and convert them into pandas dataframes<jupyter_code>import json
from pprint import pprint

cwd = os.getcwd()
# print(cwd)
frames = []
for file in os.listdir('lyrics_50_18/')[1500:]:
    date = file.split('.')[0].split('_')[-1]
    with open(cwd + '/lyrics_50_18/' + file,'r') as f:
        df = pd.DataFrame(json.load(f)).dropna()
        df['time'] = pd.Timestamp(date)
        frames.append(df)
raw_data = pd.concat(frames).reset_index()  #.drop_duplicates(['artist', 'title']) 
raw_data<jupyter_output><empty_output><jupyter_text>Data preprocessing1. str to lower2. remove stop words3. remove non-lyric content (eg: [Chorus: David McRae])4. remove punctuations<jupyter_code>import copy
billboard = copy.deepcopy(raw_data)
stop = stopwords.words('english')

def preprocess(lyric):
    lyric = lyric.lower()
    lyric = re.sub(r'\[.*?\]', '', lyric)  # remove [*] pattern
    lyric = lyric.replace("'s",'')
    lyric = lyric.replace("'ve",'')
    lyric = lyric.replace("'",'')    # ' must be ignored
    lyric = lyric.replace("-",' ')
    
    translator = str.maketrans(string.punctuation,' '*len(string.punctuation))
    lyric = lyric.translate(translator)
    tokens = nltk.word_tokenize(lyric)
    
    return tokens

billboard['lyrics'] = billboard['lyrics'].apply(lambda x: preprocess(x))
billboard['lyrics'] = billboard['lyrics'].apply(lambda x:[w for w in x if w not in stop])
billboard.head(10)<jupyter_output><empty_output><jupyter_text><jupyter_code>billboard['vocab_cnt'] = billboard['lyrics'].str.len()
billboard.head(10)
total_year_cnt = billboard.groupby(billboard.time.dt.year)['vocab_cnt'].sum()
plt.figure(figsize=(12, 9))
plt.title("Lyrics Vocabulary Count by Year", fontsize=20)
total_year_cnt.plot(kind='line', color='Red')
def get_NN_pos_tag(lyric):
    return len([w for w, pos in pos_tag(lyric) if pos == 'NN'])

billboard['nouns_cnt'] = billboard['lyrics'].apply(lambda x: get_NN_pos_tag(x))
billboard.head(10)
nouns_cnt = billboard.groupby(billboard.time.dt.year)['nouns_cnt'].sum()
plt.figure(figsize=(12, 9))
plt.title("Nouns Count by Year in Lyric", fontsize=20)
nouns_cnt.plot(kind='line', color='Blue')
def get_Adj_pos_tag(lyric):
    return len([w for w, pos in pos_tag(lyric) if pos == 'JJR'])

billboard['adjs_cnt'] = billboard['lyrics'].apply(lambda x: get_Adj_pos_tag(x))
billboard.head(10)
adjs_cnt = billboard.groupby(billboard.time.dt.year)['adjs_cnt'].sum()
plt.figure(figsize=(12, 9))
plt.title("Adjs Count by Year in Lyric", fontsize=20)
adjs_cnt.plot(kind='line', color='Green')
artist_cnt = billboard.artist.value_counts()[:25]
artist_cnt
plt.figure(figsize=(13,9))
plt.title("Maximum Vocabulary Count By Artist",fontsize=20)
vocab_cnt_by_artist = billboard.groupby('artist').agg({'vocab_cnt': sum}) \
                        .sort_values(['vocab_cnt'], ascending=False).vocab_cnt[:30]
vocab_cnt_by_artist.plot('bar',color='teal')
plt.figure(figsize=(12,5))
plt.title("Max Repeated Words In Song",fontsize=20)
words = pd.Series(billboard['lyrics'].sum()).value_counts()
words.plot(color='orange')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1740657936761418, 'fraction_numerical': 0.01756627275630789, 'mean_word_length': 5.779220779220779, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '138774', 'license': 'MIT', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1181, 'n_words': 207, 'path': 'stage1.ipynb', 'repo_name': 'KuroLight/Billboard-Trends', 'size': '3131'}