<jupyter_start><jupyter_text>Extract Information about Text  1. Counts    - Number of Upper Words / Characters    - Number of Words before text cleaning    - Number of Characters before text cleaning<jupyter_code>sample_df['upper_word_count'] = sample_df.name.apply(lambda x: len([x for x in x.split() if x.isupper()]))
sample_df['upper_char_count'] = sample_df.name.apply(lambda x: len([x for x in x if x.isupper()]))
sample_df['bef_word_count'] = sample_df.name.apply(lambda x: len(x.split()))
sample_df['bef_char_count'] = sample_df.name.apply(lambda x: len(x))                                                    
sample_df.head(5)<jupyter_output><empty_output><jupyter_text>2. Count Number of Stop words<jupyter_code>stop = stopwords.words('english')
sample_df['stopword_count'] = sample_df.name.apply(lambda x: len([x for x in x.split() if x.lower() in stop]))
sample_df.stopword_count.max()<jupyter_output><empty_output><jupyter_text>3. Count Number of Punctuations<jupyter_code>sample_df['punctuation_count'] = sample_df.name.apply(lambda x: len([x for x in x if x in string.punctuation]))
sample_df.punctuation_count.max()<jupyter_output><empty_output><jupyter_text>4. Count Number of Numerical Characters<jupyter_code>sample_df['number_count'] = sample_df.name.apply(lambda x: len([x for x in x.split() if x.isdigit()]))
sample_df.number_count.max()<jupyter_output><empty_output><jupyter_text>5. Average Word Length<jupyter_code>sample_df['bef_avg_word_len'] = sample_df['bef_char_count'] / sample_df['bef_word_count']
sample_df.head(5)<jupyter_output><empty_output><jupyter_text>Text Cleaning  1. Make All Text Lower Case<jupyter_code>sample_df['clean_name'] = sample_df.name.apply(lambda x: x.lower() )
sample_df.clean_name.head(3)<jupyter_output><empty_output><jupyter_text>2.  Removing Punctuations<jupyter_code>sample_df.clean_name = sample_df.clean_name.apply(
    lambda x: x.translate(str.maketrans('', '', string.punctuation)))<jupyter_output><empty_output><jupyter_text>3. Removing Stopwords<jupyter_code>sample_df.clean_name = sample_df.clean_name.apply(lambda x: " ".join(x for x in x.split() if x not in stop))<jupyter_output><empty_output><jupyter_text>4. Correct Spelling<jupyter_code>sample_df.clean_name = sample_df.clean_name.apply(lambda x: str(TextBlob(x).correct()))<jupyter_output><empty_output><jupyter_text>5. Lemmatization- converting a word to its base form.<jupyter_code>import ssl
ssl._create_default_https_context = _create_unverified_https_context

# # Download Wordnet
nltk.download('wordnet')

# Init the Wordnet Lemmatizer
lemmatizer = WordNetLemmatizer()
sample_df.clean_name = sample_df.clean_name.apply(lambda x: lemmatizer.lemmatize(x))
sample_df.head(5)<jupyter_output><empty_output><jupyter_text>Extract Information about Text After Text Cleaning- Number of Words after text cleaning- Number of Characters after text cleaning- Avg word length after text cleaning<jupyter_code>sample_df['aft_word_count'] = sample_df.clean_name.apply(lambda x: len(x.split()))
sample_df['aft_char_count'] = sample_df.clean_name.apply(lambda x: len(x)) 
sample_df['aft_avg_word_len'] = sample_df['aft_char_count'] / sample_df['aft_word_count']
sample_df.head(5)
sample_df.info()
sample_df.to_csv("../../data/sample_data_item_name_features.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12655161973963064, 'fraction_numerical': 0.004541326067211626, 'mean_word_length': 7.178217821782178, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '422583', 'license': 'MIT', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1178, 'n_words': 206, 'path': 'experiment/feature_extraction/FeatureExtraction_ItemName_E2.ipynb', 'repo_name': 'DSEI21000-S21/project-product-price-prediction', 'size': '3303'}