<jupyter_start><jupyter_text>Web Scraping<jupyter_code>#Importing the essential libraries
#Beautiful Soup is a Python library for pulling data out of HTML and XML files
#The Natural Language Toolkit

import requests
import nltk
nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer 
from bs4 import BeautifulSoup
import numpy as np 
import pandas as pd 
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
import random
from wordcloud import WordCloud
from html.parser import HTMLParser

import bs4 as bs
import urllib.request
import re
import string
#we are using request package to make a GET request for the website, which means we're getting data from it.
r=requests.get('http://gflenv.com/about-us/')
#Setting the correct text encoding of the HTML page
r.encoding = 'utf-8'
#Extracting the HTML from the request object
html = r.text
# Printing the first 500 characters in html
print(html[:500])
# Creating a BeautifulSoup object from the HTML
soup = BeautifulSoup(html)
# Getting the text out of the soup
text = soup.get_text()
#total length
len(text)
text=text[5000:10000]
text_nopunct=''

text_nopunct= "".join([char for char in text if char not in string.punctuation])
len(text_nopunct)
text_nopunct[2375:3980]
text_nopunct=text_nopunct.strip('\n')
text_nopunct=text_nopunct.strip('\n\n')
text_nopunct=text_nopunct.strip('\n\n\n')
text_nopunct[2375:3980]
#Creating the tokenizer
tokenizer = nltk.tokenize.RegexpTokenizer('\w+')
#Tokenizing the text
tokens = tokenizer.tokenize(text_nopunct)
len(tokens)
print(tokens[242:262])
#now we shall make everything lowercase for uniformity
#to hold the new lower case words

words = []

# Looping through the tokens and make them lower case
for word in tokens:
    words.append(word.lower())
print(words[242:304])
#Stop words are generally the most common words in a language.
#English stop words from nltk.

stopwords = nltk.corpus.stopwords.words('english')
words_new = []

#Now we need to remove the stop words from the words variable
#Appending to words_new all words that are in words but not in sw

for word in words:
    if word not in stopwords:
        words_new.append(word)
len(words_new)<jupyter_output><empty_output><jupyter_text>LemmatizationLemmatization is the algorithmic process of finding the lemma of a word depending on their meaning. Lemmatization usually refers to the morphological analysis of words, which aims to remove inflectional endings.<jupyter_code>from nltk.stem import WordNetLemmatizer 
wn = WordNetLemmatizer()
lem_words=[]

for word in words_new:
    word=wn.lemmatize(word)
    lem_words.append(word)
len(lem_words)
same=0
diff=0

for i in range(0,416):
    if(lem_words[i]==words_new[i]):
        same=same+1
    elif(lem_words[i]!=words_new[i]):
        diff=diff+1
print('Number of words Lemmatized=', diff)
print('Number of words not Lemmatized=', same)
#The frequency distribution of the words
freq_dist = nltk.FreqDist(lem_words)
#Frequency Distribution Plot
plt.subplots(figsize=(20,12))
freq_dist.plot(30)
#converting into string
res=' '.join([i for i in lem_words if not i.isdigit()])
plt.subplots(figsize=(16,10))
wordcloud = WordCloud(
                          background_color='black',
                          max_words=100,
                          width=1400,
                          height=1200
                         ).generate(res)


plt.imshow(wordcloud)
plt.title('GFL Environmental (100 words)')
plt.axis('off')
plt.show()
plt.subplots(figsize=(16,10))
wordcloud = WordCloud(
                          background_color='black',
                          max_words=200,
                          width=1400,
                          height=1200
                         ).generate(res)


plt.imshow(wordcloud)
plt.title('GFL Environmental (200 words)')
plt.axis('off')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09116445352400408, 'fraction_numerical': 0.024514811031664963, 'mean_word_length': 3.9457070707070705, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '668529', 'license': 'MIT', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1216, 'n_words': 385, 'path': 'GFL Environmental Text Analysis.ipynb', 'repo_name': 'SayantiDutta2000/Financial-Analysis', 'size': '3916'}