<jupyter_start><jupyter_text>Lab 2 Question 1Collect any 10 documents (English text documents) from the web and create inverted index bydoing necessary preprocessing steps using python.<jupyter_code># Import Statements

import os
import nltk


import requests
from bs4 import BeautifulSoup
from bs4.element import Comment
import re
import string
import nltk
from nltk.stem.porter import PorterStemmer

index = {}

def visible_text(element):
  if element.parent.name in ['style', 'title', 'script', 'head', '[document]', 'class', 'a', 'li']:
    return False
  elif isinstance(element, Comment):
    return False
  elif re.match(r"[\s\r\n]+",str(element)): 
    return False
  return True

def read_url(url, url_number):

  ps = PorterStemmer()

  r = requests.get(url)
  soup = BeautifulSoup(r.content, 'html.parser')
  text = soup.findAll(text = True)
  result = list(filter(visible_text, text))
  counter = 0;
  words = []
  for i in result:
    temp = i.split(' ')
    for word in temp:
      k = []
      temp_word = word.lower()
      for c in temp_word:
        if c not in list(string.punctuation):
          k.append(c)
      temp_word = ''.join(k)
      words.append(temp_word)
  for i in words:
    if(i.isalpha()):
      i = ps.stem(i)
      if not i in index.keys():
        index[i] = [(url_number, counter)]
        counter = counter + len(i) + 1
      else:
        index[i].append((url_number, counter))
        counter = counter + len(i) + 1

  return None

urls = [
    'https://en.wikipedia.org/wiki/Google',
    'https://en.wikipedia.org/wiki/Facebook',
    'https://en.wikipedia.org/wiki/Netflix',
    'https://en.wikipedia.org/wiki/Amazon_(company)',
    'https://en.wikipedia.org/wiki/Microsoft',
    'https://en.wikipedia.org/wiki/Tesla,_Inc.',
    'https://en.wikipedia.org/wiki/Apple_Inc.',
    'https://en.wikipedia.org/wiki/Silicon_Valley_(TV_series)',
    'https://en.wikipedia.org/wiki/Wikipedia',
    'https://en.wikipedia.org/wiki/Uber',
]
for i in range(len(urls)) :
    read_url(urls[i], (i+1))


sorted_keys = sorted(index.keys())

f = open("output2.txt", "w")
output_line = "Word".ljust(15) + "Frequency".ljust(15) + "Posting List".ljust(15) + "\n"
f.writelines(output_line)
f.writelines('-------------------------------------------------------------------------\n\n')
for i in sorted_keys:
  print(i, len(index[i]), index[i])
  output_string = str(i).ljust(15) + str(len(index[i])).ljust(15) + str(index[i]).ljust(15) + "\n"
  f.writelines(output_string)
  f.writelines('\n')
  
f.close()<jupyter_output>a 1373 [(1, 490), (1, 729), (1, 796), (1, 1137), (1, 1974), (1, 2112), (1, 2182), (1, 2297), (1, 2392), (1, 2439), (1, 2537), (1, 3040), (1, 3084), (1, 3620), (1, 3744), (1, 3765), (1, 3777), (1, 3805), (1, 3869), (1, 4179), (1, 4397), (1, 5106), (1, 5274), (1, 5720), (1, 6597), (1, 6967), (1, 7135), (1, 7272), (1, 7336), (1, 7428), (1, 7490), (1, 7590), (1, 7672), (1, 7870), (1, 8077), (1, 8218), (1, 8270), (1, 8509), (1, 9062), (1, 9188), (1, 9256), (1, 9286), (1, 9357), (1, 9463), (1, 9560), (1, 10275), (1, 10448), (1, 10628), (1, 11319), (1, 11346), (1, 12510), (1, 12606), (1, 12859), (1, 13028), (1, 13452), (1, 13923), (1, 13981), (1, 14035), (1, 14161), (1, 14283), (1, 14387), (1, 14399), (1, 14524), (1, 14567), (1, 14731), (1, 14838), (1, 14973), (1, 15029), (1, 15145), (1, 15656), (1, 15825), (1, 16146), (1, 16193), (1, 16607), (1, 17108), (1, 17212), (1, 17319), (1, 18035), (1, 18159), (1, 18210), (1, 18367), (1, 19636), (1, 19817), (1, 20805), (1, 22601), (1, 23123), (1, 2485[...]
{'fraction_non_alphanumeric': 0.2326565143824027, 'fraction_numerical': 0.1410039481105471, 'mean_word_length': 3.3952912019826518, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '600184', 'license': 'CC0-1.0', 'n_tokens_mistral': 1738, 'n_tokens_neox': 1400, 'n_words': 369, 'path': 'Web_Mining_CSE3024/Inverted_Indexing_Lab_2/Inverted_Indexing.ipynb', 'repo_name': 'aadhityasw/VIT-Labs', 'size': '3546'}