<jupyter_start><jupyter_text>Finding out the distinct categories for a particular source of newsHere we are trying to find out the naming patterns of every web source we have scraped.In __if(i[1]=='livemint'):__ the 'livemint' will be replaced by the websource name.<jupyter_code>import ast
import sys
import csv
maxInt = sys.maxsize
decrement = True

while decrement:
    decrement = False
    try:
        csv.field_size_limit(maxInt)
    except OverflowError:
        maxInt = int(maxInt/10)
        decrement = True
file = open(r'AllNews.csv','r',encoding='ISO-8859-1')
read = csv.reader(file)
master = set([])
next(read,None)
for i in read:
    if(i[1]=='livemint'):
        s1=i[2]

        if('[' in s1 and ']' in s1):
            s = ast.literal_eval(s1)
            master = master | set(s)
        else:
            master.add(s1)
file.close()
for z in master:
    print(z)
for j in master:
    if '/' in j:
        j=j.replace("/","")<jupyter_output>Consumer
Education
Money
Opinion
Industry
Companies
Leisure
Politics
Sports
Science
Home Page
Technology
{'fraction_non_alphanumeric': 0.1107981220657277, 'fraction_numerical': 0.014084507042253521, 'mean_word_length': 3.1640625, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '114775', 'license': 'MIT', 'n_tokens_mistral': 383, 'n_tokens_neox': 358, 'n_words': 110, 'path': 'Filtering the News/Clustering/DistinctCategories.ipynb', 'repo_name': 'MuditrRathore/Stock-Prediction', 'size': '1065'}