<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)<jupyter_output><empty_output><jupyter_text>Read Config File<jupyter_code>import configparser 
config = configparser.ConfigParser()
config.read('config.ini')
ip = config['DEFAULT']['IP']
port = config['DEFAULT']['MongoDB-Port']
contain_string = config['DEFAULT']['Contain-String']
create_index = config['DEFAULT']['Create-Index']
db_name = config['DEFAULT']['DB-Name']<jupyter_output><empty_output><jupyter_text>Connect MongoDB<jupyter_code>import pymongo
from pymongo import MongoClient
client = MongoClient(ip, int(port))<jupyter_output><empty_output><jupyter_text>Get Collection Names<jupyter_code>db_twitter = client[db_name]
collections_twitter = db_twitter.collection_names()
dic_collection = {}
for i in collections_twitter:
    if i.startswith("20") and contain_string in i:
        dic_collection[i] = "{:,}".format(db_twitter[i].find({}).count())

for key in sorted(dic_collection):
    print("%s: %s" % (key, dic_collection[key]))<jupyter_output>2018_W51_Twitter_Australia: 20,471
2018_W51_Twitter_Other: 8,978,243
2018_W52_Twitter_Australia: 38,065
2018_W52_Twitter_Other: 16,940,708
2019_W10_Twitter_Australia: 40,767
2019_W10_Twitter_Other: 15,941,350
2019_W11_Twitter_Australia: 30,436
2019_W11_Twitter_Other: 11,669,385
2019_W1_Twitter_Australia: 40,880
2019_W1_Twitter_Other: 16,994,959
2019_W2_Twitter_Australia: 37,645
2019_W2_Twitter_Other: 14,474,627
2019_W3_Twitter_Australia: 52,348
2019_W3_Twitter_Other: 20,416,992
2019_W4_Twitter_Australia: 59,625
2019_W4_Twitter_Other: 23,417,927
2019_W5_Twitter_Australia: 61,617
2019_W5_Twitter_Other: 23,917,345
2019_W6_Twitter_Australia: 61,435
2019_W6_Twitter_Other: 24,176,122
2019_W7_Twitter_Australia: 28,017
2019_W7_Twitter_Other: 10,484,648
2019_W8_Twitter_Australia: 60,035
2019_W8_Twitter_Other: 23,446,630
2019_W9_Twitter_Australia: 58,316
2019_W9_Twitter_Other: 22,293,795<jupyter_text>Create Index<jupyter_code>for collection in sorted(dic_collection):
    exist = 0
    # get information of index of each collection
    index_information = db_twitter[collection].index_information()
    # check if collection has "text_text" index
    for index in index_information:
#         if index == "text_text":
        if index == "combined_text_index":
            exist = 1
            print("--------------------")
            print("Text index exists in " + collection)
            print("--------------------")
    # if "text_text" index not exist then create one
    if exist == 0:
        print("--------------------")
        print("No text index exists in " + collection)
        # create index
        if create_index == "1":
            print("creating text index....")
            db_twitter[collection].create_index([("text", pymongo.TEXT),("extended_tweet.full_text", pymongo.TEXT),("retweeted_status.text", pymongo.TEXT),("retweeted_status.extended_tweet.full_text", pymongo.TEXT)],name="combined_text_index")
            print("Text index for " + collection + " is done.")
        # It only lists the collections which do not have index
        elif create_index == "2":
            pass
        print("--------------------")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1349206349206349, 'fraction_numerical': 0.09554334554334554, 'mean_word_length': 4.893884892086331, 'pattern_counts': {'":': 4, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '616664', 'license': 'Unlicense', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1076, 'n_words': 199, 'path': 'f0002/f0002.ipynb', 'repo_name': 'soda-lab/Features', 'size': '3276'}