<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import os
import glob
import re
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)<jupyter_output><empty_output><jupyter_text>Read Config File<jupyter_code>import configparser 
config = configparser.ConfigParser()
config.read('config.ini')
ip = config['DEFAULT']['IP']
port = config['DEFAULT']['MongoDB-Port']
db_name = config['DEFAULT']['DB-Name']
contain_string = config['DEFAULT']['Contain-String']
output_path = config['DEFAULT']['Output-Path']<jupyter_output><empty_output><jupyter_text>Connect MongoDB<jupyter_code>from pymongo import MongoClient
client = MongoClient(ip, int(port))<jupyter_output><empty_output><jupyter_text>Get Collection Name<jupyter_code># connect to database
db_twitter = client[db_name]
collections_twitter = db_twitter.collection_names()
dic_collection = {}
for i in collections_twitter:
    if contain_string in i:
        dic_collection[i] = "{:}".format(db_twitter[i].find({}).count())

for key in sorted(dic_collection):
    print("%s: %s" % (key, dic_collection[key]))<jupyter_output>Twitter_2014: 3811
Twitter_2015: 19659
Twitter_2016: 25764
Twitter_2017: 25153
Twitter_2018: 25456
Twitter_2019: 7997<jupyter_text>Pipeline<jupyter_code># pipeline for aggregation
pipeline = [
    {"$match": { "entities.hashtags": {"$exists":True,"$ne":[]}}},
    {"$match": { "lang" : "en"}},
    { "$group": {
        "_id": {
            "hashtags": "$entities.hashtags",
            "date": {"$substr": [ "$created_at", 4, 6 ]}
        },
        "count": { "$sum": 1 },
        }
    }
]<jupyter_output><empty_output><jupyter_text>Supporting Functions<jupyter_code># check if stringis English
def isEnglish(s):
    try:
        s.encode('ascii')
    except UnicodeEncodeError:
        return False
    else:
        return True
# create foler if not exist
def create_folder(output_path):
    if not os.path.exists(output_path):
        os.makedirs(output_path)
# delete existed collection from the list dic_collection
def delete_collection(output_path,dic_collection):
    for input_file in glob.glob(os.path.join(output_path,'*.csv')):
        collection_name = re.search(output_path+'(.+?).csv', input_file).group(1)
        if collection_name in dic_collection:
            print("Existed collection: " + collection_name)
            del dic_collection[collection_name]
    return dic_collection
def check_exist(data_format,h,date_year,exist,dic,count):
    for d in data_format:
        if (h == d["hashtag"]) and (date_year == d["date"]):
            d["count"] += count
            exist = 1
    if exist == 0:
        data_format.append(dic)
    return data_format
# count hashtag daily for each collection
def count_hashtag(length,hashtags,date_year,count,data_format):
    for i in range(0,length-1):
        exist = 0
        # get hashtag
        h = hashtags[i]["text"].lower()
        # check if it is in English
        if isEnglish(h):
            dic = {"hashtag": h, "date":date_year, "count" : count}
            if len(data_format) > 0 :
                #check if the record exists
                data_format = check_exist(data_format,h,date_year,exist,dic,count)
            else:        
                data_format.append(dic)
        else:
            num_delete += 1
    return data_format, num_delete
# create data list
def create_list(data_list,data_format,year):
    num_delete = 0
    for data in data_list:
        hashtags = data["_id"]["hashtags"]
        date_year = data["_id"]["date"] + " " + year
        count = data["count"]
        length = len(hashtags)
        data_format,num_delete = count_hashtag(length,hashtags,date_year,count,data_format)
    return data_format,num_delete
# write csv file
def write_csv(collection,data_format,output_path):
    file_name = output_path + collection + ".csv"
    with open(file_name, 'w') as f:
        # header
        f.write('hashtag,date,count\n')
        for data in data_format:
            line = data['hashtag'] + ',' + data['date'] + ',' + str(data["count"]) + '\n'
            f.write(line)<jupyter_output><empty_output><jupyter_text>Count the number of hashtag daily<jupyter_code>#create folder if not exist
create_folder(output_path)

# delete existed collection from the list dic_collection
dic_collection = delete_collection(output_path,dic_collection)

for collection in sorted(dic_collection):
    print("-----------------------\n")
    print("Processing on collection: " + collection)
    
    data_format = []
    data_list = list(db_twitter[collection].aggregate(pipeline,allowDiskUse=True))
    year = collection[-4:]

    if len(data_list) > 0:
        data_format, num_delete = create_list(data_list,data_format,year)
                    
    print("hashtag and date list is finished")
    print(str(num_delete) + " non-English hashtags have been deleted.")
    
    write_csv(collection,data_format,output_path)

    print ("hashtag csv file for collection " + collection + ' is finished.')
    print("-----------------------\n")<jupyter_output>Existed collection: Twitter_2017
Existed collection: Twitter_2016
Existed collection: Twitter_2014
Existed collection: Twitter_2015
Existed collection: Twitter_2018
Existed collection: Twitter_2019
{'fraction_non_alphanumeric': 0.13035204567078973, 'fraction_numerical': 0.017126546146527116, 'mean_word_length': 3.688670829616414, 'pattern_counts': {'":': 14, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '553816', 'license': 'Unlicense', 'n_tokens_mistral': 1740, 'n_tokens_neox': 1648, 'n_words': 351, 'path': 'f0012/f0012-a.ipynb', 'repo_name': 'soda-lab/Features', 'size': '5255'}