<jupyter_start><jupyter_text>Search Query In Twitter Collection-------  Import Libraries<jupyter_code>from pymongo import MongoClient
import pymongo
import json
from bson import json_util
import csv
import os
import logging
import time
import pandas as pd
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)<jupyter_output><empty_output><jupyter_text>Read Config File<jupyter_code>import configparser 
config = configparser.ConfigParser()
config.read('config.ini')
ip = config['DEFAULT']['IP']
port = config['DEFAULT']['MongoDB-Port']
db_name = config['DEFAULT']['DB-Name']
user_name = config['DEFAULT']['User-Name']
psword = config['DEFAULT']['Psword']
debug = config['DEFAULT']['Debug']
query_input = config['DEFAULT']['Query-Input']<jupyter_output><empty_output><jupyter_text>Setup Log File<jupyter_code>logging.basicConfig(filename='debug.log',level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(message)s')<jupyter_output><empty_output><jupyter_text>Read Query List<jupyter_code>query_pd = pd.read_csv(query_input, header = None, encoding= "UTF-8")
query_list = query_pd[0].tolist()<jupyter_output><empty_output><jupyter_text>Connect MongoDB<jupyter_code>client = MongoClient(ip, int(port), username=user_name, password=<PASSWORD>)<jupyter_output><empty_output><jupyter_text>Get Collection Names<jupyter_code>db_twitter = client[db_name]
collections_twitter = db_twitter.collection_names()<jupyter_output><empty_output><jupyter_text>Supporting Functions<jupyter_code># create folder if not exist
def create_folder(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)
    return folder
# export to csv
def write_csv(output_file,row_list,func):
    with open(output_file, func) as f:
            writer = csv.writer(f, quoting=csv.QUOTE_MINIMAL)
            writer.writerow(row_list)<jupyter_output><empty_output><jupyter_text>Create Result Folders<jupyter_code>create_folder("csv_result/")
create_folder("json_result/")<jupyter_output><empty_output><jupyter_text>Write Into CSV &JSON<jupyter_code>start= time.time()
for c in sorted(collections_twitter):
    create_folder("json_result/{}/".format(c))
    
    for q in query_list:
        if debug:
            logging.info("---------------------------------------")
            logging.info("Finding query '{}' in collection {}...".format(q,c))

        count = 0
        create_folder("json_result/{}/{}".format(c,q))
        write_csv("csv_result/{}_{}.csv".format(c,q),["tweet_id","created_at","text","user_screen_name","user_location"],"w")
        
        cursor = db_twitter[c].find({"$text":{"$search": q}})
        
        text = created_at = screen_name = location = ""
        for data in cursor:
            # export selected fields to csv
            tweet_id = data["id_str"]
            text = data["text"]
            if "created_at" in data:
                created_at = data["created_at"]
            if "user" in data:
                if "screen_name" in data["user"]:
                    screen_name = data["user"]["screen_name"]
                if "location" in data["user"]:
                    location = data["user"]["location"]
            write_csv("csv_result/{}_{}.csv".format(c,q),[tweet_id,created_at,text,screen_name,location],"a+")
            
            # export each record as json
            with open('json_result/{}/{}/{}.json'.format(c,q,tweet_id.replace(" ","_")),'w') as f:
                del data["_id"]
                json.dump(json.loads(json_util.dumps(data)), f)

            count += 1
        if debug:
            logging.info("The program has been running for " + str(time.time()-start) + " seconds")
            logging.info("Found {} records for query '{}'".format(count,q))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15460090161760806, 'fraction_numerical': 0.0010607265977194379, 'mean_word_length': 3.823529411764706, 'pattern_counts': {'":': 2, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '354749', 'license': 'Unlicense', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1189, 'n_words': 208, 'path': 'f0028/f0028.ipynb', 'repo_name': 'soda-lab/Features', 'size': '3767'}