<jupyter_start><jupyter_text><jupyter_code>!pip3 uninstall twint
!pip3 install --user --upgrade git+https://github.com/twintproject/twint.git@origin/master #egg=twint
from datetime import timedelta
from string import ascii_letters, digits
from os import mkdir, path
import nest_asyncio
import pandas as pd
import twint

nest_asyncio.apply()

def clean_name(dirname):
    
    valid = set(ascii_letters + digits)
    return ''.join(a for a in dirname if a in valid)


def twint_search(searchterm, since, until, json_name):

    c = twint.Config()
    c.Search = searchterm
    c.Since = since
    c.Until = until
    c.Limit = 100
    c.Lang = "en"
    c.Hide_output = True
    c.Store_json = True
    c.Output = json_name
    c.Debug = True

    try:
        twint.run.Search(c)    
        
    except (KeyboardInterrupt, SystemExit):
        raise

        
#     except:
#         print("Problem with %s." % since)


def twint_loop(searchterm, since, until):

    dirname = clean_name(searchterm)
    
    try:
        mkdir(dirname)
        print("Directory" , dirname ,  "Created ")
        
    except FileExistsError:
        print("Directory" , dirname ,  "already exists")

    daterange = pd.date_range(since, until)

    for start_date in daterange:

        since= start_date.strftime("%Y-%m-%d")
        until = (start_date + timedelta(days=1)).strftime("%Y-%m-%d")

        json_name = '%s.json' % since
        json_name = path.join(dirname, json_name)

        print('Getting %s ' % since )
        twint_search(searchterm, since, until, json_name)
twint_loop('discrimination OR gender OR gbv OR violence', '04-01-2021', '04-30-2021')
from glob import glob

file_names = glob(path.join('discriminationORgender','*.json'))
dfs = [pd.read_json(fn, lines = True) for fn in file_names]
gender_df = pd.concat(dfs)
gender_df.head(2)
gender_df.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1170550847457627, 'fraction_numerical': 0.012182203389830509, 'mean_word_length': 3.0536480686695278, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '541265', 'license': 'MIT', 'n_tokens_mistral': 674, 'n_tokens_neox': 628, 'n_words': 152, 'path': 'Twitter_Scrapping_Using_Twint.ipynb', 'repo_name': 'Paul-mwaura/Natural-Language-Processing', 'size': '1888'}