<commit_before>import json
import os
import collections
import argparse
import datetime
import operator
from random import randint
from pprint import pprint
from gender_name import GenderName
from util import Util

current_path = os.path.dirname(__file__)
TWEETS_PATH = current_path + '/data/geolocated_asturias.json'
SAMPLE_PATH = current_path + '/data/sample.json'
MALE_TABLE_PATH = current_path + '/data/male_table.data'
FEMALE_TABLE_PATH = current_path + '/data/female_table.data'
gender_name = GenderName()
util = Util()


def classify_tweets(face_recognition=False, min_confidence=0.75):
    print('Classifying tweets by gender name...')
    male_full_user_tweets_dict = dict()
    female_full_user_tweets_dict = dict()
    with open(TWEETS_PATH) as file:
        content = file.readlines()
        count_feedback = 1  # Count to show feedback while writing
        # for _ in range(0, 1):
        for tweet in content:
            # tweet = content[randint(0, len(content)-1)]
            tweet_json = json.loads(tweet)
            screen_name = tweet_json['user']['screen_name']
            name = tweet_json['user']['name']
            res = gender_name.get_gender_by_fullname(name)
            if face_recognition:
                # lazy evaluation, if cannot get gender by name, try to get it by face recognition
                util.perform_face_recognition(res, tweet_json)
            # Already have all gender results, build table
            if res['gender'] != 'unknown':
                tweet_text = tweet_json['text']
                if res['gender'] == 'male' and float(res['confidence']) >= min_confidence:
                    if screen_name not in male_full_user_tweets_dict:
                        male_full_user_tweets_dict[screen_name] = {tweet_text}
                    else:
                        male_full_user_tweets_dict[screen_name].add(tweet_text)
                    count_feedback += 1
                elif res['gender'] == 'female' and float(res['confidence']) >= min_confidence:
                    if screen_name not in female_full_user_tweets_dict:
                        female_full_user_tweets_dict[screen_name] = {tweet_text}
                    else:
                        female_full_user_tweets_dict[screen_name].add(tweet_text)
                    count_feedback += 1
                if count_feedback % 10000 == 0:
                    print('\tWriting tweet line {} ...'.format(count_feedback))

                    # Get number of tweets of each user (just for testing!)
                    #_get_and_store_num_tweets(male_full_user_tweets_dict, female_full_user_tweets_dict)<commit_msg>Bug fix: Face recognition reference<commit_after>import json
import os
import collections
import argparse
import datetime
import operator
from random import randint
from pprint import pprint
from gender_name import GenderName
from util import Util

current_path = os.path.dirname(__file__)
TWEETS_PATH = current_path + '/data/geolocated_asturias.json'
SAMPLE_PATH = current_path + '/data/sample.json'
MALE_TABLE_PATH = current_path + '/data/male_table.data'
FEMALE_TABLE_PATH = current_path + '/data/female_table.data'
gender_name = GenderName()
util = Util()


def classify_tweets(face_recognition=False, min_confidence=0.75):
    print('Classifying tweets by gender name...')
    male_full_user_tweets_dict = dict()
    female_full_user_tweets_dict = dict()
    with open(TWEETS_PATH) as file:
        content = file.readlines()
        count_feedback = 1  # Count to show feedback while writing
        # for _ in range(0, 1):
        for tweet in content:
            # tweet = content[randint(0, len(content)-1)]
            tweet_json = json.loads(tweet)
            screen_name = tweet_json['user']['screen_name']
            name = tweet_json['user']['name']
            res = gender_name.get_gender_by_fullname(name)
            if face_recognition:
                # lazy evaluation, if cannot get gender by name, try to get it by face recognition
                _perform_face_recognition(res, tweet_json)
            # Already have all gender results, build table
            if res['gender'] != 'unknown':
                tweet_text = tweet_json['text']
                if res['gender'] == 'male' and float(res['confidence']) >= min_confidence:
                    if screen_name not in male_full_user_tweets_dict:
                        male_full_user_tweets_dict[screen_name] = {tweet_text}
                    else:
                        male_full_user_tweets_dict[screen_name].add(tweet_text)
                    count_feedback += 1
                elif res['gender'] == 'female' and float(res['confidence']) >= min_confidence:
                    if screen_name not in female_full_user_tweets_dict:
                        female_full_user_tweets_dict[screen_name] = {tweet_text}
                    else:
                        female_full_user_tweets_dict[screen_name].add(tweet_text)
                    count_feedback += 1
                if count_feedback % 10000 == 0:
                    print('\tWriting tweet line {} ...'.format(count_feedback))

                    # Get number of tweets of each user (just for testing!)
                    #_get_and_store_num_tweets(male_full_user_tweets_dict, female_full_user_tweets_dict)
{'fraction_non_alphanumeric': 0.0845602114005285, 'fraction_numerical': 0.00604001510003775, 'mean_word_length': 2.4498697916666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4009292', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1483, 'n_words': 357}