<commit_before>import csv
import json
from textblob import TextBlob
from textblob_fr import PatternTagger, PatternAnalyzer


def parse_tweets(infiles, tweetfile, tagfile, userfile, edgefile, kwrd):
    tweetfile = open(tweetfile, "w")
    tweet_writer = csv.writer(tweetfile, delimiter='\t')
    edgefile = open(edgefile, "w")
    edge_writer = csv.writer(edgefile, delimiter='\t')
    tagfile = open(tagfile, "w")
    tag_writer = csv.writer(tagfile, delimiter='\t')
    userfile = open(userfile, "w")
    user_writer = csv.writer(userfile, delimiter='\t')
    ht_dict = {}
    ht_id_counter = 1
    user_dict = {}
    tweet_dict = {}
    tweet_header = ["tid:ID", "lang", "name", "text", "clean_text", "polarity:float",
                    "subjectivity:float", "created_at", "full_name",
                    "country", "country_code", "coordinates", ":LABEL"]
    tweet_writer.writerow(tweet_header)
    edge_header = [":START_ID", ":END_ID", ":TYPE"]
    edge_writer.writerow(edge_header)
    tag_header = ["tagid:ID", "hashtag", ":LABEL"]
    tag_writer.writerow(tag_header)
    user_header = ["uid:ID", "screen_name", ":LABEL"]
    user_writer.writerow(user_header)
    for infile in infiles:
        with open(infile, "r") as f:
            # parse the tweet
            for line in f:
                tweet = json.loads(line)
                try:
                    text = tweet["text"].replace('"', "'").replace("\\", "")
                except KeyError:
                    continue
                if kwrd not in text.lower():
                    continue
                tid = str(tweet["id"])
                user_id = str(tweet["user"]["id"])
                screen_name = tweet["user"]["screen_name"]
                user_dict[user_id] = screen_name
                rt_id = ""
                country_code = ""
                country = ""
                full_name = ""
                name = ""<commit_msg>added hack to clean the art tweets mid parse<commit_after>import csv
import json
from textblob import TextBlob
from textblob_fr import PatternTagger, PatternAnalyzer


def parse_tweets(infiles, tweetfile, tagfile, userfile, edgefile, kwrd_set):
    tweetfile = open(tweetfile, "w")
    tweet_writer = csv.writer(tweetfile, delimiter='\t')
    edgefile = open(edgefile, "w")
    edge_writer = csv.writer(edgefile, delimiter='\t')
    tagfile = open(tagfile, "w")
    tag_writer = csv.writer(tagfile, delimiter='\t')
    userfile = open(userfile, "w")
    user_writer = csv.writer(userfile, delimiter='\t')
    ht_dict = {}
    ht_id_counter = 1
    user_dict = {}
    tweet_dict = {}
    tweet_header = ["tid:ID", "lang", "name", "text", "clean_text", "polarity:float",
                    "subjectivity:float", "created_at", "full_name",
                    "country", "country_code", "coordinates", ":LABEL"]
    tweet_writer.writerow(tweet_header)
    edge_header = [":START_ID", ":END_ID", ":TYPE"]
    edge_writer.writerow(edge_header)
    tag_header = ["tagid:ID", "hashtag", ":LABEL"]
    tag_writer.writerow(tag_header)
    user_header = ["uid:ID", "screen_name", ":LABEL"]
    user_writer.writerow(user_header)
    for infile in infiles:
        with open(infile, "r") as f:
            # parse the tweet
            for line in f:
                tweet = json.loads(line)
                try:
                    text = tweet["text"].replace('"', "'").replace("\\", "")
                except KeyError:
                    continue
                # Hmmm this is a hack for the art tweets...
                text_tokens = set(w.lstrip("#") for w in text.lower().split(' '))
                if not kwrd_set & text_tokens:
                    continue
                tid = str(tweet["id"])
                user_id = str(tweet["user"]["id"])
                screen_name = tweet["user"]["screen_name"]
                user_dict[user_id] = screen_name
                rt_id = ""
                country_code = ""
                country = ""
                full_name = ""
                name = ""
{'fraction_non_alphanumeric': 0.13781971691085176, 'fraction_numerical': 0.0004966476285075739, 'mean_word_length': 2.1716535433070865, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '179361', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1231, 'n_words': 268}