<commit_before># -*- coding: UTF-8 -*-
__author__="<NAME>"
__license__="Simplified BSD"
import sys
import acscsv

class TwacsCSV(acscsv.AcsCSV):
    def __init__(self, delim, options_keypath, options_geo, options_user, options_rules, options_urls, options_lang, options_influence, options_struct):
        super(TwacsCSV, self).__init__(delim,options_keypath)
        self.options_geo = options_geo 
        self.options_user = options_user
        self.options_rules = options_rules
        self.options_urls = options_urls
        self.options_lang = options_lang
        self.options_influence = options_influence
        self.options_struct = options_struct
        
    def procRecordToList(self,d):
        record = []
        try:
            if "verb" in d:
                verb = d["verb"]
            else:
                msg = "Unidentified meta message"
                for mtype in ["error", "warning", "info"]:
                    if mtype in d:
                        if "message" in d[mtype]:
                            msg = d[mtype]["message"]
                        elif "message" in d:
                            msg = d["message"]
                        continue
                    mtype = "Unidentified"
                record.append('-'.join([acscsv.gnipRemove, mtype]))
                record.append(acscsv.gnipDateTime)
                record.append(msg)
                return record
            if verb == "delete":
                record.append(d["object"]["id"])
                record.append(acscsv.gnipDateTime)
                record.append('-'.join([acscsv.gnipRemove, verb]))
                return record
            elif verb == "scrub_geo":
                record.append(d["actor"]["id"])
                record.append(acscsv.gnipDateTime)
                record.append('-'.join([acscsv.gnipRemove, verb]))
                return record
            # always first 2 items
            record.append(d["id"])
            record.append(d["postedTime"])
            # add some handling so calling gnacs without a pub is still useful
            #   n.b.: -stocktwits is native, so no 'verb' to get here
            #         -put more-specific fields at the beginning to catch them 
            obj = d["object"]
            if d["id"].rfind("getglue") != -1 : # getglue 
                record.append(verb)     # 'body' is inconsistent in gg
            elif "foursquareCategories" in obj or "foursquareCheckinOffset" in obj:      # fsq
                record.append(str(obj["geo"]["coordinates"]))
            elif "wpBlogId" in obj:     # wp
                record.append(str(obj["wpBlogId"]))
            elif "tumblrType" in obj:   # tumblr
                record.append(obj["tumblrType"])
            elif "body" in d:           # tw, disqus, stocktw,  
                record.append(self.cleanField(d["body"]))       
            elif "link" in d:           # ng
                record.append(d["link"])
            else:                       # ? 
                record.append("None")
            #
            gnip = {}
            actor = {}
            if "gnip" in d:
                gnip = d["gnip"]
            if "actor" in d:    # no 'actor' in ng
                actor = d["actor"]
            #
            if self.options_urls:
                urls = "None"
                if "urls" in gnip:
                    urls = self.buildListString([ l["expanded_url"] for l in gnip["urls"]])
                record.append(urls)
                twitter_urls = "None"
                twitter_un_urls = "None"
                if "twitter_entities" in d:
                    if "urls" in d["twitter_entities"]:
                        if len(d["twitter_entities"]["urls"]) > 0:
                            tmp_url_list = []
                            tmp_un_url_list = []
                            for tmp_rec in d["twitter_entities"]["urls"]:
                                try:
                                    if "url" in tmp_rec and tmp_rec["url"] is not None:
                                        tmp_url_list.append(tmp_rec["url"])
                                    else:
                                        tmp_url_list.append("None")
                                except TypeError:
                                    tmp_url_list = ["twitter_entiteis:urls:url"]
                                try:
                                    if "expanded_url" in tmp_rec and tmp_rec["expanded_url"] is not None:
                                        tmp_un_url_list.append(tmp_rec["expanded_url"])
                                    else:
                                        tmp_un_url_list.append("None")
                                except TypeError:
                                    tmp_un_urls_list = ["twitter_entities:urls:expanded_url"]
                            twitter_urls = self.buildListString(tmp_url_list)
                            twitter_un_urls = self.buildListString(tmp_un_url_list)
                record.append(twitter_urls)
                record.append(twitter_un_urls)
            if self.options_lang:
                try:
                    record.append(str([str(l) for l in actor["languages"]]))
                except UnicodeEncodeError, e:
                    record.append(str("bad lang"))
                    raise UnicodeEncodeError("unicode encoding problem in actor:language")
                glang = "None"
                if "language" in gnip:
                    glang = gnip["language"]["value"]
                record.append(glang)
                tlang = "None"
                if "twitter_lang" in d:
                    tlang = d["twitter_lang"]
                record.append(tlang)
            if self.options_rules:
                rules = '[]'
                if "matching_rules" in gnip:
                    rules = self.buildListString([ "%s (%s)"%(l["value"], l["tag"]) for l in gnip["matching_rules"]])<commit_msg>make twacscsv deal well with poorly-formatted language lists<commit_after># -*- coding: UTF-8 -*-
__author__="<NAME>"
__license__="Simplified BSD"
import sys
import acscsv

class TwacsCSV(acscsv.AcsCSV):
    def __init__(self, delim, options_keypath, options_geo, options_user, options_rules, options_urls, options_lang, options_influence, options_struct):
        super(TwacsCSV, self).__init__(delim,options_keypath)
        self.options_geo = options_geo 
        self.options_user = options_user
        self.options_rules = options_rules
        self.options_urls = options_urls
        self.options_lang = options_lang
        self.options_influence = options_influence
        self.options_struct = options_struct
        
    def procRecordToList(self,d):
        record = []
        try:
            if "verb" in d:
                verb = d["verb"]
            else:
                msg = "Unidentified meta message"
                for mtype in ["error", "warning", "info"]:
                    if mtype in d:
                        if "message" in d[mtype]:
                            msg = d[mtype]["message"]
                        elif "message" in d:
                            msg = d["message"]
                        continue
                    mtype = "Unidentified"
                record.append('-'.join([acscsv.gnipRemove, mtype]))
                record.append(acscsv.gnipDateTime)
                record.append(msg)
                return record
            if verb == "delete":
                record.append(d["object"]["id"])
                record.append(acscsv.gnipDateTime)
                record.append('-'.join([acscsv.gnipRemove, verb]))
                return record
            elif verb == "scrub_geo":
                record.append(d["actor"]["id"])
                record.append(acscsv.gnipDateTime)
                record.append('-'.join([acscsv.gnipRemove, verb]))
                return record
            # always first 2 items
            record.append(d["id"])
            record.append(d["postedTime"])
            # add some handling so calling gnacs without a pub is still useful
            #   n.b.: -stocktwits is native, so no 'verb' to get here
            #         -put more-specific fields at the beginning to catch them 
            obj = d["object"]
            if d["id"].rfind("getglue") != -1 : # getglue 
                record.append(verb)     # 'body' is inconsistent in gg
            elif "foursquareCategories" in obj or "foursquareCheckinOffset" in obj:      # fsq
                record.append(str(obj["geo"]["coordinates"]))
            elif "wpBlogId" in obj:     # wp
                record.append(str(obj["wpBlogId"]))
            elif "tumblrType" in obj:   # tumblr
                record.append(obj["tumblrType"])
            elif "body" in d:           # tw, disqus, stocktw,  
                record.append(self.cleanField(d["body"]))       
            elif "link" in d:           # ng
                record.append(d["link"])
            else:                       # ? 
                record.append("None")
            #
            gnip = {}
            actor = {}
            if "gnip" in d:
                gnip = d["gnip"]
            if "actor" in d:    # no 'actor' in ng
                actor = d["actor"]
            #
            if self.options_urls:
                urls = "None"
                if "urls" in gnip:
                    urls = self.buildListString([ l["expanded_url"] for l in gnip["urls"]])
                record.append(urls)
                twitter_urls = "None"
                twitter_un_urls = "None"
                if "twitter_entities" in d:
                    if "urls" in d["twitter_entities"]:
                        if len(d["twitter_entities"]["urls"]) > 0:
                            tmp_url_list = []
                            tmp_un_url_list = []
                            for tmp_rec in d["twitter_entities"]["urls"]:
                                try:
                                    if "url" in tmp_rec and tmp_rec["url"] is not None:
                                        tmp_url_list.append(tmp_rec["url"])
                                    else:
                                        tmp_url_list.append("None")
                                except TypeError:
                                    tmp_url_list = ["twitter_entiteis:urls:url"]
                                try:
                                    if "expanded_url" in tmp_rec and tmp_rec["expanded_url"] is not None:
                                        tmp_un_url_list.append(tmp_rec["expanded_url"])
                                    else:
                                        tmp_un_url_list.append("None")
                                except TypeError:
                                    tmp_un_urls_list = ["twitter_entities:urls:expanded_url"]
                            twitter_urls = self.buildListString(tmp_url_list)
                            twitter_un_urls = self.buildListString(tmp_un_url_list)
                record.append(twitter_urls)
                record.append(twitter_un_urls)
            if self.options_lang:
                try:
                    record.append(str([str(l) for l in actor["languages"]]))
                except UnicodeEncodeError, e:
                    record.append(str("bad encoding"))
                glang = "None"
                if "language" in gnip:
                    glang = gnip["language"]["value"]
                record.append(glang)
                tlang = "None"
                if "twitter_lang" in d:
                    tlang = d["twitter_lang"]
                record.append(tlang)
            if self.options_rules:
                rules = '[]'
                if "matching_rules" in gnip:
                    rules = self.buildListString([ "%s (%s)"%(l["value"], l["tag"]) for l in gnip["matching_rules"]])
{'fraction_non_alphanumeric': 0.10013486176668915, 'fraction_numerical': 0.0006743088334457181, 'mean_word_length': 1.2518504460049344, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720912', 'n_tokens_mistral': 3202, 'n_tokens_neox': 3054, 'n_words': 676}