<commit_before> #!/usr/bin/python
# _*_ coding: utf-8

import xlrd
import json
import sys
import codecs

def _clean( value , empty):
    try:
        return int( value )
    except:
        value = value.encode('utf-8')
        if value in empty:
            return None
        return value

input = xlrd.open_workbook( sys.argv[1] ).sheets()[0]

data = []

for row in range( 1 , input.nrows ):
    candidate = {}

    for column in range( input.ncols ):
        header = input.cell( 0, column ).value.encode('utf8')

        if not header[0] == 'q':
            candidate[ header ] = _clean( input.cell( row, column ).value, ['', '-', 'NULL'] )

    data.append( candidate )

json.dump( data, codecs.open( sys.argv[1] + '.json', 'w' ), ensure_ascii = False)<commit_msg>Twitter cleaning<commit_after> #!/usr/bin/python
# _*_ coding: utf-8

import xlrd
import json
import sys
import codecs
import re

def _clean( value , empty):
    try:
        return int( value )
    except:
        value = value.encode('utf-8')
        if value in empty:
            return None
        return value

_clean_twitter = re.compile('((http)?[s]?(://)?(www.)?twitter(.com)?/|@)' , re.IGNORECASE )
_clean_slash = re.compile('/')

input = xlrd.open_workbook( sys.argv[1] ).sheets()[0]

data = []

for row in range( 1 , input.nrows ):
    candidate = {}

    for column in range( input.ncols ):
        header = input.cell( 0, column ).value.encode('utf8')

        if not header[0] == 'q':
            candidate[ header ] = _clean( input.cell( row, column ).value, ['', '-', 'NULL'] )

    ## clean urls from data
    if candidate[ 'twitter' ]:
        candidate[ 'twitter' ] = _clean_twitter.sub( '', candidate[ 'twitter' ] )
        candidate[ 'twitter' ] = _clean_slash.sub( '', candidate['twitter'] )

        ## urgh urgh
        if candidate[ 'twitter' ] == '' or 'http' in candidate['twitter'] or 'www.' in candidate['twitter']:
            candidate[ 'twitter' ] = None

    data.append( candidate )

json.dump( data, codecs.open( sys.argv[1] + '.json', 'w' ), ensure_ascii = False)
{'fraction_non_alphanumeric': 0.1640926640926641, 'fraction_numerical': 0.008687258687258687, 'mean_word_length': 2.630472854640981, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4003645', 'n_tokens_mistral': 689, 'n_tokens_neox': 681, 'n_words': 189}