<commit_before>#!/usr/bin/python
import csv
from dbfpy import dbf
import os
import sys

for filename in sys.argv[1:]:
    if filename.endswith('.dbf'):
        print "Converting {} to csv".format(filename)
        csv_fn = filename[:-4]+ ".csv"
        with open(csv_fn, 'wb') as csvfile:
            in_db = dbf.Dbf(filename)
            out_csv = csv.writer(csvfile)
            names = []
            for field in in_db.header.fields:
                names.append(field.name)
            out_csv.writerow(names)
            for rec in in_db:
                out_csv.writerow(rec.fieldData)
            in_db.close()
            print "Done..."
    else:
      print "Filename does not end with .dbf"<commit_msg>clean up on covnertions file<commit_after>#!/usr/bin/python
import csv
from dbfpy import dbf
import os
import sys

for filename in sys.argv[1:]:
    if filename.endswith('.dbf'):
        print "Converting {} to csv".format(filename)
        csv_fn = filename[:-4]+ ".csv"
        with open(csv_fn, 'wb') as csvfile:
            in_dbf = dbf.Dbf(filename)
            out_csv = csv.writer(csvfile)
            names = []

            for field in in_db.header.fields:
                names.append(field.name)
            out_csv.writerow(names)

            for rec in in_dbf:
                out_csv.writerow(rec.fieldData)
            in_dbf.close()
            print "Completed"
    else:
      print "invalid filetype"
{'fraction_non_alphanumeric': 0.10871080139372823, 'fraction_numerical': 0.0027874564459930314, 'mean_word_length': 1.8662674650698603, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1251240', 'n_tokens_mistral': 466, 'n_tokens_neox': 443, 'n_words': 111}