<commit_before><commit_msg>convert lcc npy to mat<commit_after>import scipy.io as sio
import numpy as np
import os
import argparse

def convertLCCtoMat(lcc_fn, toDir):
  '''
  Convert a npy largest connected components file to an equivalent .mat file
  lcc_fn - largest connected components full file name which should be a .npy
  '''
  lcc = np.load(lcc_fn).item().toarray()
  mat_fn = os.path.join(toDir, os.path.splitext(lcc_fn.split('/')[-1])[0])
  sio.savemat(os.path.splitext(mat_fn)[0],{'lcc': lcc}, appendmat = True)
  print ('%s sucessfully saved as mat') % os.path.splitext(lcc_fn.split('/')[-1])[0]

def main():
    
    parser = argparse.ArgumentParser(description='Convert directory of derivatives to .mat format')
    parser.add_argument('fromDir', action='store',help='Full path of directory containing .npy files to be converted')
    parser.add_argument('toDir', action='store', help='Full path of directory where you want .mat file to go')
    
    result = parser.parse_args()
    
    if not os.path.exists(result.toDir):
      os.makedirs(result.toDir)
    
    for theFile in os.listdir(result.fromDir):
      if not (result.fromDir.startswith('.')):
        convertLCCtoMat(os.path.join(result.fromDir, theFile), result.toDir)

if __name__ == '__main__':
  main()
  
  
{'fraction_non_alphanumeric': 0.1346749226006192, 'fraction_numerical': 0.003869969040247678, 'mean_word_length': 4.031128404669261, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6596678', 'n_tokens_mistral': 440, 'n_tokens_neox': 438, 'n_words': 113}