<commit_before>import numpy as _np

def input_check_Nx1(x):
    x = _np.atleast_1d(x)
    theSize = _np.shape(x)

    if(len(theSize)>1):
        #1. Input must be of size N x 1
        if ((theSize[0]!=1) & (theSize[1]!=1)):
            raise ValueError('Not an N x 1 array')
        #2. Make it into a 1-D array
        x = x.reshape(_np.size(x))
    elif (theSize[0]==1):
        x = x[0]
    
    return x,_np.size(x)

def input_check_Nx3(x):
    x = _np.atleast_2d(x)
    theSize = _np.shape(x)
    
    if(len(theSize)>1):
        #1. Input must be of size N x 3
        if ((theSize[0]!=3) & (theSize[1]!=3)):
            raise ValueError('Not a N x 3 array')
        #2. Make it into a Nx3 array
        if (theSize[1]!=3):
            x = x.T
        N = x.shape[0]
        #3. If N == 1, make it into a 1-D array
        if (x.shape[0]==1):
            x = x.reshape(x.shape[1])

    return x,N<commit_msg>Migrated functions from Hamid's misc_utilities.<commit_after>"""
Utilities Functions.

Copyright (c) 2014 NavPy Developers. All rights reserved.
Use of this source code is governed by a BSD-style license that can be found in
LICENSE.txt
"""

import numpy as _np

def input_check_Nx1(x):
    """
    Check x to be of dimension Nx1 and reshape it as a 1-D array
    
    <NAME>
    """
    x = _np.atleast_1d(x)
    theSize = _np.shape(x)

    if(len(theSize)>1):
        #1. Input must be of size N x 1
        if ((theSize[0]!=1) & (theSize[1]!=1)):
            raise ValueError('Not an N x 1 array')
        #2. Make it into a 1-D array
        x = x.reshape(_np.size(x))
    elif (theSize[0]==1):
        x = x[0]
    
    return x,_np.size(x)

def input_check_Nx3(x):
    """
    Check x to be of dimension Nx3
    
    <NAME>
    """
    x = _np.atleast_2d(x)
    theSize = _np.shape(x)
    
    if(len(theSize)>1):
        #1. Input must be of size N x 3
        if ((theSize[0]!=3) & (theSize[1]!=3)):
            raise ValueError('Not a N x 3 array')
        #2. Make it into a Nx3 array
        if (theSize[1]!=3):
            x = x.T
        N = x.shape[0]
        #3. If N == 1, make it into a 1-D array
        if (x.shape[0]==1):
            x = x.reshape(x.shape[1])

    return x,N

def loadtxt2dic(filename):
    """
    Loads text file of key:value pairs into a dictionary.  
    Usage notes:
    -Lines begining with '#' are treated as comments and skipped.  
    -Blank lines are also skipped
    -Keys and values should be separated by '=' or ':', extra spaces are fine.
    -A matrix/scalar are stored floats ONLY if the text has a decimal
    
    <NAME>. (original)
    <NAME> 
    """
    fid = open(filename,'r')
    param = {}
    for line in fid:
        # Remove Extra Spaces
        line = line.strip()
        
        # Skip lines beginning with # or blank
        # Note: Python treats '' as False
        if(line.startswith('#') or line.startswith('\n') or (not line)):
            continue
        
        # Split item
        item = line.split('#',1)[0].strip() # May have comment after the line
        item = item.replace(':',' ').replace('=',' ').split(None,1)
        
        if(len(item)==0):
            continue
            
        try:
            param[item[0]] = eval(item[1])
            
            if( type(eval(item[1]))==list ):
                param[item[0]] = _np.array(eval(item[1]))
        
        except NameError:
            param[item[0]] = item[1]
    
    fid.close()
    return param

def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
    """
    Prompt user for for 'yes' or 'no' response
    Taken from Python Documentation with small modifications
    http://docs.python.org/tutorial/controlflow.html
    
    Example:
    >>> ask_ok('Do you really want to quit?')
    
    <NAME> 2012
    """
    while True:
    
        ok = raw_input(prompt).lower()
        
        if ok in ('y', 'ye', 'yes','1'):
            return True
        if ok in ('n', 'no', 'nop', 'nope','0'):
            return False
        retries = retries - 1
        if retries < 0:
            raise IOError('refusenik user')
        print complaint

def status_update(i,drl,message,bar_length=20):
    """
    To create progress bar + short error message
    
    Parameters
    ----------
    i is the counter of where you are in the "count"
    drl is the total data record length
    message is your message line (keep it short!)
    
    <NAME>
    """
    percent = float(i) / drl
    hashes = '#' * int(round(percent * bar_length))
    spaces = ' ' * (bar_length - len(hashes))
    
    mmsg = message
    if(len(mmsg)>0):
        if(mmsg[-1]!='\n'):
            mmsg = mmsg + '\n'
    
    sys.stdout.write("\r[%s] %3d%% :: %s" % (hashes + spaces, int(round(percent * 100)),mmsg))
    sys.stdout.flush()
{'fraction_non_alphanumeric': 0.14070666945431737, 'fraction_numerical': 0.022579970729667574, 'mean_word_length': 1.8976377952755905, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2380536', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1647, 'n_words': 493}