<commit_before>import sys
import glob
import json
import os.path
import argparse
import urllib2
import shutil
import tempfile
from datetime import datetime
from subprocess import call
from StringIO import StringIO
from zipfile import ZipFile
from find_package import find_github_repo, find_matlabcentral_repo

HOMEDIR = os.path.expanduser("~")
MATLABDIR = os.path.join(HOMEDIR, 'Documents', 'MATLAB')

def copytree(src, dst, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def readzip(url):
    try:
        urlc = urllib2.urlopen(url)
        return ZipFile(StringIO(urlc.read()))
    except Exception, e:
        # print url, e
        return
    # except urllib2.HTTPError, e:
    #     print e.code
    #     return
    # except urllib2.URLError, e:
    #     print e.args
    #     return

def write_to_mpmfile(outdir, data, filename='mpm.json'):
    mpmfile = os.path.join(outdir, filename)
    if os.path.exists(mpmfile):
        with open(mpmfile) as f:
            objs = json.load(f)
    else:
        objs = []
    cur_ind = [i for i,x in enumerate(objs) if x['name'] == data['name']]
    if cur_ind:
        print 'Overwriting entry for "{0}" in mpmfile'.format(data['name'])
        objs[cur_ind[0]] = data
    else:
        objs.append(data)
    with open(mpmfile, 'w') as f:
        json.dump(objs, f, sort_keys=False, indent=4, separators=(',', ': '))

M_DIR_ORDER = ['bin', 'src', 'lib', 'code']
def find_mfile_dir(indir, internaldir=None, dirs_to_check=M_DIR_ORDER):
    mfls = glob.glob(os.path.join(indir, '*.m'))
    if mfls: # .m files in main dir -- all is well
        if internaldir is not None:
            print 'WARNING: Ignoring internaldir "{0}"" because .m files were found in the base directory'.format(internaldir)
        return indir
    if internaldir is not None:
        cdir = os.path.join(indir, internaldir)
        if not os.path.exists(cdir):
            print 'WARNING: Ignoring internaldir "{0}"" because it did not exist.'.format(internaldir)
        else:
            dirs_to_check = [internaldir]<commit_msg>counts folders starting with + as valid mfile dir<commit_after>import sys
import glob
import json
import os.path
import argparse
import urllib2
import shutil
import tempfile
from datetime import datetime
from subprocess import call
from StringIO import StringIO
from zipfile import ZipFile
from find_package import find_github_repo, find_matlabcentral_repo

HOMEDIR = os.path.expanduser("~")
MATLABDIR = os.path.join(HOMEDIR, 'Documents', 'MATLAB')

def copytree(src, dst, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def readzip(url):
    try:
        urlc = urllib2.urlopen(url)
        return ZipFile(StringIO(urlc.read()))
    except Exception, e:
        # print url, e
        return
    # except urllib2.HTTPError, e:
    #     print e.code
    #     return
    # except urllib2.URLError, e:
    #     print e.args
    #     return

def write_to_mpmfile(outdir, data, filename='mpm.json'):
    mpmfile = os.path.join(outdir, filename)
    if os.path.exists(mpmfile):
        with open(mpmfile) as f:
            objs = json.load(f)
    else:
        objs = []
    cur_ind = [i for i,x in enumerate(objs) if x['name'] == data['name']]
    if cur_ind:
        print 'Overwriting entry for "{0}" in mpmfile'.format(data['name'])
        objs[cur_ind[0]] = data
    else:
        objs.append(data)
    with open(mpmfile, 'w') as f:
        json.dump(objs, f, sort_keys=False, indent=4, separators=(',', ': '))

M_DIR_ORDER = ['bin', 'src', 'lib', 'code']
def find_mfile_dir(indir, internaldir=None, dirs_to_check=M_DIR_ORDER):
    mfls = glob.glob(os.path.join(indir, '*.m'))
    mfls += glob.glob(os.path.join(indir, '+*'))
    if mfls: # .m files in main dir -- all is well
        if internaldir is not None:
            print 'WARNING: Ignoring internaldir "{0}"" because .m files were found in the base directory'.format(internaldir)
        return indir
    if internaldir is not None:
        cdir = os.path.join(indir, internaldir)
        if not os.path.exists(cdir):
            print 'WARNING: Ignoring internaldir "{0}"" because it did not exist.'.format(internaldir)
        else:
            dirs_to_check = [internaldir]
{'fraction_non_alphanumeric': 0.12086486486486486, 'fraction_numerical': 0.004324324324324324, 'mean_word_length': 2.7918032786885245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5223725', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1583, 'n_words': 426}