<commit_before>"""
get_data.py: functions to download data for myconnectome project from Stanford archive
Options:
    --all: get all basic data needed for analysis
    --rawfunc: get raw fMRI data
    --diffusion: get raw diffusion data
    --anatomy: get raw anatomical data
    -h: print this help message
"""

import os,getopt,sys
import urllib
import re
import datetime
import hashlib

def timestamp():
    return datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')

# from  http://stackoverflow.com/questions/3431825/generating-a-md5-checksum-of-a-file
def hashfile(fname, blocksize=65536):
    afile=open(fname,'rb')
    buf = afile.read(blocksize)
    hasher=hashlib.md5()
    while len(buf) > 0:
        hasher.update(buf)
        buf = afile.read(blocksize)
    return hasher.hexdigest()

   
basedir=os.environ['MYCONNECTOME_DIR']

dataurl='http://web.stanford.edu/group/poldracklab/myconnectome-data/'


# my kludgey code requires the base dir to end in a slash
if not basedir[-1]=='/':
    basedir=basedir+'/'
    
try:
    assert os.path.exists(basedir)
except:
    print 'creating base directory:',basedir
    os.mkdir(basedir)
  
 
def get_children(url):
    children=[]
    if not url[-1]=='/':
        return [url]
    for i in re.findall('''href=["'](.[^"']+)["']''', urllib.urlopen(url).read(), re.I):
            if i.find('?')==0 or i.find('/')==0:
                continue
            #print url+i
            c=get_children(url+i)
            if c:
                children=children+c
    return children

def get_file(f,dataurl,outdir,logfile=None,overwrite=False):
    outfile=f.replace(dataurl,outdir)
    if not os.path.exists(os.path.dirname(outfile)):
        os.makedirs(os.path.dirname(outfile))
    if not os.path.exists(outfile) or overwrite:
        open(outfile,'wb').write(urllib.urlopen(f).read())
        hash=hashfile(outfile)
        open(logfile,'a').write('%s\t%s\t%s\n'%(outfile,timestamp(),hash))

def get_directory(dir,outdir,dataurl,overwrite=False,logfile=None):
    c=get_children(dataurl+dir)
    for file in c:
        get_file(file,dataurl+dir,outdir,logfile=logfile,overwrite=overwrite)
        
def get_base_data(overwrite=False,logfile=None):
    """ 
    grab all necessary data from S3
    note that right now, it doesn't check to make sure all files are there, just the directory
    """
    print 'getting base data, logging to',logfile<commit_msg>add global dataurl arg<commit_after>"""
get_data.py: functions to download data for myconnectome project from Stanford archive
Options:
    --all: get all basic data needed for analysis
    --rawfunc: get raw fMRI data
    --diffusion: get raw diffusion data
    --anatomy: get raw anatomical data
    -h: print this help message
"""

import os,getopt,sys
import urllib
import re
import datetime
import hashlib

def timestamp():
    return datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')

# from  http://stackoverflow.com/questions/3431825/generating-a-md5-checksum-of-a-file
def hashfile(fname, blocksize=65536):
    afile=open(fname,'rb')
    buf = afile.read(blocksize)
    hasher=hashlib.md5()
    while len(buf) > 0:
        hasher.update(buf)
        buf = afile.read(blocksize)
    return hasher.hexdigest()

   
basedir=os.environ['MYCONNECTOME_DIR']

dataurl='http://web.stanford.edu/group/poldracklab/myconnectome-data/'


# my kludgey code requires the base dir to end in a slash
if not basedir[-1]=='/':
    basedir=basedir+'/'
    
try:
    assert os.path.exists(basedir)
except:
    print 'creating base directory:',basedir
    os.mkdir(basedir)
  
 
def get_children(url):
    children=[]
    if not url[-1]=='/':
        return [url]
    for i in re.findall('''href=["'](.[^"']+)["']''', urllib.urlopen(url).read(), re.I):
            if i.find('?')==0 or i.find('/')==0:
                continue
            #print url+i
            c=get_children(url+i)
            if c:
                children=children+c
    return children

def get_file(f,dataurl,outdir,logfile=None,overwrite=False):
    outfile=f.replace(dataurl,outdir)
    if not os.path.exists(os.path.dirname(outfile)):
        os.makedirs(os.path.dirname(outfile))
    if not os.path.exists(outfile) or overwrite:
        open(outfile,'wb').write(urllib.urlopen(f).read())
        hash=hashfile(outfile)
        open(logfile,'a').write('%s\t%s\t%s\n'%(outfile,timestamp(),hash))

def get_directory(dir,outdir,dataurl=dataurl,overwrite=False,logfile=None):
    c=get_children(dataurl+dir)
    for file in c:
        get_file(file,dataurl+dir,outdir,logfile=logfile,overwrite=overwrite)
        
def get_base_data(overwrite=False,logfile=None):
    """ 
    grab all necessary data from S3
    note that right now, it doesn't check to make sure all files are there, just the directory
    """
    print 'getting base data, logging to',logfile
{'fraction_non_alphanumeric': 0.14167528438469493, 'fraction_numerical': 0.008273009307135471, 'mean_word_length': 3.3255813953488373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '496084', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1639, 'n_words': 362}