<commit_before># Downlaod the example files from an external source

import os
from urllib2 import urlopen

example_dir = "data"
example_web = "http://atmosviz1.york.ac.uk/~bn506/data/AC_tools/"


file_list = [
"test.nc",
"tracerinfo.dat",
"diaginfo.dat",
]


if not os.path.exists(example_dir):
    os.makedirs(example_dir)

for _file in file_list:
    if _file == "test.nc":
        new_filename = os.path.join(example_dir, "ctm.nc")
    else:
        new_filename = os.path.join(example_dir, _file)

    if not os.path.isfile( new_filename ):
        new_file = open(new_filename, 'wb')
        file_url = example_web + _file
        print file_url
        file_data = urlopen( file_url ).read()
        new_file.write( file_data )
        new_file.close()
    
    

    

<commit_msg>Get data gets all the data files needed for running code, tests and examples.<commit_after># Downlaod the example files from an external source

import os
from urllib2 import urlopen
import logging

data_dir = "../data"
data_url = "http://atmosviz1.york.ac.uk/~bn506/data/AC_tools/"


file_list = [
"HEMCO_Diagnostics.nc",
"ctm.nc",
"diaginfo.dat",
"test.bpch",
"test.log",
"tracerinfo.dat",
"LANDMAP/LANDMAP_LWI_ctm/ctm.nc",
"LANDMAP/LANDMAP_LWI_ctm_025x03125/ctm.nc",
"LANDMAP/LANDMAP_LWI_ctm_05x0666/ctm.nc",
"LANDMAP/LANDMAP_LWI_ctm_2x25.lod/ctm.nc",
"LANDMAP/LANDMAP_LWI_ctm_2x25/ctm.nc",
"LANDMAP/LANDMAP_ctm_2x25/ctm.nc",
]


if not os.path.exists(data_dir):
    os.makedirs(data_dir)

for _file in file_list:
#    if _file == "test.nc":
#        new_filename = os.path.join(data_dir, "ctm.nc")
#    else:
    new_filename = os.path.join(data_dir, _file)

    if not os.path.isfile( new_filename ):
        try:
            os.makedirs(os.path.dirname(new_filename))
        except:
            logging.error("Could not create folder for {file}".format(file=new_filename))
        logging.debug( new_filename + " not found. Downloading now.")
        new_file = open(new_filename, 'wb')
        file_url = data_url + _file
        try:
            logging.debug("downloading from {url}".format(url=file_url))
            file_data = urlopen( file_url ).read()
            new_file.write( file_data )
            logging.debug( new_filename+" downloaded.")
        except:
            logging.error("Failed to download {url}".format(url=file_url))
        new_file.close()
    
    

    


{'fraction_non_alphanumeric': 0.12578616352201258, 'fraction_numerical': 0.013836477987421384, 'mean_word_length': 2.9899665551839463, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7025479', 'n_tokens_mistral': 910, 'n_tokens_neox': 870, 'n_words': 169}