<commit_before>import json
import adapter

def go():
    
    # Load JSON file containing instructions for converting data
    instructions = json.load(open("loader/canada.json",'r'))
    
    # For each dataset mentioned in the instructions
    for dataset in instructions['data']:
        # Get a reader for the specified format
        reader = adapter.get_reader(dataset['format'])
        
        # Run through file, and perform conversion.
        with reader(dataset['file']) as r:
            for line in r:
                print line
    <commit_msg>More general<commit_after># -*- coding: utf-8 -*-

import json
import adapter

def go(instructions_file, index=0):
    
    # Load JSON file containing instructions for converting data
    instructions = json.load(open(instructions_file,'r'))
    
    # The dataset at index in the instructions
    dataset = instructions['data'][index]
    print dataset
    
    # Get an adapter for the specified format
    a = adapter.get_adapter(dataset)
    
    return a
{'fraction_non_alphanumeric': 0.08431372549019608, 'fraction_numerical': 0.00196078431372549, 'mean_word_length': 2.712727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584472', 'n_tokens_mistral': 263, 'n_tokens_neox': 262, 'n_words': 95}