<commit_before>#!/usr/bin/python

import os
from parse import parse
from translate import translate
import json
import argparse

def subfilesRelative(directory, filter_by):
    """ Returns a generator of pairs (subpath, filename) in a given directory.
        Filenames are filtered by the filter_by function.
    """
    for dir_, _, files in os.walk(directory):
        for file in files:
            if filter_by(file):
                yield os.path.relpath(dir_, inputDir), file

def parseCtags(filename):
    tags = {}
    if not os.path.exists(filename):
        raise Exception('Failed to found ctags file at %s' % (filename))
    with open(filename) as file:
        for line in file:
            # in ctags format it is TAG\tLOCATION\t..
            symbol, location = line.strip().split('\t')[:2]
            # we assume sources are in src/
            tags[symbol] = location.split('src/')[-1]
    return tags

def translateExamples(inputDir, outputDir, targetsDir, ctagsFile,
                      includedTargets=None, storeVars=False):

    tags = parseCtags(ctagsFile)
    # Load all target dictionaries
    targets = []
    for target in os.listdir(targetsDir):
        # Ignore targets not in includedTargets
        if includedTargets and not os.path.basename(target).split(".")[0] in includedTargets:
            continue

        with open(os.path.join(targetsDir, target)) as tFile:
            targets.append(json.load(tFile))
<commit_msg>Fixes #3253 Cookbook generation failure missing responsible file indicator.<commit_after>#!/usr/bin/python

import os
from parse import parse
from translate import translate
import json
import argparse

def subfilesRelative(directory, filter_by):
    """ Returns a generator of pairs (subpath, filename) in a given directory.
        Filenames are filtered by the filter_by function.
    """
    for dir_, _, files in os.walk(directory):
        for file in files:
            if filter_by(file):
                yield os.path.relpath(dir_, inputDir), file

def parseCtags(filename):
    tags = {}
    if not os.path.exists(filename):
        raise Exception('Failed to found ctags file at %s' % (filename))
    with open(filename) as file:
        for line in file:
            # in ctags format it is TAG\tLOCATION\t..
            symbol, location = line.strip().split('\t')[:2]
            # we assume sources are in src/
            tags[symbol] = location.split('src/')[-1]
    return tags

def translateExamples(inputDir, outputDir, targetsDir, ctagsFile,
                      includedTargets=None, storeVars=False):

    tags = parseCtags(ctagsFile)
    # Load all target dictionaries
    targets = []
    for target in os.listdir(targetsDir):
        # Ignore targets not in includedTargets
        if includedTargets and not os.path.basename(target).split(".")[0] in includedTargets:
            continue
        
        translate_file = os.path.join(targetsDir, target)
        
        with open(translate_file) as tFile:
            try:
                targets.append(json.load(tFile))
            except Exception, err:
                print("Error translating file: %s\n%s" % (translate_file, err))
    
{'fraction_non_alphanumeric': 0.09619616472807294, 'fraction_numerical': 0.0031436655139893115, 'mean_word_length': 2.519911504424779, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3611063', 'n_tokens_mistral': 894, 'n_tokens_neox': 873, 'n_words': 280}