<commit_before>import os
import re
import sys

from collections import namedtuple
from generate_utils import Generate
from os.path import exists, dirname


MapInclude = namedtuple('MapIncludes', 'name origin')


class GenerateMapFiles(Generate):
    '''Create the mapfiles based on the template in the portal folder and the configuration.
    '''
    def generate(self):
        includes = self._get_map_to_include()
        self._generate_map_files(includes)

    def _get_map_to_include(self):
        origin = self.path(self.src['base_include'], 'portals', self.portal, ext='.map.in')
        included_map = re.compile(r'''^\s*(?:(?!#))INCLUDE\s+["'](.*)["']''', re.M)

        files_to_parse = [MapInclude(name=origin, origin=origin)]
        files_parsed = []

        while len(files_to_parse) > 0:
            current_include = files_to_parse.pop()<commit_msg>Allow portal template to have .in.map and .map.in extensions<commit_after>import os
import re
import sys

from collections import namedtuple
from generate_utils import Generate
from os.path import exists, dirname


MapInclude = namedtuple('MapIncludes', 'name origin')


class GenerateMapFiles(Generate):
    '''Create the mapfiles based on the template in the portal folder and the configuration.
    '''
    def generate(self):
        includes = self._get_map_to_include()
        self._generate_map_files(includes)

    def _get_map_to_include(self):
        portal_dir = self.path(self.src['base_include'], 'portals')
        origin = self.path(portal_dir, self.portal, ext='.map.in')
        if not exists(origin):
            origin = self.path(portal_dir, self.portal, ext='.in.map')
        included_map = re.compile(r'''^\s*(?:(?!#))INCLUDE\s+["'](.*)["']''', re.M)

        files_to_parse = [MapInclude(name=origin, origin=origin)]
        files_parsed = []

        while len(files_to_parse) > 0:
            current_include = files_to_parse.pop()
{'fraction_non_alphanumeric': 0.1292834890965732, 'fraction_numerical': 0.0010384215991692627, 'mean_word_length': 3.6433734939759037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '750688', 'n_tokens_mistral': 592, 'n_tokens_neox': 581, 'n_words': 142}