<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import

import argparse
import json
import os
from pinliner import __version__
import sys


TEMPLATE_FILE = 'importer.template'
TEMPLATE_PATTERN = '${CONTENTS}'


inliner_packages = {}


def output(cfg, what, newline=True):
    # We need indentation for PEP8
    cfg.outfile.write(what)
    if newline:
        cfg.outfile.write(os.linesep)


def process_file(cfg, base_dir, package_path):
    if cfg.tagging:
        output(cfg, '<tag:' + package_path + '>')
    path = os.path.splitext(package_path)[0].replace(os.path.sep, '.')
    package_start = cfg.outfile.tell()
    with open(os.path.join(base_dir, package_path), 'r') as f:
        # Read the whole file
        code = f.read()

        # Insert escape character before ''' since we'll be using ''' to insert
        # the code as a string
        output(cfg, code.replace("'''", r"\'''"), newline=cfg.tagging)
    package_end = cfg.outfile.tell()
    is_package = 1 if path.endswith('__init__') else 0
    if is_package:
        path = path[:-9]
    return path, is_package, package_start, package_end


def template(cfg):
    template_path = os.path.join(os.path.dirname(__file__), TEMPLATE_FILE)
    with open(template_path) as f:
        template = f.read()
    prefix_end = template.index(TEMPLATE_PATTERN)
    prefix_data = template[:prefix_end].replace('%{FORCE_EXC_HOOK}',
                                                str(cfg.set_hook))
    cfg.outfile.write(prefix_data)
    postfix_begin = prefix_end + len(TEMPLATE_PATTERN)
    return template[postfix_begin:]


def process_directory(cfg, base_dir, package_path):
    files = []
    contents = os.listdir(os.path.join(base_dir, package_path))
    for content in contents:
        next_path = os.path.join(package_path, content)
        path = os.path.join(base_dir, next_path)
        if is_module(path):
            files.append(process_file(cfg, base_dir, next_path))
        elif is_package(path):
            files.extend(process_directory(cfg, base_dir, next_path))
    return files

inliner_packages = {
    "a": [
        1, 2334, 2811],
    "a.c": [1, 2852, 2877],
    "a.b": [0, 2186, 2326],
    "a.c.d": [0, 2819, 2844]
}


def process_files(cfg):
    # template would look better as a context manager
    postfix = template(cfg)
    files = []

    output(cfg, "'''")<commit_msg>Remove global inliner_packages variable<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import

import argparse
import json
import os
from pinliner import __version__
import sys


TEMPLATE_FILE = 'importer.template'
TEMPLATE_PATTERN = '${CONTENTS}'


def output(cfg, what, newline=True):
    # We need indentation for PEP8
    cfg.outfile.write(what)
    if newline:
        cfg.outfile.write(os.linesep)


def process_file(cfg, base_dir, package_path):
    if cfg.tagging:
        output(cfg, '<tag:' + package_path + '>')
    path = os.path.splitext(package_path)[0].replace(os.path.sep, '.')
    package_start = cfg.outfile.tell()
    with open(os.path.join(base_dir, package_path), 'r') as f:
        # Read the whole file
        code = f.read()

        # Insert escape character before ''' since we'll be using ''' to insert
        # the code as a string
        output(cfg, code.replace("'''", r"\'''"), newline=cfg.tagging)
    package_end = cfg.outfile.tell()
    is_package = 1 if path.endswith('__init__') else 0
    if is_package:
        path = path[:-9]
    return path, is_package, package_start, package_end


def template(cfg):
    template_path = os.path.join(os.path.dirname(__file__), TEMPLATE_FILE)
    with open(template_path) as f:
        template = f.read()
    prefix_end = template.index(TEMPLATE_PATTERN)
    prefix_data = template[:prefix_end].replace('%{FORCE_EXC_HOOK}',
                                                str(cfg.set_hook))
    cfg.outfile.write(prefix_data)
    postfix_begin = prefix_end + len(TEMPLATE_PATTERN)
    return template[postfix_begin:]


def process_directory(cfg, base_dir, package_path):
    files = []
    contents = os.listdir(os.path.join(base_dir, package_path))
    for content in contents:
        next_path = os.path.join(package_path, content)
        path = os.path.join(base_dir, next_path)
        if is_module(path):
            files.append(process_file(cfg, base_dir, next_path))
        elif is_package(path):
            files.extend(process_directory(cfg, base_dir, next_path))
    return files


def process_files(cfg):
    # template would look better as a context manager
    postfix = template(cfg)
    files = []

    output(cfg, "'''")
{'fraction_non_alphanumeric': 0.123982869379015, 'fraction_numerical': 0.010278372591006424, 'mean_word_length': 3.0511708586296615, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7517152', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1526, 'n_words': 349}