<commit_before>#!/usr/bin/env python3

import os, sys, re, yaml
import argparse
from copy import copy
from parse import parse

class Dep2YAML:

    PRELUDE = "## DISTANT ##"
    GLOBAL_DEFINE_RULE = "global_define_rule"
    WRITE_BUILD_SECTION_BEGIN = '>>>'
    WRITE_BUILD_SECTION_END = '<<<'
    OUTPUT = "output"
    RULE = "rule"
    VARIABLE_DEFINITION = "variable_definition"
    EXPLICIT_DEPENDENCY = "explicit_dependency"
    IMPLICIT_DEPENDENCY = "implicit_dependency"

    ### INITIALIZATION ###

    def __init__(self, dep_file):

        self.root_dir = os.getcwd()
        self.defined_rules = {}

        self._dep_file = dep_file
        self._data = None
        self._load_data()

    def __del__(self):
        self._close_input_stream()

    def _load_data(self):
        self.data = ""
        for line in self._dep_file:
            if (not self._is_with_prelude(line)):
                continue
            self.data = self.data + line

    def _close_input_stream(self):
        if self._dep_file is not sys.stdin:
            self._dep_file.close()

    ### HELPER METHODS ###

    def _is_with_prelude(self, line):
        return Dep2YAML.PRELUDE in line

    def _adj_line(self, line):
        nline = copy(line)
        nline = nline.replace(Dep2YAML.PRELUDE, "")
        return nline.strip()

    def _create_string_stream(self, separators="\n"):
        start = 0
        for end in range(len(self.data)):
            if self.data[end] in separators:
                yield self.data[start:end]
                start = end + 1
        if start < end:
            yield self.data[start:end+1]

    def _format_rule(self, rule):<commit_msg>Get proper lines during file loading<commit_after>#!/usr/bin/env python3

import os, sys, re, yaml
import argparse
from copy import copy
from parse import parse


lineno = 0

class Dep2YAML:

    PRELUDE = "## DISTANT ##"
    GLOBAL_DEFINE_RULE = "global_define_rule"
    WRITE_BUILD_SECTION_BEGIN = '>>>'
    WRITE_BUILD_SECTION_END = '<<<'
    OUTPUT = "output"
    RULE = "rule"
    VARIABLE_DEFINITION = "variable_definition"
    EXPLICIT_DEPENDENCY = "explicit_dependency"
    IMPLICIT_DEPENDENCY = "implicit_dependency"

    ### INITIALIZATION ###


    def __init__(self, dep_file):

        self.root_dir = os.getcwd()
        self.defined_rules = {}

        self._dep_file = dep_file
        self._data = None
        self._load_data()

    def __del__(self):
        self._close_input_stream()

    def _load_data(self):
        self.data = list(filter(lambda x: Dep2YAML.PRELUDE in x,self._dep_file.readlines()))
        self.data = "".join(self.data)

    def _close_input_stream(self):
        if self._dep_file is not sys.stdin:
            self._dep_file.close()

    ### HELPER METHODS ###

    def _is_with_prelude(self, line):
        return Dep2YAML.PRELUDE in line

    def _adj_line(self, line):
        nline = copy(line)
        nline = nline.replace(Dep2YAML.PRELUDE, "")
        return nline.strip()

    def _create_string_stream(self, separators="\n"):
        start = 0
        for end in range(len(self.data)):
            if self.data[end] in separators:
                yield self.data[start:end]
                start = end + 1
        if start < end:
            yield self.data[start:end+1]

    def _format_rule(self, rule):
{'fraction_non_alphanumeric': 0.10703547805171378, 'fraction_numerical': 0.004810583283223091, 'mean_word_length': 2.415811088295688, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1421615', 'n_tokens_mistral': 1168, 'n_tokens_neox': 1093, 'n_words': 234}