<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from collections import OrderedDict
import json

from parser_data import InlineList, DuplicationList
from state import State, StateMachine
from type_check import is_int, is_float
from format import format



class ParserStateMachine(StateMachine):
    def __init__(self, options):
        self.data = OrderedDict()

        initial = NeutralState(self.data)
        initial.parent = initial

        super(ParserStateMachine, self).__init__(initial, options)

    def get_data(self):
        return self.data

    def preprocess(self, val):
        return val.strip()

class DataState(State):
    def __init__(self, data):
        super(DataState, self).__init__()
        self.data = data


class NeutralState(DataState):
    def run(self, line):
        if '=' in line:
            key, val = [val.strip() for val in line.split('=')]

            # Check if the data exists, in which case its actually a list so we have to change its type
            old_data = self.data.get(key, None)
            if old_data is None:
                self.data[key] = self.read_data(val)
            elif not isinstance(old_data, DuplicationList):
                val = self.data[key]
                self.data[key] = DuplicationList()
                self.data[key].append(val)
            else:
                self.data[key].append(val)

            return self.finish_state()
        else:
            self.debug('= DICT =')
            return self.rerun_with_state(
                DictState(self.data).set_parent(self.parent)
            )

    def read_data(self, val):
        if ',' in val:
            val = [subval.strip() for subval in val.split(',')]
            val = [self.read_data(subval) for subval in val]
            val = InlineList(val)
        elif val == 'True':
            val = True
        elif val == 'False':
            val = False
        elif is_int(val):
            val = int(val)
        elif is_float(val):
            val = float(val)
<commit_msg>Fixes a bug with duplication lists<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from collections import OrderedDict
import json

from parser_data import InlineList, DuplicationList
from state import State, StateMachine
from type_check import is_int, is_float
from format import format



class ParserStateMachine(StateMachine):
    def __init__(self, options):
        self.data = OrderedDict()

        initial = NeutralState(self.data)
        initial.parent = initial

        super(ParserStateMachine, self).__init__(initial, options)

    def get_data(self):
        return self.data

    def preprocess(self, val):
        return val.strip()

class DataState(State):
    def __init__(self, data):
        super(DataState, self).__init__()
        self.data = data


class NeutralState(DataState):
    def run(self, line):
        if '=' in line:
            key, val = [val.strip() for val in line.split('=')]

            old_data = self.data.get(key, None)
            if old_data is None:
                # First time we got said data, just add it in
                self.data[key] = self.read_data(val)
            elif isinstance(old_data, DuplicationList):
                # The stored data is a list, append to it
                self.data[key].append(val)
            else:
                # We got the same key? Turn the stored data into a list
                old_val = self.data[key]
                self.data[key] = DuplicationList()
                self.data[key].append(old_val)
                self.data[key].append(val)

            return self.finish_state()
        else:
            self.debug('= DICT =')
            return self.rerun_with_state(
                DictState(self.data).set_parent(self.parent)
            )

    def read_data(self, val):
        if ',' in val:
            val = [subval.strip() for subval in val.split(',')]
            val = [self.read_data(subval) for subval in val]
            val = InlineList(val)
        elif val == 'True':
            val = True
        elif val == 'False':
            val = False
        elif is_int(val):
            val = int(val)
        elif is_float(val):
            val = float(val)

{'fraction_non_alphanumeric': 0.10232117479867361, 'fraction_numerical': 0.0004737091425864519, 'mean_word_length': 1.9367176634214187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3243093', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1195, 'n_words': 339}