<commit_before>
import os
import collections
from collections import deque
from functools import partial

from solar import utils
from solar.interfaces.db import get_db

from enum import Enum


db = get_db()


STATES = Enum('States', 'error inprogress pending success')


def state_file(name):
    if 'log' in name:
        return Log(name)
    elif 'data' in name:
        return Data(name)


CD = partial(state_file, 'commited_data')
SL = partial(state_file, 'stage_log')
CL = partial(state_file, 'commit_log')


class LogItem(object):

    def __init__(self, uid, res, log_action, diff, state=None):
        self.uid = uid
        self.res = res
        self.log_action = log_action
        self.diff = diff
        self.state = state or STATES.pending

    def to_yaml(self):
        return utils.yaml_dump(self.to_dict())

    def to_dict(self):
        return {'uid': self.uid,
                'res': self.res,
                'log_action': self.log_action,
                'diff': self.diff,
                'state': self.state.name}

    @classmethod
    def from_dict(cls, **kwargs):
        state = getattr(STATES, kwargs.get('state', ''), STATES.pending)
        kwargs['state'] = state
        return cls(**kwargs)

    def __str__(self):
        return self.compact

    def __repr__(self):
        return self.compact

    @property
    def compact(self):
        return 'log task={} uid={}'.format(self.log_action, self.uid)

    @property
    def details(self):
        rst = []
        for type_, val, change in self.diff:
            if type_ == 'add':
                for it in change:
                    if isinstance(it, dict):
                        rst.append('++ {}: {}'.format(it[0], it[1]['value']))
                    else:
                        rst.append('++ {}: {}'.format(it[0], str(it[1])))
            elif type_ == 'change':
                rst.append('-+ {}: {} >> {}'.format(val, change[0], change[1]))
        return rst


class Log(object):

    def __init__(self, path):
        self.ordered_log = db.get_set(path)

    def append(self, logitem):
        self.ordered_log.add([(logitem.log_action, logitem.to_dict())])

    def pop(self, uid):
        item = self.get(uid)
        if not item:
            return None
        self.ordered_log.rem([uid])
        return item
<commit_msg>Unwrap diff structure till value is found<commit_after>
import os
import collections
from collections import deque
from functools import partial

from solar import utils
from solar.interfaces.db import get_db

from enum import Enum


db = get_db()


STATES = Enum('States', 'error inprogress pending success')


def state_file(name):
    if 'log' in name:
        return Log(name)
    elif 'data' in name:
        return Data(name)


CD = partial(state_file, 'commited_data')
SL = partial(state_file, 'stage_log')
CL = partial(state_file, 'commit_log')


class LogItem(object):

    def __init__(self, uid, res, log_action, diff, state=None):
        self.uid = uid
        self.res = res
        self.log_action = log_action
        self.diff = diff
        self.state = state or STATES.pending

    def to_yaml(self):
        return utils.yaml_dump(self.to_dict())

    def to_dict(self):
        return {'uid': self.uid,
                'res': self.res,
                'log_action': self.log_action,
                'diff': self.diff,
                'state': self.state.name}

    @classmethod
    def from_dict(cls, **kwargs):
        state = getattr(STATES, kwargs.get('state', ''), STATES.pending)
        kwargs['state'] = state
        return cls(**kwargs)

    def __str__(self):
        return self.compact

    def __repr__(self):
        return self.compact

    @property
    def compact(self):
        return 'log task={} uid={}'.format(self.log_action, self.uid)

    @property
    def details(self):
        rst = []
        for type_, val, change in self.diff:
            if type_ == 'add':
                for it in change:
                    rst.append('++ {}: {}'.format(it[0], unwrap_val(it[1])))
            elif type_ == 'change':
                rst.append('-+ {}: {} >> {}'.format(val, change[0], change[1]))
        return rst


def unwrap_val(it):
    if isinstance(it, dict):
        if it['emitter']:
            return '{}::{}'.format(it['emitter'], it['value'])
        return it['value']
    elif isinstance(it, list):
        return [unwrap_val(i) for i in it]
    else:
        return it[1]


class Log(object):

    def __init__(self, path):
        self.ordered_log = db.get_set(path)

    def append(self, logitem):
        self.ordered_log.add([(logitem.log_action, logitem.to_dict())])

    def pop(self, uid):
        item = self.get(uid)
        if not item:
            return None
        self.ordered_log.rem([uid])
        return item

{'fraction_non_alphanumeric': 0.1312837557125052, 'fraction_numerical': 0.002285002077274616, 'mean_word_length': 2.192970822281167, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2338208', 'n_tokens_mistral': 1525, 'n_tokens_neox': 1464, 'n_words': 389}