<commit_before>from oelite.meta import *

import sys
import copy
import warnings
import cPickle
import operator
import types
import os
import bb.utils


def unpickle(file):
    return DictMeta(meta=file)


class DictMeta(MetaData):


    def pickle(self, file):
        cPickle.dump(self.dict, file, 2)
        cPickle.dump(self.expand_cache, file, 2)
        return


    INDEXED_FLAGS = ("python", "task", "autoimport", "precondition", "export")


    def __init__(self, meta=None):
        if isinstance(meta, file):
            self.dict = cPickle.load(meta)
            self.expand_cache = cPickle.load(meta)
            meta = None
        elif isinstance(meta, DictMeta):
            self.dict = {}
            #self.dict = meta.dict.copy()
            self.dict = copy.deepcopy(meta.dict)
            #for var in meta.dict:
            #    self.dict[var] = meta.dict[var].copy()
            self.expand_cache = meta.expand_cache.copy()
            #self.expand_cache = copy.deepcopy(meta.expand_cache)
            meta = None
        else:
            self.dict = {}
            self.expand_cache = {}
            self.dict["__flag_index"] = {}
            for flag in self.INDEXED_FLAGS:
                self.dict["__flag_index"][flag] = set([])
        super(DictMeta, self).__init__(meta=meta)
        return


    def copy(self):
        return DictMeta(meta=self)


    def __repr__(self):
        return '%s()'%(self.__class__.__name__)


    def __str__(self):
        return str(self.dict)


    def __len__(self): # required by Sized
        return len(self.dict)


    def keys(self):
        return self.dict.keys()


    def set(self, var, val):
        assert not " " in var
        try:
            self.dict[var][""] = val
        except KeyError:
            self.dict[var] = {"": val}
        self.trim_expand_cache(var)
        return


    def trim_expand_cache(self, var):
        for (cached_var, (cached_val, deps)) in self.expand_cache.items():
            if cached_var == var or var in deps:
                # FIXME: is it safe to delete from the dict we are iterating ?
                del self.expand_cache[cached_var]
        return


    def set_flag(self, var, flag, val):
        #print "set_flag %s[%s]=%s"%(var, flag, val)
        assert not " " in var
        try:
            self.dict[var][flag] = val
        except KeyError:
            self.dict[var] = {flag: val}
        if flag in self.dict["__flag_index"]:
            if val:
                self.dict["__flag_index"][flag].add(var)
            else:
                self.dict["__flag_index"][flag].discard(var)
        if flag == "":
            self.trim_expand_cache(var)
        return


    def weak_set_flag(self, var, flag, val):
        if not var in self.dict.keys() or not flag in self.dict[var].keys():
            self.set_flag(var, flag, val)


    def set_override(self, var, override, val):
        assert var not in ("OVERRIDES", "__overrides", "", ">", "<")
        assert override[0] in ("", ">", "<")
        try:
            self.dict[var]["__overrides"][override[0]][override[1]] = val
        except KeyError, e:
            if e.args[0] == var:
                self.dict[var] = {"__overrides": {'':{}, '>':{}, '<':{}}}
            else:
                assert e.args[0] == "__overrides"
                self.dict[var]["__overrides"] = {'':{}, '>':{}, '<':{}}
            self.dict[var]["__overrides"][override[0]][override[1]] = val
        self.trim_expand_cache(var)
        return


    def get(self, var, expand=FULL_EXPANSION):
        #print "get var=%s expand=%s"%(var, expand)
        assert isinstance(expand, int)
        val = self._get(var, expand)[0]
        #print "get returning %s"%(val)
        return val


    def _get(self, var, expand=FULL_EXPANSION):
        #print "_get expand=%s"%(expand)
        assert isinstance(expand, int)
        try:
            val = self.dict[var][""]
        except KeyError:
            try:
                val = self.dict[var]["defaultval"]
            except KeyError:
                val = None
        if not expand:
            return (val, None)
        if not var in self.dict:
            return (None, None)
        if not isinstance(val, (basestring, types.NoneType)):
            return (val, None)
        if expand != OVERRIDES_EXPANSION:
            try:
                return self.expand_cache[var]
            except KeyError:
                pass

        override_dep = None
        if "__overrides" in self.dict[var]:
            current_overrides, override_dep = self._get_overrides()
            override_dep.add("OVERRIDES")
            var_overrides = self.dict[var]["__overrides"]['']
            append_overrides = self.dict[var]["__overrides"]['>']
            prepend_overrides = self.dict[var]["__overrides"]['<']
            oval = None
            append = ""
            prepend = ""
            for override in current_overrides:
                if oval is None:
                    try:
                        oval = var_overrides[override]
                    except KeyError:
                        pass
                try:
                    append += append_overrides[override] or ""
                except KeyError:
                    pass
                try:
                    prepend = prepend_overrides[override] or "" + prepend
                except KeyError:
                    pass
            if oval is not None:
                val = oval
            val = prepend + (val or "") + append

        if expand == OVERRIDES_EXPANSION:
            return (val, None)

        deps = set()
        #print "get expanding %s=%s"%(var, repr(val))
        expand_method = self.get_flag(var, "expand")<commit_msg>lib/oelite: Set EXTRA_ARCH when variables use MACHINE override<commit_after>from oelite.meta import *

import sys
import copy
import warnings
import cPickle
import operator
import types
import os
import bb.utils


def unpickle(file):
    return DictMeta(meta=file)


class DictMeta(MetaData):


    def pickle(self, file):
        cPickle.dump(self.dict, file, 2)
        cPickle.dump(self.expand_cache, file, 2)
        return


    INDEXED_FLAGS = ("python", "task", "autoimport", "precondition", "export")


    def __init__(self, meta=None):
        if isinstance(meta, file):
            self.dict = cPickle.load(meta)
            self.expand_cache = cPickle.load(meta)
            meta = None
        elif isinstance(meta, DictMeta):
            self.dict = {}
            #self.dict = meta.dict.copy()
            self.dict = copy.deepcopy(meta.dict)
            #for var in meta.dict:
            #    self.dict[var] = meta.dict[var].copy()
            self.expand_cache = meta.expand_cache.copy()
            #self.expand_cache = copy.deepcopy(meta.expand_cache)
            meta = None
        else:
            self.dict = {}
            self.expand_cache = {}
            self.dict["__flag_index"] = {}
            for flag in self.INDEXED_FLAGS:
                self.dict["__flag_index"][flag] = set([])
        super(DictMeta, self).__init__(meta=meta)
        return


    def copy(self):
        return DictMeta(meta=self)


    def __repr__(self):
        return '%s()'%(self.__class__.__name__)


    def __str__(self):
        return str(self.dict)


    def __len__(self): # required by Sized
        return len(self.dict)


    def keys(self):
        return self.dict.keys()


    def set(self, var, val):
        assert not " " in var
        try:
            self.dict[var][""] = val
        except KeyError:
            self.dict[var] = {"": val}
        self.trim_expand_cache(var)
        return


    def trim_expand_cache(self, var):
        for (cached_var, (cached_val, deps)) in self.expand_cache.items():
            if cached_var == var or var in deps:
                # FIXME: is it safe to delete from the dict we are iterating ?
                del self.expand_cache[cached_var]
        return


    def set_flag(self, var, flag, val):
        #print "set_flag %s[%s]=%s"%(var, flag, val)
        assert not " " in var
        try:
            self.dict[var][flag] = val
        except KeyError:
            self.dict[var] = {flag: val}
        if flag in self.dict["__flag_index"]:
            if val:
                self.dict["__flag_index"][flag].add(var)
            else:
                self.dict["__flag_index"][flag].discard(var)
        if flag == "":
            self.trim_expand_cache(var)
        return


    def weak_set_flag(self, var, flag, val):
        if not var in self.dict.keys() or not flag in self.dict[var].keys():
            self.set_flag(var, flag, val)


    def set_override(self, var, override, val):
        assert var not in ("OVERRIDES", "__overrides", "", ">", "<")
        assert override[0] in ("", ">", "<")
        try:
            self.dict[var]["__overrides"][override[0]][override[1]] = val
        except KeyError, e:
            if e.args[0] == var:
                self.dict[var] = {"__overrides": {'':{}, '>':{}, '<':{}}}
            else:
                assert e.args[0] == "__overrides"
                self.dict[var]["__overrides"] = {'':{}, '>':{}, '<':{}}
            self.dict[var]["__overrides"][override[0]][override[1]] = val
        self.trim_expand_cache(var)
        return


    def get(self, var, expand=FULL_EXPANSION):
        #print "get var=%s expand=%s"%(var, expand)
        assert isinstance(expand, int)
        val = self._get(var, expand)[0]
        #print "get returning %s"%(val)
        return val


    def _get(self, var, expand=FULL_EXPANSION):
        #print "_get expand=%s"%(expand)
        assert isinstance(expand, int)
        try:
            val = self.dict[var][""]
        except KeyError:
            try:
                val = self.dict[var]["defaultval"]
            except KeyError:
                val = None
        if not expand:
            return (val, None)
        if not var in self.dict:
            return (None, None)
        if not isinstance(val, (basestring, types.NoneType)):
            return (val, None)
        if expand != OVERRIDES_EXPANSION:
            try:
                return self.expand_cache[var]
            except KeyError:
                pass

        override_dep = None
        if "__overrides" in self.dict[var]:
            current_overrides, override_dep = self._get_overrides()
            override_dep.add("OVERRIDES")
            var_overrides = self.dict[var]["__overrides"]['']
            append_overrides = self.dict[var]["__overrides"]['>']
            prepend_overrides = self.dict[var]["__overrides"]['<']
            oval = None
            append = ""
            prepend = ""
            var_override_used = None
            overrides_used = set()
            for override in current_overrides:
                if oval is None:
                    try:
                        oval = var_overrides[override]
                        var_override_used = override
                    except KeyError:
                        pass
                try:
                    append += append_overrides[override] or ""
                    overrides_used.add(override)
                except KeyError:
                    pass
                try:
                    prepend = prepend_overrides[override] or "" + prepend
                    overrides_used.add(override)
                except KeyError:
                    pass
            if oval is not None:
                val = oval
                overrides_used.add(var_override_used)
            val = prepend + (val or "") + append
            for override in overrides_used:
                if override.startswith('MACHINE_'):
                    self.dict['EXTRA_ARCH'] = '.%s'%(self.dict['MACHINE'])
                    break

        if expand == OVERRIDES_EXPANSION:
            return (val, None)

        deps = set()
        #print "get expanding %s=%s"%(var, repr(val))
        expand_method = self.get_flag(var, "expand")
{'fraction_non_alphanumeric': 0.12221944513871533, 'fraction_numerical': 0.0016662501041406314, 'mean_word_length': 1.7023863124718595, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5289232', 'n_tokens_mistral': 3626, 'n_tokens_neox': 3507, 'n_words': 814}