<commit_before>import json
import sys
from collections import OrderedDict

def getSizeOf(obj):
    import cPickle
    return "%0.3fMB" % (len(cPickle.dumps(obj))/1024.0/1024)

class LimitedSizeDict(OrderedDict):
  def __init__(self, *args, **kwds):
    self.size_limit = kwds.pop("size_limit", None)
    OrderedDict.__init__(self, *args, **kwds)
    self._check_size_limit()

  def __setitem__(self, key, value):
    OrderedDict.__setitem__(self, key, value)
    self._check_size_limit()

  def _check_size_limit(self):
    if self.size_limit is not None:
      while len(self) > self.size_limit:
        self.popitem(last=False)

# Holds information regarding budget items
class BudgetItems(object):
    def __init__(self,in_fn,curatedInputs,errors):
        self.minYear = 10000
        self.maxYear = 0
        self.titleCodeYears = {}
        self.num_budgets = 0
        self.codes = {}
        self.skipped = {}
        for line in file(in_fn):
            data = json.loads(line)
            self.num_budgets += 1

            year = data['year']
            title = data['title']
            code = data['code']
            test_value = sum(data.get(x,0)**2 for x in ['net_allocated','gross_allocated','net_revised','commitment_allocated','net_used'])
            active = data.get('active',True)
            if test_value == 0 or not active:
                print "SKIPPING non-active %d/%s" % (year,code)
                errors.skipped(year,code)
                self.skipped.setdefault(year, []).append(code)
                continue

            self.codes.setdefault(year, []).append(code)

            self.minYear = min(self.minYear,year)
            self.maxYear = max(self.maxYear,year)

            self.titleCodeYears.setdefault((title,code),[]).append(year)

        for curatedInput in curatedInputs:
            curated = json.load(file(curatedInput))
            for single in curated:
                src,dest = single
                srcYear,srcCode = src
                if len(srcCode)==10:
                    self.codes[srcYear].append(srcCode)

        print "Budgets processed #%d recs" % self.num_budgets
        print "Years %d..%d" % (self.minYear,self.maxYear)

    def allCodes(self,year):
        return sorted(self.codes[year],key=lambda x:-len(x))

    def skippedCodes(self,year):
        return self.skipped.get(year,[])

class EquivsBase(object):
    def __init__(self,size_limit=None):
        self.equivs = LimitedSizeDict()

    def key(self,year,code):
        return "%d/%s" % (year,code)

    def _put(self,srcYear,srcCode,dstYear,dstCodes):
        eqs = self.equivs.setdefault(self.key(srcYear,srcCode),{})
        if eqs.has_key(dstYear) and eqs[dstYear] != dstCodes:
            print "DUP %s/%s <-- %s/%s (already has %s)" % (srcYear,srcCode,dstYear,dstCodes,eqs[dstYear])
        eqs[dstYear] = dstCodes

    def get(self,srcYear,srcCode):
        return self.equivs.get(self.key(srcYear,srcCode),{})

    def getMinYear(self,srcYear,srcCode,minYear):
        per_year = self.get(srcYear,srcCode)
        per_year = list(per_year.iteritems())
        per_year = filter(lambda x:x[0]>=minYear,per_year)
        if len(per_year) > 0:
            per_year.sort(key=lambda x:x[0]) # sort by year ascending
            first = per_year[0]
            if first[1] == '':
                return first[0],[]  # handle empty lists
            else:
                return first[0],first[1].split(";") # return first batch of codes
        else:
            return None,None

    def setEq(self,srcYear,srcCode,dstYear,dstCodes):
        dstCodes = ";".join(sorted(dstCodes))
        self._put(srcYear,srcCode,dstYear,dstCodes)
        #self._put(dstYear,dstCodes,srcYear,srcCode)

class YearlyEquivs(EquivsBase):
    def __init__(self,budgetItems,curatedInputs,errors):
        super(YearlyEquivs,self).__init__()
        for key,years in budgetItems.titleCodeYears.iteritems():
            title,code = key
            years = sorted(years)
            years = zip(years[1:],years[:-1])
            for srcYear,dstYear in years:
                self.setEq(srcYear,code,dstYear,[code])
        budgetItems.titleCodeYears = None
        for curatedInput in curatedInputs:
            curated = json.load(file(curatedInput))
            for single in curated:
                src,dest = single
                srcYear,srcCode = src
                dstYear = list(set(k[0] for k in dest))
                dstCodes = list(set(k[1] for k in dest))
                if len(dstYear) != 1:
                    raise AssertionError("bad curated from %s: %r" % (curatedInput, single))
                dstYear = dstYear[0]
                self.setEq(srcYear,srcCode,dstYear,dstCodes)
                errors.curated(srcYear, srcCode, dstYear, dstCodes)<commit_msg>filter out empty items in curated<commit_after>import json
import sys
from collections import OrderedDict

def getSizeOf(obj):
    import cPickle
    return "%0.3fMB" % (len(cPickle.dumps(obj))/1024.0/1024)

class LimitedSizeDict(OrderedDict):
  def __init__(self, *args, **kwds):
    self.size_limit = kwds.pop("size_limit", None)
    OrderedDict.__init__(self, *args, **kwds)
    self._check_size_limit()

  def __setitem__(self, key, value):
    OrderedDict.__setitem__(self, key, value)
    self._check_size_limit()

  def _check_size_limit(self):
    if self.size_limit is not None:
      while len(self) > self.size_limit:
        self.popitem(last=False)

# Holds information regarding budget items
class BudgetItems(object):
    def __init__(self,in_fn,curatedInputs,errors):
        self.minYear = 10000
        self.maxYear = 0
        self.titleCodeYears = {}
        self.num_budgets = 0
        self.codes = {}
        self.skipped = {}
        for line in file(in_fn):
            data = json.loads(line)
            self.num_budgets += 1

            year = data['year']
            title = data['title']
            code = data['code']
            test_value = sum(data.get(x,0)**2 for x in ['net_allocated','gross_allocated','net_revised','commitment_allocated','net_used'])
            active = data.get('active',True)
            if test_value == 0 or not active:
                print "SKIPPING non-active %d/%s" % (year,code)
                errors.skipped(year,code)
                self.skipped.setdefault(year, []).append(code)
                continue

            self.codes.setdefault(year, []).append(code)

            self.minYear = min(self.minYear,year)
            self.maxYear = max(self.maxYear,year)

            self.titleCodeYears.setdefault((title,code),[]).append(year)

        for curatedInput in curatedInputs:
            curated = json.load(file(curatedInput))
            for single in curated:
                src,dest = single
                srcYear,srcCode = src
                if len(srcCode)==10:
                    self.codes[srcYear].append(srcCode)

        print "Budgets processed #%d recs" % self.num_budgets
        print "Years %d..%d" % (self.minYear,self.maxYear)

    def allCodes(self,year):
        return sorted(self.codes[year],key=lambda x:-len(x))

    def skippedCodes(self,year):
        return self.skipped.get(year,[])

class EquivsBase(object):
    def __init__(self,size_limit=None):
        self.equivs = LimitedSizeDict()

    def key(self,year,code):
        return "%d/%s" % (year,code)

    def _put(self,srcYear,srcCode,dstYear,dstCodes):
        eqs = self.equivs.setdefault(self.key(srcYear,srcCode),{})
        if eqs.has_key(dstYear) and eqs[dstYear] != dstCodes:
            print "DUP %s/%s <-- %s/%s (already has %s)" % (srcYear,srcCode,dstYear,dstCodes,eqs[dstYear])
        eqs[dstYear] = dstCodes

    def get(self,srcYear,srcCode):
        return self.equivs.get(self.key(srcYear,srcCode),{})

    def getMinYear(self,srcYear,srcCode,minYear):
        per_year = self.get(srcYear,srcCode)
        per_year = list(per_year.iteritems())
        per_year = filter(lambda x:x[0]>=minYear,per_year)
        if len(per_year) > 0:
            per_year.sort(key=lambda x:x[0]) # sort by year ascending
            first = per_year[0]
            if first[1] == '':
                return first[0],[]  # handle empty lists
            else:
                return first[0],first[1].split(";") # return first batch of codes
        else:
            return None,None

    def setEq(self,srcYear,srcCode,dstYear,dstCodes):
        dstCodes = ";".join(sorted(dstCodes))
        self._put(srcYear,srcCode,dstYear,dstCodes)
        #self._put(dstYear,dstCodes,srcYear,srcCode)

class YearlyEquivs(EquivsBase):
    def __init__(self,budgetItems,curatedInputs,errors):
        super(YearlyEquivs,self).__init__()
        for key,years in budgetItems.titleCodeYears.iteritems():
            title,code = key
            years = sorted(years)
            years = zip(years[1:],years[:-1])
            for srcYear,dstYear in years:
                self.setEq(srcYear,code,dstYear,[code])
        budgetItems.titleCodeYears = None
        for curatedInput in curatedInputs:
            curated = json.load(file(curatedInput))
            for single in curated:
                src,dest = single
                srcYear,srcCode = src
                dest = [ k for k in dest if len(k)==2 ]
                dstYear = list(set(k[0] for k in dest))
                dstCodes = list(set(k[1] for k in dest))
                if len(dstYear) != 1:
                    raise AssertionError("bad curated from %s: %r" % (curatedInput, single))
                dstYear = dstYear[0]
                self.setEq(srcYear,srcCode,dstYear,dstCodes)
                errors.curated(srcYear, srcCode, dstYear, dstCodes)
{'fraction_non_alphanumeric': 0.12298303682250725, 'fraction_numerical': 0.007964418700868846, 'mean_word_length': 2.3631304347826085, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4415710', 'n_tokens_mistral': 3154, 'n_tokens_neox': 3057, 'n_words': 590}