<commit_before>#!/usr/bin/python2.7
# -*- coding: utf8 -*-

"""
Copyright [2014,2015] [beholder developers]

This file is part of beholder project

Provides a generic interface to compare rainbow resources
 - input resources sets
 - output resources deltas computing

 # Here we split our custom Redis value format:
 # <base64_encoded_data> <epoch_redis> <epoch_mtime> <file_checksum>
 #         î___diff

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import bz2

class rainbow_compare_error(Exception):
    pass

class rainbowCompare():

  def __init__(self, type):
    self.type = type

    pass

  def cmp(self, resourceA, resourceB):
    ''' resource[3] (md5sum) comparaison

    '''
    return resourceA.split(' ')[3] != resourceB.split(' ')[3]

  def diff(self, resourcesA, resourcesB):
    # 1. select changed resources
    keysA = set(resourcesA.keys())
    keysB = set(resourcesB.keys())
    inter = keysA.intersection(keysB)
    addResources = keysB - inter
    remResources = keysA - inter
    chgResources = []
    for r in inter:
      if self.cmp(resourcesA[r],resourcesB[r]):
        chgResources.append(r)
    #print addResources, remResources
    #print chgResources
    # 2. foreach chgt diff compute
    for r in chgResources:
      dataA = bz2.decompress(resourcesA[r].split(' ')[0].decode("base64"))
      dataB = bz2.decompress(resourcesB[r].split(' ')[0].decode("base64"))
      A = set( dataA.split('\n') )
      B = set( dataB.split('\n') )
      interAB = A.intersection(B)
    # 3. print the diff
      for d in A - interAB:
        print "%s,%s,-,%s" % (resourcesA[r].split(' ')[2], self.type, d)
      for d in B - interAB:
        print "%s,%s,+,%s" % (resourcesB[r].split(' ')[2], self.type, d)
    for r in addResources:
      for d in bz2.decompress(resourcesB[r].split(' ')[0].decode("base64")).split('\n'):
        print "%s,%s,+,%s" % (resourcesB[r].split(' ')[2], self.type, d)
    pass

    # packages resource diff fmt
    # add "$(_ts),$(b),+,$(_cur)"
    # del "$(_ts),$(b),-,$(_prev)"
    # with _ts epoch_mtime, b packages, $(_cur)|$(_prev) line

if __name__ == '__main__':
  t1 = { 'h1': bz2.compress('<base64_encoded_data11>').encode("base64") + ' <epoch_redis> t1h1<epoch_mtime> <file_checksum11>',<commit_msg>epoch date for loptimize<commit_after>#!/usr/bin/python2.7
# -*- coding: utf8 -*-

"""
Copyright [2014,2015] [beholder developers]

This file is part of beholder project

Provides a generic interface to compare rainbow resources
 - input resources sets
 - output resources deltas computing

 # Here we split our custom Redis value format:
 # <base64_encoded_data> <epoch_redis> <epoch_mtime> <file_checksum>
 #         î___diff

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import bz2

class rainbow_compare_error(Exception):
    pass

class rainbowCompare():

  def __init__(self, type):
    self.type = type

    pass

  def cmp(self, resourceA, resourceB):
    ''' resource[3] (md5sum) comparaison

    '''
    return resourceA.split(' ')[3] != resourceB.split(' ')[3]

  def diff(self, resourcesA, resourcesB):
    # 1. select changed resources
    keysA = set(resourcesA.keys())
    keysB = set(resourcesB.keys())
    inter = keysA.intersection(keysB)
    addResources = keysB - inter
    remResources = keysA - inter
    chgResources = []
    for r in inter:
      if self.cmp(resourcesA[r],resourcesB[r]):
        chgResources.append(r)
    #print addResources, remResources
    #print chgResources
    # 2. foreach chgt diff compute
    for r in chgResources:
      rA = resourcesA[r].split(' ')
      rB = resourcesB[r].split(' ')
      dataA = bz2.decompress(rA[0].decode("base64"))
      dataB = bz2.decompress(rB[0].decode("base64"))
      A = set( dataA.split('\n') )
      B = set( dataB.split('\n') )
      interAB = A.intersection(B)
    # 3. print the diff
      for d in A - interAB:
        print "%s,%s,-,%s" % (rA[2], self.type, d)
      for d in B - interAB:
        print "%s,%s,+,%s" % (rB[2], self.type, d)
    for r in addResources:
      for d in bz2.decompress(rB[0].decode("base64")).split('\n'):
        print "%s,%s,+,%s" % (rB[2], self.type, d)
    pass

    # packages resource diff fmt
    # add "$(_ts),$(b),+,$(_cur)"
    # del "$(_ts),$(b),-,$(_prev)"
    # with _ts epoch_mtime, b packages, $(_cur)|$(_prev) line

if __name__ == '__main__':
  t1 = { 'h1': bz2.compress('<base64_encoded_data11>').encode("base64") + ' <epoch_redis> t1h1<epoch_mtime> <file_checksum11>',
{'fraction_non_alphanumeric': 0.1404090657822001, 'fraction_numerical': 0.019531969780726, 'mean_word_length': 3.4491803278688526, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259352', 'n_tokens_mistral': 1844, 'n_tokens_neox': 1774, 'n_words': 534}