# Path: checkmate/helpers/hashing.py
# class Hasher(object):
# 
#     def __init__(self):
#         self.digest = hashlib.md5()
# 
#     def add(self,value):
#         if isinstance(value,unicode):
#             v = value.encode("utf-8","ignore")
#         elif isinstance(value,(str)):
#             v = unicode(value,errors = 'replace').encode("utf-8","ignore")
#         elif isinstance(value,(int,float,long,complex,bool)):
#             v = unicode(str(value),errors = 'replace').encode("utf-8","ignore")
#         elif isinstance(value,(tuple,list)):
#             for v in value:
#                 self.add(v)
#             return
#         elif isinstance(value,dict):
#             for key,v in sorted(value.items(),key = lambda x : x[0]):
#                 self.add(key)
#                 self.add(v)
#             return
#         elif value is None:
#             v = '1bcdadabdf0de99dbdb747e951e967c5'
#         else:
#             raise AttributeError("Unhashable type: %s" % str(type(value)))
# 
#         self.digest.update(v)
# 
#     def digest(self):
#         return self.digest
# 
# Path: checkmate/lib/stats/helpers.py
# def directory_splitter(path,include_filename = False):
#     if include_filename:
#         path_hierarchy = path.split("/")
#     else:
#         path_hierarchy = path.split("/")[:-1]
#     if path.startswith('/'):
#         path_hierarchy = path_hierarchy[1:]
#     paths = []
#     current_path = ''
#     for partial_path in path_hierarchy:
#         paths.append(current_path)
#         if current_path != '':
#             current_path+='/'
#         current_path+=partial_path
#     paths.append(current_path)
#     return paths
# 
# Path: checkmate/helpers/issue.py
# class IssuesMapReducer(MapReducer):
# 
#     def __init__(self,aggregators,count_column = 'count',group_by = ['language','analyzer','code']):
#         self.aggregators = aggregators
#         self.count_column = count_column
#         self.group_by = group_by
# 
#     def map(self,item):
#         for group in self.group_by:
#             if not group in item:
#                 return []
#         return [(key,item) for aggregator in self.aggregators 
#                 for key in aggregator(item)]
# 
#     def reduce(self,key,items):
#         grouped_issues ={}
# 
#         if self.group_by:
#             for item in items:
#                 invalid_item = False
#                 current_dict = grouped_issues
#                 for group in self.group_by[:-1]:
#                     if not group in item:
#                         invalid_item = True
#                         break
#                     if not item[group] in current_dict:
#                         current_dict[item[group]] = {}
#                     current_dict = current_dict[item[group]]
#                 if not self.group_by[-1] in item:
#                     invalid_item = True
#                 if invalid_item:
#                     continue
#                 if not item[self.group_by[-1]] in current_dict:
#                     current_dict[item[self.group_by[-1]]]= [0,0]
#                 cd = current_dict[item[self.group_by[-1]]]
#                 cd[0]+=1
#                 cd[1]+=item[self.count_column]
#             return grouped_issues
#         else:
#             issues_sum = [0,0]
#             for item in items:
#                 issues_sum[0]+=1
#                 issues_sum[1]+=item[self.count_column]
#             return issues_sum


from blitzdb import Document, SqlBackend
from blitzdb.fields import (BooleanField,
                            CharField,
                            DateTimeField,
                            ForeignKeyField,
                            ManyToManyField,
                            TextField,
                            EnumField,
                            IntegerField)
from checkmate.helpers.hashing import Hasher
from checkmate.lib.stats.helpers import directory_splitter
from checkmate.lib.analysis import AnalyzerSettingsError
from checkmate.helpers.issue import IssuesMapReducer
    from sqlalchemy.sql import (select,
                                insert,
                                func,
                                and_,
                                expression,
                                exists)
import os
import uuid
import time
import datetime
import logging
import traceback
        table = diff_issue_occurrence_table\
        .join(issue_occurrence_table,
              issue_occurrence_table.c.pk == diff_issue_occurrence_table.c.issue_occurrence)\
        .join(issue_table, and_(issue_occurrence_table.c.issue == issue_table.c.pk, issue_table.c.ignore == ignore))\
        .join(file_revision_table)

        #we select the aggregated issues for all file revisions in this snapshot
        s = select(group_columns+[func.count().label('count')])\
        .select_from(table)\
        .where(and_(exists(subselect),diff_issue_occurrence_table.c.diff == self.pk))\
        .group_by(*group_columns)\
        .order_by(file_revision_table.c.path)

        #we fetch the result
        with self.backend.transaction():
            result  = self.backend.connection.execute(s).fetchall()

        #we aggregate the issues by path fragments
        aggregator = lambda f: directory_splitter(f['path'],include_filename = include_filename)

        added_issues = []
        fixed_issues = []

        for row in result:
            if row['key'] == 'added':
                added_issues.append(row)
            else:
                fixed_issues.append(row)

        #we perform a map/reduce on the result

# Path: checkmate/helpers/hashing.py
# class Hasher(object):
# 
#     def __init__(self):
#         self.digest = hashlib.md5()
# 
#     def add(self,value):
#         if isinstance(value,unicode):
#             v = value.encode("utf-8","ignore")
#         elif isinstance(value,(str)):
#             v = unicode(value,errors = 'replace').encode("utf-8","ignore")
#         elif isinstance(value,(int,float,long,complex,bool)):
#             v = unicode(str(value),errors = 'replace').encode("utf-8","ignore")
#         elif isinstance(value,(tuple,list)):
#             for v in value:
#                 self.add(v)
#             return
#         elif isinstance(value,dict):
#             for key,v in sorted(value.items(),key = lambda x : x[0]):
#                 self.add(key)
#                 self.add(v)
#             return
#         elif value is None:
#             v = '1bcdadabdf0de99dbdb747e951e967c5'
#         else:
#             raise AttributeError("Unhashable type: %s" % str(type(value)))
# 
#         self.digest.update(v)
# 
#     def digest(self):
#         return self.digest
# 
# Path: checkmate/lib/stats/helpers.py
# def directory_splitter(path,include_filename = False):
#     if include_filename:
#         path_hierarchy = path.split("/")
#     else:
#         path_hierarchy = path.split("/")[:-1]
#     if path.startswith('/'):
#         path_hierarchy = path_hierarchy[1:]
#     paths = []
#     current_path = ''
#     for partial_path in path_hierarchy:
#         paths.append(current_path)
#         if current_path != '':
#             current_path+='/'
#         current_path+=partial_path
#     paths.append(current_path)
#     return paths
# 
# Path: checkmate/helpers/issue.py
# class IssuesMapReducer(MapReducer):
# 
#     def __init__(self,aggregators,count_column = 'count',group_by = ['language','analyzer','code']):
#         self.aggregators = aggregators
#         self.count_column = count_column
#         self.group_by = group_by
# 
#     def map(self,item):
#         for group in self.group_by:
#             if not group in item:
#                 return []
#         return [(key,item) for aggregator in self.aggregators 
#                 for key in aggregator(item)]
# 
#     def reduce(self,key,items):
#         grouped_issues ={}
# 
#         if self.group_by:
#             for item in items:
#                 invalid_item = False
#                 current_dict = grouped_issues
#                 for group in self.group_by[:-1]:
#                     if not group in item:
#                         invalid_item = True
#                         break
#                     if not item[group] in current_dict:
#                         current_dict[item[group]] = {}
#                     current_dict = current_dict[item[group]]
#                 if not self.group_by[-1] in item:
#                     invalid_item = True
#                 if invalid_item:
#                     continue
#                 if not item[self.group_by[-1]] in current_dict:
#                     current_dict[item[self.group_by[-1]]]= [0,0]
#                 cd = current_dict[item[self.group_by[-1]]]
#                 cd[0]+=1
#                 cd[1]+=item[self.count_column]
#             return grouped_issues
#         else:
#             issues_sum = [0,0]
#             for item in items:
#                 issues_sum[0]+=1
#                 issues_sum[1]+=item[self.count_column]
#             return issues_sum


# Path: checkmate/lib/models.py
from blitzdb import Document, SqlBackend
from blitzdb.fields import (BooleanField,
                            CharField,
                            DateTimeField,
                            ForeignKeyField,
                            ManyToManyField,
                            TextField,
                            EnumField,
                            IntegerField)
from checkmate.helpers.hashing import Hasher
from checkmate.lib.stats.helpers import directory_splitter
from checkmate.lib.analysis import AnalyzerSettingsError
from checkmate.helpers.issue import IssuesMapReducer
    from sqlalchemy.sql import (select,
                                insert,
                                func,
                                and_,
                                expression,
                                exists)
import os
import uuid
import time
import datetime
import logging
import traceback

        table = diff_issue_occurrence_table\
        .join(issue_occurrence_table,
              issue_occurrence_table.c.pk == diff_issue_occurrence_table.c.issue_occurrence)\
        .join(issue_table, and_(issue_occurrence_table.c.issue == issue_table.c.pk, issue_table.c.ignore == ignore))\
        .join(file_revision_table)

        #we select the aggregated issues for all file revisions in this snapshot
        s = select(group_columns+[func.count().label('count')])\
        .select_from(table)\
        .where(and_(exists(subselect),diff_issue_occurrence_table.c.diff == self.pk))\
        .group_by(*group_columns)\
        .order_by(file_revision_table.c.path)

        #we fetch the result
        with self.backend.transaction():
            result  = self.backend.connection.execute(s).fetchall()

        #we aggregate the issues by path fragments
        aggregator = lambda f: directory_splitter(f['path'],include_filename = include_filename)

        added_issues = []
        fixed_issues = []

        for row in result:
            if row['key'] == 'added':
                added_issues.append(row)
            else:
                fixed_issues.append(row)

        #we perform a map/reduce on the result

        map_reducer = IssuesMapReducer(aggregators = [aggregator],