<commit_before>from __future__ import print_function
from queue import Queue
from collections import defaultdict
from time import time
from abridger.extraction_model import Relation, merge_relations


class ResultsRow(object):
    def __init__(self, table, row, subjects=None, sticky=False, count=1):
        if subjects is None:
            subjects = set()
        self.table = table
        self.row = row
        self.subjects = subjects
        self.sticky = sticky
        self.count = 1

    def __str__(self):
        return 'row=%s subjects=%s sticky=%s' % (
            self.row, self.subjects, self.sticky)

    def __repr__(self):
        return '<ResultsRow %s>' % str(self)

    def __lt__(self, other):
        return self.row < other.row

    def merge(self, other):
        '''
            Merge two results rows. not-null values take precedence over nulls
        '''

        changed_row = None
        for i, (self_val, other_val) in enumerate(zip(self.row, other.row)):
            if (self_val is None) and (other_val is not None):
                if not changed_row:
                    changed_row = list(self.row)
                changed_row[i] = other_val

        if changed_row is not None:
            self.row = tuple(changed_row)


class WorkItem(object):
    def __init__(self, subject, table, cols, values, sticky,
                 parent_work_item=None, parent_results_row=None):
        assert (cols is None) == (values is None)

        self.subject = subject
        self.table = table
        self.cols = cols
        self.values = values
        self.sticky = sticky
        self.depth = 0

        if parent_work_item is not None:
            self.depth = parent_work_item.depth + 1

        self.set_history(parent_work_item, parent_results_row)

    def value_hash(self, value):
        return hash(tuple([self.subject, self.table, self.cols, value,
                          self.sticky]))

    def fetch_rows(self, database):
        fetched_rows = database.fetch_rows(self.table, self.cols, self.values)
        fetched_rows = [ResultsRow(self.table, fr) for fr in fetched_rows]
        return fetched_rows

    def make_work_item_history(self):
        if self.values is not None:
            cols_csv = ','.join([c.name for c in self.cols])
            values_csv = ','.join([str(v) for v in list(self.values[0])])
            if len(self.values[0]) > 1:
                values_csv = '(%s)' % values_csv
                cols_csv = '(%s)' % cols_csv
            return (self.table, cols_csv, values_csv, self.sticky)
        else:
            return (self.table, None, None, self.sticky)

    def make_results_row_history(self, results_row):
        epk = results_row.table.effective_primary_key
        col_indexes = [results_row.table.cols.index(c) for c in epk]

        cols_csv = ','.join([c.name for c in epk])
        values = [results_row.row[i] for i in col_indexes]
        values_csv = ','.join([str(v) for v in values])
        if len(values) > 1:
            values_csv = '(%s)' % values_csv
            cols_csv = '(%s)' % cols_csv
        return (results_row.table, cols_csv, values_csv, self.sticky)

    def set_history(self, work_item, results_row):
        if work_item is None:
            self.history = [self.make_work_item_history()]
            return

        self.history = list(work_item.history)
        work_item_history = self.make_work_item_history()

        if results_row is not None:
            results_row_history = self.make_results_row_history(results_row)
            if self.history[-1] != results_row_history:
                self.history.append(results_row_history)

            if work_item_history != results_row_history:
                self.history.append(work_item_history)

    def print_history(self):
        if len(self.history) == 0:
            return

        first = True
        for (table, cols, values, sticky) in self.history:
            if not first:
                print(' -> ', end='')
            else:
                first = False

            if cols is not None:
                print('%s.%s=%s' % (table, cols, values), end='')
            else:
                print(table, end='')

            if sticky:
                print('*', end='')
        print()


class Extractor(object):
    def __init__(self, database, extraction_model, explain=False,
                 verbosity=0):
        self.database = database
        self.extraction_model = extraction_model
        self.explain = explain
        self.verbosity = verbosity
        self.work_queue = Queue()
        self.results = defaultdict(lambda: defaultdict(dict))
        self.fetch_count = 0
        self.fetched_row_count = 0
        self.fetched_row_count_per_table = defaultdict(int)
        self.max_depth = 0<commit_msg>Removed statement that never executes<commit_after>from __future__ import print_function
from queue import Queue
from collections import defaultdict
from time import time
from abridger.extraction_model import Relation, merge_relations


class ResultsRow(object):
    def __init__(self, table, row, subjects=None, sticky=False, count=1):
        if subjects is None:
            subjects = set()
        self.table = table
        self.row = row
        self.subjects = subjects
        self.sticky = sticky
        self.count = 1

    def __str__(self):
        return 'row=%s subjects=%s sticky=%s' % (
            self.row, self.subjects, self.sticky)

    def __repr__(self):
        return '<ResultsRow %s>' % str(self)

    def __lt__(self, other):
        return self.row < other.row

    def merge(self, other):
        '''
            Merge two results rows. not-null values take precedence over nulls
        '''

        changed_row = None
        for i, (self_val, other_val) in enumerate(zip(self.row, other.row)):
            if (self_val is None) and (other_val is not None):
                if not changed_row:
                    changed_row = list(self.row)
                changed_row[i] = other_val

        if changed_row is not None:
            self.row = tuple(changed_row)


class WorkItem(object):
    def __init__(self, subject, table, cols, values, sticky,
                 parent_work_item=None, parent_results_row=None):
        assert (cols is None) == (values is None)

        self.subject = subject
        self.table = table
        self.cols = cols
        self.values = values
        self.sticky = sticky
        self.depth = 0

        if parent_work_item is not None:
            self.depth = parent_work_item.depth + 1

        self.set_history(parent_work_item, parent_results_row)

    def value_hash(self, value):
        return hash(tuple([self.subject, self.table, self.cols, value,
                          self.sticky]))

    def fetch_rows(self, database):
        fetched_rows = database.fetch_rows(self.table, self.cols, self.values)
        fetched_rows = [ResultsRow(self.table, fr) for fr in fetched_rows]
        return fetched_rows

    def make_work_item_history(self):
        if self.values is not None:
            cols_csv = ','.join([c.name for c in self.cols])
            values_csv = ','.join([str(v) for v in list(self.values[0])])
            if len(self.values[0]) > 1:
                values_csv = '(%s)' % values_csv
                cols_csv = '(%s)' % cols_csv
            return (self.table, cols_csv, values_csv, self.sticky)
        else:
            return (self.table, None, None, self.sticky)

    def make_results_row_history(self, results_row):
        epk = results_row.table.effective_primary_key
        col_indexes = [results_row.table.cols.index(c) for c in epk]

        cols_csv = ','.join([c.name for c in epk])
        values = [results_row.row[i] for i in col_indexes]
        values_csv = ','.join([str(v) for v in values])
        if len(values) > 1:
            values_csv = '(%s)' % values_csv
            cols_csv = '(%s)' % cols_csv
        return (results_row.table, cols_csv, values_csv, self.sticky)

    def set_history(self, work_item, results_row):
        if work_item is None:
            self.history = [self.make_work_item_history()]
            return

        self.history = list(work_item.history)
        work_item_history = self.make_work_item_history()

        if results_row is not None:
            results_row_history = self.make_results_row_history(results_row)
            if self.history[-1] != results_row_history:
                self.history.append(results_row_history)

            if work_item_history != results_row_history:
                self.history.append(work_item_history)

    def print_history(self):
        first = True
        for (table, cols, values, sticky) in self.history:
            if not first:
                print(' -> ', end='')
            else:
                first = False

            if cols is not None:
                print('%s.%s=%s' % (table, cols, values), end='')
            else:
                print(table, end='')

            if sticky:
                print('*', end='')
        print()


class Extractor(object):
    def __init__(self, database, extraction_model, explain=False,
                 verbosity=0):
        self.database = database
        self.extraction_model = extraction_model
        self.explain = explain
        self.verbosity = verbosity
        self.work_queue = Queue()
        self.results = defaultdict(lambda: defaultdict(dict))
        self.fetch_count = 0
        self.fetched_row_count = 0
        self.fetched_row_count_per_table = defaultdict(int)
        self.max_depth = 0
{'fraction_non_alphanumeric': 0.10001043950307965, 'fraction_numerical': 0.0028186658315064203, 'mean_word_length': 2.086340206185567, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3187785', 'n_tokens_mistral': 2924, 'n_tokens_neox': 2794, 'n_words': 717}