<commit_before>#!/usr/bin/python

import os
import sys
import argparse
import traceback
from input_handling import findNumber
from decimal import Decimal
from group import Group,run_grouping

def dist(tx, ty):
    return abs(tx - ty)
    
class Neighbor(object):
    def __init__(self, t, line):
        self.t = t
        self.line = line
        self.count = 0
        self.label = -1
        self.next = self.prev = None

    @classmethod    
    def iterate_forward(cls, head):
        i = head
        while i != None:
            yield i
            i = i.next

    @classmethod    
    def iterate_reverse(cls, tail):
        i = tail
        while i != None:
            yield i
            i = i.prev
            
    def neighbors(self):
        i = self.prev
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.prev
        i = self.next
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.next

class DBSCANGroup(Group):
    def __init__(self, tup):
        super(DBSCANGroup, self).__init__(tup)
        self.head = self.tail = None

    def add(self, chunks):
        n = Neighbor(findNumber(chunks[args.column]), args.jdelim.join(chunks))

        # Update neighbor counts
        oldest = None
        for nn in Neighbor.iterate_reverse(self.tail):
            if dist(nn.t, n.t) <= args.epsilon:
                n.count += 1
                nn.count += 1
                oldest = nn
            else:
                break

        # No neighbors for the new element, clear out old data
        if n.count == 0:
            self.empty()
        elif oldest.count >= args.min_samples: # Oldest is core
            if oldest.label != -1:
                label = oldest.label
            else:
                label = args.label
                args.label += 1

            for nn in oldest.neighbors():
                if nn.label == -1:
                    nn.label = label

        # Append onto the end of the list<commit_msg>new plot<commit_after>#!/usr/bin/python

import os
import sys
import argparse
import traceback
from input_handling import findNumber
from decimal import Decimal
from group import Group,run_grouping

def dist(tx, ty):
    return abs(tx - ty)
    
class Neighbor(object):
    def __init__(self, t, line):
        self.t = t
        self.line = line
        self.count = 0
        self.label = -1
        self.next = self.prev = None

    @classmethod    
    def iterate_forward(cls, head):
        i = head
        while i != None:
            yield i
            i = i.next

    @classmethod    
    def iterate_reverse(cls, tail):
        i = tail
        while i != None:
            yield i
            i = i.prev
            
    def neighbors(self):
        i = self.prev
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.prev
        i = self.next
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.next

class DBSCANGroup(Group):
    def __init__(self, tup):
        super(DBSCANGroup, self).__init__(tup)
        self.head = self.tail = None

    def add(self, chunks):
        n = Neighbor(findNumber(chunks[args.column]), args.jdelim.join(chunks))

        # Update neighbor counts
        oldest = None
        for nn in Neighbor.iterate_reverse(self.tail):
            if dist(nn.t, n.t) <= args.epsilon:
                n.count += 1
                nn.count += 1
                oldest = nn
            else:
                break

        # No neighbors for the new element, clear out old data
        if n.count == 0:
            self.empty()
        elif oldest.count >= args.min_samples: # Oldest is core
            if oldest.label != -1:
                label = oldest.label
            else:
                label = args.label
                args.label += 1

            n.label = label
            for nn in oldest.neighbors():
                if nn.label == -1:
                    nn.label = label

        # Append onto the end of the list
{'fraction_non_alphanumeric': 0.09301193084976869, 'fraction_numerical': 0.0038957876795714633, 'mean_word_length': 1.4136310223266746, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7438859', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1170, 'n_words': 364}