<commit_before>#!/usr/bin/env python

import sys

class PermuteFingerprints(object):
    def __init__(self):
        self.fingerprints = set()

    def aslist(self):
        return sorted(list(self.fingerprints))

    def insert(self,s):
        if self.validate(s):
            self.fingerprints.add(s)

    def validate(self, s):
        if len(s) == 0:
            return False

        if len(s) > 5:
            s = s[0:5]

        if 'X' in s:
            return False

        if '11' in s:
            return False

        if 'v1' in s:
            return False

        if 'nv' in s:
            return False

        # all 'ns' in union statements
        if 'U' not in s and 'ns' in s:
            return False

        if 'nn' in s:
            return False

        if 'kk' in s:
            return False
<commit_msg>add fingerprint X<commit_after>#!/usr/bin/env python

import sys

class PermuteFingerprints(object):
    def __init__(self):
        self.fingerprints = set()

    def aslist(self):
        return sorted(list(self.fingerprints))

    def insert(self,s):
        if self.validate(s):
            self.fingerprints.add(s)

    def validate(self, s):
        if len(s) == 0:
            return False

        if len(s) > 5:
            s = s[0:5]

        if '11' in s:
            return False

        if 'v1' in s:
            return False

        if 'nv' in s:
            return False

        # all 'ns' in union statements
        if 'U' not in s and 'ns' in s:
            return False

        if 'nn' in s:
            return False

        if 'kk' in s:
            return False

{'fraction_non_alphanumeric': 0.09820877084620136, 'fraction_numerical': 0.008647313156269302, 'mean_word_length': 1.3964497041420119, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4070432', 'n_tokens_mistral': 526, 'n_tokens_neox': 489, 'n_words': 164}