<commit_before>import math
import random
import numpy as np

import warnings
#remove matplotlib warning because of fc-cache building
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    import matplotlib.mlab as mlab
    import matplotlib.pyplot as plt

from collections import Counter
from pprint import pprint

#importing my set of functions
from functions import *


BOUND_MAX = 2**8
L_LENGTH = 10000
SL_LENGTH = 10
NB_INSERT = 20
NB_STD = 4

print("generation...")

L   = [random.randrange(0, BOUND_MAX) for i in range(L_LENGTH)]  #the long sequence
SL  = [random.randrange(0, BOUND_MAX) for i in range(SL_LENGTH)] #the shorter sequence
idx = [random.randrange(0, L_LENGTH) for i in range(NB_INSERT)]  #index in the lonsequence to appen the shorter
idx.sort()


for i in idx:
    L = L[:i] + SL + L[i:]

NB_STD = 4

##f = open("/Users/mac/Desktop/scene_detection/cache/L.txt", 'r')
##L = eval(f.read())
##f.close()

################################
# THE COMPRESSION LZW APPROACH #
################################

def index_of_sublist_in_list(lst, sub_lst):
    for i in range(len(lst) - len(sub_lst)):
        if lst[i:i+len(sub_lst)] == sub_lst:
            return i
    raise ValueError("Sub list not found !! :'(")

reconstitued_sequence = cseq(L)

print(reconstitued_sequence)
print("found exactly the same sequence : ", reconstitued_sequence == SL)

############################
# THE STATISTICAL APPROCAH #
############################

print("searching...")

print(len(L))

NB_STD = 4

d = {}
for i in range(len(L) - 2): #-1):#- 2):
    tpl = ((L[i], L[i + 1]), L[i + 2]) #(L[i], L[i + 1]) #((L[i], L[i + 1]), L[i + 2])
    if tpl in d:
        d[tpl] += 1
    else:
        d[tpl] = 1


a = list(sorted(list(d.values())))
a = np.array(a)


#keep number bigger than 6 standard deviation (cf. anomally detection)
#keep keys in d that have a counter bigger than NB_STD * std
dd = {}
for k in d:
    if d[k] >= a.mean() + NB_STD * a.std():
        dd[k] = d[k]

#remap and keep the keys
dd = {(a, b, c) : [] for ((a, b), c) in dd.keys()}

#create all possibles paths
for (a, b, c) in dd:
    dd[(a, b, c)] = filter(lambda t: (b, c) == (t[0], t[1]), dd.keys())

#get the longest path without cycle --> should be the SL (or close to, statistically)
longest = []

visited = set()
path = []
def search(v, d={}):
    visited.add(v)
    path.append(v)
    
    if len(path) > len(longest):
        longest[:] = path[:]
        
    for w in d[v]:
        if w not in visited:
            search(w, d)
            
    path.pop()
    visited.remove(v)

#try all possibilities
for v in dd:
    search(v, dd)

#remake the list of longest
L = list(longest[0])
for t in longest[1:]:
    L.append(t[2])

#print(L == SL)
print(L)
#print(SL)

<commit_msg>Display some infos for testing difference between 2 pattern finding methods...<commit_after>import math
import random
import numpy as np

import warnings
#remove matplotlib warning because of fc-cache building
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    import matplotlib.mlab as mlab
    import matplotlib.pyplot as plt

from collections import Counter
from pprint import pprint

#importing my set of functions
from functions import *


BOUND_MAX = 2**8
L_LENGTH = 10000
SL_LENGTH = 10
NB_INSERT = 12
NB_STD = 4

print("generation...")

L   = [random.randrange(0, BOUND_MAX) for i in range(L_LENGTH)]  #the long sequence
SL  = [random.randrange(0, BOUND_MAX) for i in range(SL_LENGTH)] #the shorter sequence
idx = [random.randrange(0, L_LENGTH) for i in range(NB_INSERT)]  #index in the lonsequence to appen the shorter
idx.sort()


for i in idx:
    L = L[:i] + SL + L[i:]

NB_STD = 4

##f = open("/Users/mac/Desktop/scene_detection/cache/L.txt", 'r')
##L = eval(f.read())
##f.close()

print("sequence solution : ", end='')
print(SL)
print("sequence hash of hashes : ", end='')
h_o_h = get_hash_of_hashes(SL)
print(h_o_h)


################################
# THE COMPRESSION LZW APPROACH #
################################

def index_of_sublist_in_list(lst, sub_lst):
    for i in range(len(lst) - len(sub_lst)):
        if lst[i:i+len(sub_lst)] == sub_lst:
            return i
    raise ValueError("Sub list not found !! :'(")

reconstitued_sequence = cseq(L)
 
print("LZW approcah...")
print(reconstitued_sequence)
h_o_lzw = get_hash_of_hashes(reconstitued_sequence)
print("hash of hahses : ", end='')
print(h_o_lzw)
print("hamming distance between hashes : ", end='')
print(hamming(h_o_h, h_o_lzw))
print("found exactly the same sequence : ", reconstitued_sequence == SL)

############################
# THE STATISTICAL APPROCAH #
############################

print("searching...")

print(len(L))

NB_STD = 4

d = {}
for i in range(len(L) - 2): #-1):#- 2):
    tpl = ((L[i], L[i + 1]), L[i + 2]) #(L[i], L[i + 1]) #((L[i], L[i + 1]), L[i + 2])
    if tpl in d:
        d[tpl] += 1
    else:
        d[tpl] = 1


a = list(sorted(list(d.values())))
a = np.array(a)


#keep number bigger than 6 standard deviation (cf. anomally detection)
#keep keys in d that have a counter bigger than NB_STD * std
dd = {}
for k in d:
    if d[k] >= a.mean() + NB_STD * a.std():
        dd[k] = d[k]

#remap and keep the keys
dd = {(a, b, c) : [] for ((a, b), c) in dd.keys()}

#create all possibles paths
for (a, b, c) in dd:
    dd[(a, b, c)] = filter(lambda t: (b, c) == (t[0], t[1]), dd.keys())

#get the longest path without cycle --> should be the SL (or close to, statistically)
longest = []

visited = set()
path = []
def search(v, d={}):
    visited.add(v)
    path.append(v)
    
    if len(path) > len(longest):
        longest[:] = path[:]
        
    for w in d[v]:
        if w not in visited:
            search(w, d)
            
    path.pop()
    visited.remove(v)

#try all possibilities
for v in dd:
    search(v, dd)

#remake the list of longest
L = list(longest[0])
for t in longest[1:]:
    L.append(t[2])

print("statistical approach...")
print("reconstructed sequence : ", end='')
#print(L == SL)
print(L)
#print(SL)

h_o_s = get_hash_of_hashes(L)
print("reconstitued sequence hash of hashes : ", end='')
print(h_o_s)
print("distance between hashes : ", end='')
print(hamming(h_o_h, h_o_s))
{'fraction_non_alphanumeric': 0.18778207607994843, 'fraction_numerical': 0.010799484203739523, 'mean_word_length': 3.559147685525349, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3159791', 'n_tokens_mistral': 2383, 'n_tokens_neox': 2288, 'n_words': 690}