<commit_before><commit_msg>add training parts grouping script<commit_after>from __future__ import division, print_function, absolute_import 

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('model', metavar='<parts net file>', type=argparse.FileType('wb'), help='Filename of model file')
parser.add_argument('--log', action='store_true')

args = parser.parse_args()

if args.log:
    from pnet.vzlog import default as vz
import numpy as np
import amitgroup as ag
import sys
ag.set_verbose(True)

from pnet.bernoullimm import BernoulliMM
import pnet
from sklearn.svm import LinearSVC 


layers = [
    #pnet.IntensityThresholdLayer(),
    
    pnet.EdgeLayer(k=5, radius=1, spread='orthogonal', minimum_contrast=0.05),
    #pnet.IntensityThresholdLayer(),
    pnet.PartsLayer(100, (6, 6), settings=dict(outer_frame=0, 
                                              threshold=40, 
                                              samples_per_image=40, 
                                              max_samples=1000000, 
                                              min_prob=0.005,
                                              )),
    
    pnet.ExtensionPartsLayer(num_parts = 100, num_components = 10, part_shape = (12,12),lowerLayerShape = (6,6)), 
    pnet.PoolingLayer(shape=(8,8), strides=(8, 8)),
    #pnet.MixtureClassificationLayer(n_components=1, min_prob=0.0001,block_size=200),
    #pnet.ExtensionPoolingLayer(n_parts = 1000, grouping_type='mixture_model',pooling_type = 'distance', pooling_distance = 5, weights_file = './weights100Hidden_pool8*8.npy',save_weights_file = None, settings = {}),
    pnet.ExtensionPoolingLayer(n_parts = 1000, grouping_type='mixture_model',pooling_type = 'distance', pooling_distance = 5, weights_file = None, save_weights_file = None, settings = {}),
    pnet.SVMClassificationLayer(C=1.0)
]

net = pnet.PartsNet(layers)

digits = range(10)
ims = ag.io.load_mnist('training', selection=slice(10000), return_labels=False)

#print(net.sizes(X[[0]]))

net.train(ims)

sup_ims = []
sup_labels = []
# Load supervised training data
for d in digits:
    ims0 = ag.io.load_mnist('training', [d], selection=slice(10), return_labels=False)
    sup_ims.append(ims0)
    sup_labels.append(d * np.ones(len(ims0), dtype=np.int64))

sup_ims = np.concatenate(sup_ims, axis=0)
sup_labels = np.concatenate(sup_labels, axis=0)

net.train(sup_ims, sup_labels)


net.save(args.model)

if args.log:
    net.infoplot(vz)
    vz.finalize()
{'fraction_non_alphanumeric': 0.12343686970552642, 'fraction_numerical': 0.033884630899556276, 'mean_word_length': 3.4444444444444446, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654742', 'n_tokens_mistral': 860, 'n_tokens_neox': 792, 'n_words': 147}