<commit_before>import os, sys, re
import numpy as np
import pandas as pd
import skimage.io
import skimage.transform
from sklearn.preprocessing import LabelBinarizer
from keras.applications import InceptionV3
from keras.applications import imagenet_utils
from keras.models import Model
from keras.layers import Input, Dense, Flatten, Dropout, Reshape
from keras import backend as K
from sklearn.externals import joblib

def extract_features():

    # Load dataset (any image-based dataset)
    matches = [(re.match(r'^(([a-zA-Z]+)\d+\.png)', fname), path) 
	    for path, dirs, files in os.walk('./datasets/ucm') for fname in files]
    patches = [skimage.transform.resize( # resize image to (256, 256) TODO argv param based on patch size
	    skimage.io.imread(os.path.join(path, match.group(1))), # open each image
    	    (256, 256)) for match, path in matches if match]
	
    # change color channel order and shift mean color for ImageNet		
    patches = imagenet_utils.preprocess_input(np.array(patches)) 
    labels = [match.group(2) for match, path in matches if match]
    labels = LabelBinarizer().fit_transform(list(labels)) # one-hot encoding
    print('patches', patches[0].shape, len(patches), 'labels', len(labels))

    # Construct model (using ImageNet weights)
    inceptionV3 = InceptionV3(weights = "imagenet", include_top = False, 
        input_shape = patches[0].shape)
	
    #for layer in inceptionV3.layers:
    #    layer.trainable = False

    x = inceptionV3.output
    x = Flatten()(x)
    x = Dense(1024, activation="relu")(x)
    x = Dropout(0.5)(x)
    x = Dense(1024, activation="relu")(x)
    extractor = Model(inputs=[inceptionV3.input],outputs=[x])
    features = extractor.predict(x=patches, batch_size=2)
    return features

features = extract_features()
print("\n[INFO] Output array shape:", features.shape)
np.savetxt('np_test.csv', features, delimiter=',')<commit_msg>> %f formatter for output .csv, removed bad imports<commit_after>import os, sys, re
import numpy as np
import skimage.io
import skimage.transform
from sklearn.preprocessing import LabelBinarizer
from keras.applications import InceptionV3
from keras.applications import imagenet_utils
from keras.models import Model
from keras.layers import Input, Dense, Flatten, Dropout, Reshape

def extract_features():

    # Load dataset (any image-based dataset)
    matches = [(re.match(r'^(([a-zA-Z]+)\d+\.png)', fname), path) 
	    for path, dirs, files in os.walk('./datasets/ucm') for fname in files] # TODO argv directory
    patches = [skimage.transform.resize( # resize image to (256, 256) TODO argv patch size?
	    skimage.io.imread(os.path.join(path, match.group(1))), # open each image
    	    (256, 256)) for match, path in matches if match]
	
    # change color channel order and shift mean color for ImageNet		
    patches = imagenet_utils.preprocess_input(np.array(patches)) 
    labels = [match.group(2) for match, path in matches if match]
    labels = LabelBinarizer().fit_transform(list(labels)) # one-hot encoding
    print('patches', patches[0].shape, len(patches), 'labels', len(labels))

    # Construct model (using ImageNet weights)
    inceptionV3 = InceptionV3(weights = "imagenet", include_top = False, 
        input_shape = patches[0].shape)
	
    #for layer in inceptionV3.layers:
    #    layer.trainable = False

    x = inceptionV3.output
    x = Flatten()(x)
    x = Dense(1024, activation="relu")(x)
    x = Dropout(0.5)(x)
    x = Dense(1024, activation="relu")(x)
    extractor = Model(inputs=[inceptionV3.input],outputs=[x])
    features = extractor.predict(x=patches, batch_size=2)
    return features

features = extract_features()
# print("\n[INFO] Output array shape:", features.shape)
np.savetxt('features.csv', features, fmt='%f') # TODO argv filename

{'fraction_non_alphanumeric': 0.1214436248682824, 'fraction_numerical': 0.01738672286617492, 'mean_word_length': 4.158967391304348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1906898', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1192, 'n_words': 348}