<commit_before>__author__ = 'Amin'

import numpy as np
import matplotlib.pyplot as plt

from skimage import exposure
from skimage import data
from skimage.feature import local_binary_pattern
from skimage import img_as_float
from skimage.color import rgb2gray
from skimage import img_as_ubyte

import glob

import time

from sklearn.preprocessing import normalize


def calculate_histogram(image, number_of_bins):
    # different ways to calculate histogram
    histogram_from_np, bins_from_np = np.histogram(image, bins=number_of_bins)
    histogram_from_scimage, bins_from_scimage = exposure.histogram(image, nbins=number_of_bins)

    print histogram_from_np
    print histogram_from_scimage

    # how to normalize
    #normalized_histogram = normalize(img_as_float(histogram_from_np[:, np.newaxis]), axis=0).ravel()

# settings for LBP
radius = 3
n_points = 8 * radius
METHOD = 'uniform'

# settings for histograms
number_of_image_bins = 255
number_of_lbp_bins = 25

# plot histograms of LBP of textures
fig, (ax_img, ax_hist) = plt.subplots(nrows=2, ncols=2, figsize=(9, 6))
plt.gray()
plt.ion()
plt.show()

files_directory = "F:\\Amin\\Desktop\\INRIAPerson\\"

# version 1
# load each image in directory
#png_filenames = glob.glob(files_directory + "Train\\pos\\*.png")
png_filenames = glob.glob(files_directory + "96X160H96\\Train\\pos\\*.png")
for filename in png_filenames[:5]:
    print filename

# version 2
# load each image listed in specified text file
#f = open(files_directory + "Train\\pos.lst", "r")
#lines_from_file = f.read().split("\n")
#image_from_file = None
#for line in lines_from_file[:5]:
    #print files_directory + line.replace("/", "\\") + "\n"
    #filename = files_directory + line.replace("/", "\\")

    image_from_file = img_as_ubyte(data.imread(filename, as_grey=True))
    lbp_image = local_binary_pattern(image_from_file, n_points, radius, METHOD)

    calculate_histogram(lbp_image, number_of_lbp_bins)

    image_set = (image_from_file, lbp_image)
    bins_set = (number_of_image_bins, number_of_lbp_bins)

    for ax, image in zip(ax_img, image_set):
        ax.imshow(image)

    for ax, image, number_of_bins in zip(ax_hist, image_set, bins_set):
        ax.cla()
        n, bins, patches = ax.hist(image.ravel(), normed=False, bins=number_of_bins, range=(0, number_of_bins))
        print n, bins, patches, "\n"

    plt.draw()

    plt.pause(0.0001)
    plt.waitforbuttonpress()

    #time.sleep(0.05)
    #raw_input("Press Enter to continue...")


#original_image = img_as_ubyte(data.imread("crop_000001a.png", as_grey=True))
#original_image = img_as_ubyte(data.imread("F:\\Amin\\Desktop\\INRIAPerson\\Train\\pos\\person_and_bike_209.png", as_grey=True))
#original_image = img_as_ubyte(rgb2gray(data.camera()))
#original_image = img_as_ubyte(rgb2gray(data.lena()))







<commit_msg>Some minor clean-ups. Also added part responsible for generating negative samples.<commit_after>__author__ = 'Amin'

import numpy as np
import matplotlib.pyplot as plt

from skimage import exposure
from skimage import data
from skimage.feature import local_binary_pattern
from skimage import img_as_float
from skimage.color import rgb2gray
from skimage import img_as_ubyte
from skimage.viewer import ImageViewer
from skimage.io import imshow
from skimage.io import show

import glob

import time

import random

from sklearn.preprocessing import normalize

# this is test function for checking different ways of calculating the histogram
def calculate_histogram(image, number_of_bins):
    # different ways to calculate histogram
    histogram_from_np, bins_from_np = np.histogram(image, bins=number_of_bins)
    histogram_from_scimage, bins_from_scimage = exposure.histogram(image, nbins=number_of_bins)
    # important - the image data has to be processed by ravel method
    histogram_from_matplotlib, bins_from_matplotlib, patches_from_matplotlib = plt.hist(image.ravel(), bins=number_of_bins, normed=False)

    # all of these methods return same result
    print histogram_from_np
    print histogram_from_scimage
    print histogram_from_matplotlib

    # how to normalize
    #normalized_histogram = normalize(img_as_float(histogram_from_np[:, np.newaxis]), axis=0).ravel()
    # this can also be done as a parameter for matplotlib hist method

# settings for LBP
radius = 3
n_points = 8 * radius
METHOD = 'uniform'

# settings for histograms
number_of_image_bins = 255
number_of_lbp_bins = 25

fig, (ax_img, ax_hist) = plt.subplots(nrows=2, ncols=2, figsize=(9, 6))
plt.gray()
plt.ion()
plt.show()

train_histogram_positive = []
train_histogram_negative = []

files_directory = "F:\\Amin\\Desktop\\INRIAPerson\\"

# version 1
# load each image in directory
#png_filenames = glob.glob(files_directory + "Train\\pos\\*.png")
png_filenames = glob.glob(files_directory + "96X160H96\\Train\\pos\\*.png")
for filename in png_filenames[:1]:
    print filename

# version 2
# load each image listed in specified text file
#f = open(files_directory + "Train\\pos.lst", "r")
#lines_from_file = f.read().split("\n")
#image_from_file = None
#for line in lines_from_file[:5]:
    #print files_directory + line.replace("/", "\\") + "\n"
    #filename = files_directory + line.replace("/", "\\")

    image_from_file = img_as_ubyte(data.imread(filename, as_grey=True))
    lbp_image = local_binary_pattern(image_from_file, n_points, radius, METHOD)

    #calculate_histogram(lbp_image, number_of_lbp_bins)

    # showing the results in the window
    image_set = (image_from_file, lbp_image)
    bins_set = (number_of_image_bins, number_of_lbp_bins)

    lbp_histogram, lbp_bins, lbp_patches = plt.hist(lbp_image.ravel(), bins=number_of_lbp_bins, normed=False)

    train_histogram_positive.append(lbp_histogram)

    for ax, image in zip(ax_img, image_set):
        ax.imshow(image)

    for ax, image, number_of_bins in zip(ax_hist, image_set, bins_set):
        ax.cla()
        ax.hist(image.ravel(), normed=False, bins=number_of_bins, range=(0, number_of_bins))

    plt.draw()

    plt.pause(0.0001)
    plt.waitforbuttonpress()

    #time.sleep(0.05)
    #raw_input("Press Enter to continue...")

# prepare negative examples
# version 1
# load each image in directory
#png_filenames = glob.glob(files_directory + "Train\\pos\\*.png")
png_filenames = glob.glob(files_directory + "\\Train\\neg\\*.png")
for filename in png_filenames:
    image_from_file = img_as_ubyte(data.imread(filename, as_grey=True))
    # we have to extract the patch of size 96x160 so negative samples will be the same as positive
    width = 96
    height = 160
    # choose left upper corner
    nrows, ncols = image_from_file.shape
    #print nrows, ncols
    x = random.randint(0, nrows-1-width)
    y = random.randint(0, ncols-1-height)
    #print x, y
    part_of_image = image_from_file[x:x+width, y:y+height]
    #print part_of_image
    #viewer = ImageViewer(part_of_image)
    #viewer.show()
    lbp_image = local_binary_pattern(image_from_file, n_points, radius, METHOD)

    lbp_histogram, lbp_bins, lbp_patches = plt.hist(lbp_image.ravel(), bins=number_of_lbp_bins, normed=False)

    train_histogram_negative.append(lbp_histogram)

print len(train_histogram_positive)
print len(train_histogram_negative)

#original_image = img_as_ubyte(data.imread("crop_000001a.png", as_grey=True))
#original_image = img_as_ubyte(data.imread("F:\\Amin\\Desktop\\INRIAPerson\\Train\\pos\\person_and_bike_209.png", as_grey=True))
#original_image = img_as_ubyte(rgb2gray(data.camera()))
#original_image = img_as_ubyte(rgb2gray(data.lena()))








{'fraction_non_alphanumeric': 0.10904995362395654, 'fraction_numerical': 0.013647807075659203, 'mean_word_length': 4.744292237442922, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4178267', 'n_tokens_mistral': 2805, 'n_tokens_neox': 2591, 'n_words': 593}