<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon May 26 11:00:57 2014
Detect_feature
detect the multi-scale LBP feature of the face images
@author: mountain
"""
import os

from LBP import *
import cv2
import pickle
from util.commons_util.logger_utils.Timer import Timer

# the radius in LBP
radius = 1
# the number of the neighbors in LBP
nei = 8
# the scale in Multi-scale LBP
scale = 4
# the ratio in scaling
scale_step = 1.25
# the patch size for getting hist
winsize = int((25 - 1) / 2)
# the stride between the patch in getting hist
stride = 2
# under this condition, the feature dimension is 816*236

# dir_path: the path of the folder containing the image
dir_path = 'E:\\GPforFR\\data\\lfw_p'

# dst_path: the path of the file saving the feature
dst_path = 'E:\\GPforFR\\data\\lfw_feature'

timer = Timer()
# Traversing files
for root, dirs, files in os.walk(dir_path):
    if files:
        # get the image name
        img_name = root.split('\\')[-1]

        for f in files:
            timer.start()
            image = cv2.imread(root + '\\' + f)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            ftr_name = os.path.join(dst_path, f.split('.')[0] + '.txt')
            # if not os.path.exists(ftr_name):

            feature = Mulscl_lbp_feature(image, radius, nei, scale, scale_step, winsize, stride)

            output = open(ftr_name, 'w')
            pickle.dump(feature, output)
            print timer.end()
            output.close()
    print root<commit_msg>add worker pool to speed up feature extraction<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon May 26 11:00:57 2014
Detect_feature
detect the multi-scale LBP feature of the face images
@author: mountain
"""
import os

from LBP import *
import cv2
import pickle
from util.commons_util.logger_utils.Timer import Timer
from multiprocessing import Pool

# the radius in LBP
radius = 1
# the number of the neighbors in LBP
nei = 8
# the scale in Multi-scale LBP
scale = 4
# the ratio in scaling
scale_step = 1.25
# the patch size for getting hist
winsize = int((25 - 1) / 2)
# the stride between the patch in getting hist
stride = 2
# under this condition, the feature dimension is 816*236

# dir_path: the path of the folder containing the image
dir_path = 'E:\\GPforFR\\data\\lfw_p'

# dst_path: the path of the file saving the feature
dst_path = 'E:\\GPforFR\\data\\lfw_feature'


def extract_feature(args):
    image, ftr_name = args
    timer = Timer()
    out_file = open(ftr_name, 'w')
    timer.start()
    feature = Mulscl_lbp_feature(image, radius, nei, scale, scale_step, winsize, stride)
    pickle.dump(feature, out_file)
    out_file.close()
    print timer.end()

if __name__=="__main__":
    # Traversing files
    p = Pool(5)
    for root, dirs, files in os.walk(dir_path):
        if files:
            # get the image name
            img_name = root.split('\\')[-1]

            load = []
            for f in files:
                image = cv2.imread(root + '\\' + f)
                image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

                ftr_name = os.path.join(dst_path, f.split('.')[0] + '.txt')
                # if not os.path.exists(ftr_name):
                load.append((image, ftr_name))

            p.map(extract_feature, load)
        print root

{'fraction_non_alphanumeric': 0.10832072617246596, 'fraction_numerical': 0.0226928895612708, 'mean_word_length': 2.6570796460176993, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1858899', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1075, 'n_words': 358}