<commit_before>from SimpleCV.base import *
from SimpleCV.ImageClass import Image
from SimpleCV.DrawingLayer import *
from SimpleCV.FeatureExtractorBase import *
import orange
import os
import glob
import time
"""
This is more or less a hack space right now so I can do a proof of concept
on some of the ML stuff
"""

class SimpleBinaryClassifier:
    mClassAName = "A"
    mClassBName = "B"
    mDataSetRaw = [] 
    mDataSetOrange = []
    mClassifier = None
    mFeatureExtractors = None
    mOrangeDomain = None
    mClassVals = None
    
    def __init__(self,classAName,classBName,featureExtractors):
        self.mClassAName = classAName
        self.mClassBName = classBName
        self.mFeatureExtractors =  featureExtractors
        
    def train(self,pathA,pathB,disp=None,subset=-1):
        count = 0
        files = glob.glob( os.path.join(pathA, '*.jpg'))
        if(subset > 0):
            nfiles = min(subset,len(files))
        else:
            nfiles = len(files)
            
        for i in range(nfiles):
            infile = files[i]
            print "Class A opening file: " + infile
            img = Image(infile)
            featureVector = []
            for extractor in self.mFeatureExtractors:
                featureVector.extend(extractor.extract(img))
            featureVector.extend([self.mClassAName])
            self.mDataSetRaw.append(featureVector)
            text = 'Training: ' + self.mClassAName
            self._WriteText(disp,img,text,Color.WHITE)
            del img
            count = count + 1
            
        files = glob.glob( os.path.join(pathB, '*.jpg'))
        if(subset > 0):
            nfiles = min(subset,len(files))
        else:
            nfiles = len(files)
            
        for i in range(nfiles):
            infile = files[i]
            print "Class B opening file: " + infile
            img = Image(infile)
            featureVector = []
            for extractor in self.mFeatureExtractors:
                featureVector.extend(extractor.extract(img))
            featureVector.extend([self.mClassBName])
            self.mDataSetRaw.append(featureVector)
            text = 'Training: ' + self.mClassBName
            self._WriteText(disp,img,text,Color.WHITE)
            del img
            count = count + 1
        
        colNames = []
        for extractor in self.mFeatureExtractors:
            colNames.extend(extractor.getFieldNames())
        
        self.mClassVals = [self.mClassAName,self.mClassBName]
        self.mOrangeDomain = orange.Domain(map(orange.FloatVariable,colNames),orange.EnumVariable("type",values=self.mClassVals))
        self.mDataSetOrange = orange.ExampleTable(self.mOrangeDomain,self.mDataSetRaw)
        orange.saveTabDelimited ("image_data.tab", self.mDataSetOrange)

        self.mClassifier = orange.BayesLearner(self.mDataSetOrange)
        #svmProto = orange.SVMLearner()
        #svmProto.kernel_type = orange.SVMLearner.Linear
        #svmProto.svm_type = orange.SVMLearner.Nu_SVC
        #svmProto.probability = True
        #svmProto.nu = 0.5
        #self.mClassifier = svmProto(self.mDataSetOrange)
        correct = 0
        incorrect = 0
        for i in range(count):
            
            c = self.mClassifier(self.mDataSetOrange[i])
            test = self.mDataSetOrange[i].getclass()
            print "original", test, "classified as", c 
            if(test==c):
                correct = correct + 1
            else:
                incorrect = incorrect + 1
        print(correct)
        print(incorrect)
        good = 100*(float(correct)/float(count))
        bad = 100*(float(incorrect)/float(count))
        print("Correct: "+str(good))
        print("Incorrect: "+str(bad))
        
    def load(self,path):
        self.mDataSetOrange = orange.ExampleTable(path)
        if(self.mDataSetOrange is not None):
            print('Loading '+path)
            self.mClassifier = orange.BayesLearner(self.mDataSetOrange)
            colNames = []
            for i in range(100):
                c = self.mClassifier(self.mDataSetOrange[i])
                print(c)
                print(self.mDataSetOrange[i].getclass())
                print("\n")
            for extractor in self.mFeatureExtractors:
                colNames.extend(extractor.getFieldNames())
            self.mClassVals = [self.mClassAName,self.mClassBName]
            self.mOrangeDomain = self.mClassifier.domain


    
    def test(self,pathA,pathB,disp=None,subset=-1):
        count = 0
        totalC = 0<commit_msg>NYTM - forgot to get rid of a line<commit_after>from SimpleCV.base import *
from SimpleCV.ImageClass import Image
from SimpleCV.DrawingLayer import *
from SimpleCV.FeatureExtractorBase import *
import orange
import os
import glob
import time
"""
This is more or less a hack space right now so I can do a proof of concept
on some of the ML stuff
"""

class SimpleBinaryClassifier:
    mClassAName = "A"
    mClassBName = "B"
    mDataSetRaw = [] 
    mDataSetOrange = []
    mClassifier = None
    mFeatureExtractors = None
    mOrangeDomain = None
    mClassVals = None
    
    def __init__(self,classAName,classBName,featureExtractors):
        self.mClassAName = classAName
        self.mClassBName = classBName
        self.mFeatureExtractors =  featureExtractors
        
    def train(self,pathA,pathB,disp=None,subset=-1):
        count = 0
        files = glob.glob( os.path.join(pathA, '*.jpg'))
        if(subset > 0):
            nfiles = min(subset,len(files))
        else:
            nfiles = len(files)
            
        for i in range(nfiles):
            infile = files[i]
            print "Class A opening file: " + infile
            img = Image(infile)
            featureVector = []
            for extractor in self.mFeatureExtractors:
                featureVector.extend(extractor.extract(img))
            featureVector.extend([self.mClassAName])
            self.mDataSetRaw.append(featureVector)
            text = 'Training: ' + self.mClassAName
            self._WriteText(disp,img,text,Color.WHITE)
            del img
            count = count + 1
            
        files = glob.glob( os.path.join(pathB, '*.jpg'))
        if(subset > 0):
            nfiles = min(subset,len(files))
        else:
            nfiles = len(files)
            
        for i in range(nfiles):
            infile = files[i]
            print "Class B opening file: " + infile
            img = Image(infile)
            featureVector = []
            for extractor in self.mFeatureExtractors:
                featureVector.extend(extractor.extract(img))
            featureVector.extend([self.mClassBName])
            self.mDataSetRaw.append(featureVector)
            text = 'Training: ' + self.mClassBName
            self._WriteText(disp,img,text,Color.WHITE)
            del img
            count = count + 1
        
        colNames = []
        for extractor in self.mFeatureExtractors:
            colNames.extend(extractor.getFieldNames())
        
        self.mClassVals = [self.mClassAName,self.mClassBName]
        self.mOrangeDomain = orange.Domain(map(orange.FloatVariable,colNames),orange.EnumVariable("type",values=self.mClassVals))
        self.mDataSetOrange = orange.ExampleTable(self.mOrangeDomain,self.mDataSetRaw)
        orange.saveTabDelimited ("image_data.tab", self.mDataSetOrange)

        self.mClassifier = orange.BayesLearner(self.mDataSetOrange)
        #svmProto = orange.SVMLearner()
        #svmProto.kernel_type = orange.SVMLearner.Linear
        #svmProto.svm_type = orange.SVMLearner.Nu_SVC
        #svmProto.probability = True
        #svmProto.nu = 0.5
        #self.mClassifier = svmProto(self.mDataSetOrange)
        correct = 0
        incorrect = 0
        for i in range(count):
            
            c = self.mClassifier(self.mDataSetOrange[i])
            test = self.mDataSetOrange[i].getclass()
            print "original", test, "classified as", c 
            if(test==c):
                correct = correct + 1
            else:
                incorrect = incorrect + 1
        print(correct)
        print(incorrect)
        good = 100*(float(correct)/float(count))
        bad = 100*(float(incorrect)/float(count))
        print("Correct: "+str(good))
        print("Incorrect: "+str(bad))
        
    def load(self,path):
        self.mDataSetOrange = orange.ExampleTable(path)
        if(self.mDataSetOrange is not None):
            print('Loading '+path)
            self.mClassifier = orange.BayesLearner(self.mDataSetOrange)
            colNames = []
            for extractor in self.mFeatureExtractors:
                colNames.extend(extractor.getFieldNames())
            self.mClassVals = [self.mClassAName,self.mClassBName]
            self.mOrangeDomain = self.mClassifier.domain


    
    def test(self,pathA,pathB,disp=None,subset=-1):
        count = 0
        totalC = 0
{'fraction_non_alphanumeric': 0.09846360883705282, 'fraction_numerical': 0.00504654031624986, 'mean_word_length': 2.048888888888889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5433768', 'n_tokens_mistral': 2676, 'n_tokens_neox': 2647, 'n_words': 515}