<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2015 <NAME> <<EMAIL>>
# This file is part of PREDICTOR.
#
# PREDICTOR is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
from itertools import product
from sklearn.svm import SVR
from sklearn.feature_extraction import DictVectorizer
from sklearn.utils import shuffle
from sklearn.cross_validation import KFold
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt
import numpy as np


class Model(object):
    def __init__(self, descriptors, svmparams, nfold=5, repetitions=1, dispcoef=0,
                 fit='rmse', normalize=False, **kwargs):
        self.__sparse = DictVectorizer(sparse=False)
        self.__descriptors = descriptors
        self.__trainparam = dict(nfold=nfold, repetitions=repetitions)

        y, x, _ = descriptors.get(**kwargs)
        self.__sparse.fit(x)
        self.__x, self.__y = self.__sparse.transform(x), np.array(y)
        self.__normalize = normalize
        self.__dispcoef = dispcoef
        self.__fitscore = fit
        self.__crossval(svmparams)

    def setworkpath(self, path):
        self.__descriptors.setpath(path)

    def __splitrange(self, param, dep=0):
        tmp = {}
        fdep = dep
        stepindex = list(range(0, len(param), round(len(param)/10) or 1))
        stepindex.insert(0, -1)
        stepindex.append(len(param))
        for i, j, k in zip(stepindex, stepindex[1:], stepindex[2:]):
            tmp[param[j]], tmpd = self.__splitrange(param[i+1:j] + param[j+1:k], dep=dep+1)
            if tmpd > fdep:
                fdep = tmpd
        return tmp, fdep

    def __crossval(self, svmparams):
        fcount = 0
        depindex = []
        maxdep = []
        for param in svmparams:
            di = {}
            md = 0
            for i in param:
                if i != 'kernel':
                    param[i], di[i] = self.__splitrange(param[i])
                    if di[i] > md:
                        md = di[i]
            depindex.append(di)
            maxdep.append(md)

        bestmodel = dict(model=None, r2=np.inf, rmse=np.inf)
        for param, md, di in zip(svmparams, maxdep, depindex):
            model = dict(model=None, r2=np.inf, rmse=np.inf)
            while True:
                stepmodel = dict(model=None, r2=np.inf, rmse=np.inf)
                tmp = self.__prepareparams(param)
                for i in tmp:
                    fcount += 1
                    print('fit model with params:', i)
                    fittedmodel = self.__fit(i)
                    print('R2 = -%(r2)s\nRMSE = %(rmse)s' % fittedmodel)
                    if fittedmodel[self.__fitscore] < stepmodel[self.__fitscore]:
                        stepmodel = fittedmodel

                if stepmodel[self.__fitscore] < model[self.__fitscore]:
                    model = stepmodel
                    tmp = {}
                    for i, j in model['params'].items():
                        if i == 'kernel':
                            tmp[i] = j
                        elif di[i] < md and not param[i][j]:
                            tmp[i] = param[i]
                        else:
                            tmp[i] = param[i][j]
                    param = tmp
                else:
                    break
            if model[self.__fitscore] < bestmodel[self.__fitscore]:
                bestmodel = model

        print('========\nSVM params %(params)s\nR2 = -%(r2)s\nRMSE = %(rmse)s' % bestmodel)
        print('%s variants checked' % fcount)
        self.__model = bestmodel

    @staticmethod
    def __prepareparams(param):
        tmp = []
        baseparams = [x for x in product(param['C'], param['epsilon'], param['tol'])]
        if param['kernel'] == 'linear':  # u'*v
            tmp.extend([dict(kernel='linear', C=c, epsilon=e, tol=t) for c, e, t in baseparams])
        elif param['kernel'] == 'rbf':  # exp(-gamma*|u-v|^2)
            tmp.extend([dict(kernel='rbf', C=c, epsilon=e, tol=t, gamma=g) for g, (c, e, t)
                        in product(param['gamma'], baseparams)])
        elif param['kernel'] == 'sigmoid':  # tanh(gamma*u'*v + coef0)
            tmp.extend([dict(kernel='sigmoid', C=c, epsilon=e, tol=t, gamma=g, coef0=f) for g, f, (c, e, t)
                        in product(param['gamma'], param['coef0'], baseparams)])
        elif param['kernel'] == 'poly':  # (gamma*u'*v + coef0)^degree
            tmp.extend([dict(kernel='poly', C=c, epsilon=e, tol=t, gamma=g, coef0=f, degree=d)
                        for g, f, d, (c, e, t)
                        in product(param['gamma'], param['coef0'], param['degree'], baseparams)])
        return tmp

    def __fit(self, svmparams):
        models = []
        rmse = []
        r2 = []
        y_pred = np.empty_like(self.__y)
        for i in range(self.__trainparam['repetitions']):
            xs, ys = shuffle(self.__x, self.__y, random_state=i)
            kf = KFold(len(self.__y), n_folds=self.__trainparam['nfold'])
            for train, test in kf:
                x_train, y_train = xs[train], ys[train]
                x_test, y_test = xs[test], ys[test]
                x_min = np.amin(x_train, axis=0)
                x_max = np.amax(x_train, axis=0)

                if self.__normalize:
                    normal = MinMaxScaler()
                    normal.fit(x_train)
                    x_test = normal.transform(x_test)
                    x_train = normal.transform(x_train)
                else:
                    normal = None

                model = SVR(**svmparams)
                model.fit(x_train, y_train)

                y_pred[test] = model.predict(x_test)
                models.append(dict(model=model, x_min=x_min, x_max=x_max, normal=normal))
            print('repetition No %s completed' % (i + 1))
            rmse.append(sqrt(mean_squared_error(ys, y_pred)))
            r2.append(r2_score(ys, y_pred))

        return dict(model=models,
                    rmse=np.mean(rmse) - self.__dispcoef * np.var(rmse),
                    r2=-np.mean(r2) + self.__dispcoef * np.var(r2),
                    params=svmparams)

    def predict(self, structure, **kwargs):
        _, d_x, d_ad = self.__descriptors.get(inputfile=structure, **kwargs)
        res = []
        for model in self.__model['model']:
            x_test = self.__sparse.transform(d_x)
            ad = d_ad and (x_test - model['x_min']).min() >= 0 and (model['x_max'] - x_test).min() >= 0
            if model['normal']:
                x_test = model['normal'](x_test)

            res.append(dict(prediction=model['model'].predict(x_test), domain=ad))
        return res
<commit_msg>modelbuilder release<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2015 <NAME> <<EMAIL>>
# This file is part of PREDICTOR.
#
# PREDICTOR is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
from itertools import product
from sklearn.svm import SVR
from sklearn.feature_extraction import DictVectorizer
from sklearn.utils import shuffle
from sklearn.cross_validation import KFold
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt
import numpy as np


class Model(object):
    def __init__(self, descriptors, svmparams, nfold=5, repetitions=1, dispcoef=0,
                 fit='rmse', normalize=False, **kwargs):
        self.__sparse = DictVectorizer(sparse=False)
        self.__descriptors = descriptors
        self.__trainparam = dict(nfold=nfold, repetitions=repetitions)

        y, x, _ = descriptors.get(**kwargs)
        self.__sparse.fit(x)
        self.__x, self.__y = self.__sparse.transform(x), np.array(y)
        self.__normalize = normalize
        self.__dispcoef = dispcoef
        self.__fitscore = fit
        self.__crossval(svmparams)

    def setworkpath(self, path):
        self.__descriptors.setpath(path)

    def __splitrange(self, param, dep=0):
        tmp = {}
        fdep = dep
        stepindex = list(range(0, len(param), round(len(param)/10) or 1))
        stepindex.insert(0, -1)
        stepindex.append(len(param))
        for i, j, k in zip(stepindex, stepindex[1:], stepindex[2:]):
            tmp[param[j]], tmpd = self.__splitrange(param[i+1:j] + param[j+1:k], dep=dep+1)
            if tmpd > fdep:
                fdep = tmpd
        return tmp, fdep

    def __crossval(self, svmparams):
        fcount = 0
        depindex = []
        maxdep = []
        for param in svmparams:
            di = {}
            md = 0
            for i in param:
                if i != 'kernel':
                    param[i], di[i] = self.__splitrange(param[i])
                    if di[i] > md:
                        md = di[i]
            depindex.append(di)
            maxdep.append(md)

        print('========================================\n'
              'Y mean +- variance = %f +- %f\n'
              '  max = %f, min = %f' % (self.__y.mean(), self.__y.var(), self.__y.max(), self.__y.min()))

        bestmodel = dict(model=None, r2=np.inf, rmse=np.inf)
        for param, md, di in zip(svmparams, maxdep, depindex):
            model = dict(model=None, r2=np.inf, rmse=np.inf)
            while True:
                stepmodel = dict(model=None, r2=np.inf, rmse=np.inf)
                tmp = self.__prepareparams(param)
                for i in tmp:
                    fcount += 1
                    print('fit model with params:', i)
                    fittedmodel = self.__fit(i)
                    print('R2 = -%(r2)s\nRMSE = %(rmse)s' % fittedmodel)
                    if fittedmodel[self.__fitscore] < stepmodel[self.__fitscore]:
                        stepmodel = fittedmodel

                if stepmodel[self.__fitscore] < model[self.__fitscore]:
                    model = stepmodel
                    tmp = {}
                    for i, j in model['params'].items():
                        if i == 'kernel':
                            tmp[i] = j
                        elif di[i] < md and not param[i][j]:
                            tmp[i] = param[i]
                        else:
                            tmp[i] = param[i][j]
                    param = tmp
                else:
                    break
            if model[self.__fitscore] < bestmodel[self.__fitscore]:
                bestmodel = model

        print('========================================\n'
              'SVM params %(params)s\nR2 = -%(r2)s\nRMSE = %(rmse)s' % bestmodel)
        print('========================================\n%s variants checked' % fcount)
        self.__model = bestmodel

    @staticmethod
    def __prepareparams(param):
        tmp = []
        baseparams = [x for x in product(param['C'], param['epsilon'], param['tol'])]
        if param['kernel'] == 'linear':  # u'*v
            tmp.extend([dict(kernel='linear', C=c, epsilon=e, tol=t) for c, e, t in baseparams])
        elif param['kernel'] == 'rbf':  # exp(-gamma*|u-v|^2)
            tmp.extend([dict(kernel='rbf', C=c, epsilon=e, tol=t, gamma=g) for g, (c, e, t)
                        in product(param['gamma'], baseparams)])
        elif param['kernel'] == 'sigmoid':  # tanh(gamma*u'*v + coef0)
            tmp.extend([dict(kernel='sigmoid', C=c, epsilon=e, tol=t, gamma=g, coef0=f) for g, f, (c, e, t)
                        in product(param['gamma'], param['coef0'], baseparams)])
        elif param['kernel'] == 'poly':  # (gamma*u'*v + coef0)^degree
            tmp.extend([dict(kernel='poly', C=c, epsilon=e, tol=t, gamma=g, coef0=f, degree=d)
                        for g, f, d, (c, e, t)
                        in product(param['gamma'], param['coef0'], param['degree'], baseparams)])
        return tmp

    def __fit(self, svmparams):
        models = []
        rmse = []
        r2 = []
        y_pred = np.empty_like(self.__y)
        for i in range(self.__trainparam['repetitions']):
            xs, ys = shuffle(self.__x, self.__y, random_state=i)
            kf = KFold(len(self.__y), n_folds=self.__trainparam['nfold'])
            for train, test in kf:
                x_train, y_train = xs[train], ys[train]
                x_test, y_test = xs[test], ys[test]
                x_min = np.amin(x_train, axis=0)
                x_max = np.amax(x_train, axis=0)

                if self.__normalize:
                    normal = MinMaxScaler()
                    normal.fit(x_train)
                    x_test = normal.transform(x_test)
                    x_train = normal.transform(x_train)
                else:
                    normal = None

                model = SVR(**svmparams)
                model.fit(x_train, y_train)

                y_pred[test] = model.predict(x_test)
                models.append(dict(model=model, x_min=x_min, x_max=x_max, normal=normal))
            print('repetition No %s completed' % (i + 1))
            rmse.append(sqrt(mean_squared_error(ys, y_pred)))
            r2.append(r2_score(ys, y_pred))

        return dict(model=models,
                    rmse=np.mean(rmse) - self.__dispcoef * np.var(rmse),
                    r2=-np.mean(r2) + self.__dispcoef * np.var(r2),
                    params=svmparams)

    def predict(self, structure, **kwargs):
        _, d_x, d_ad = self.__descriptors.get(inputfile=structure, **kwargs)
        res = []
        for model in self.__model['model']:
            x_test = self.__sparse.transform(d_x)
            ad = d_ad and (x_test - model['x_min']).min() >= 0 and (model['x_max'] - x_test).min() >= 0
            if model['normal']:
                x_test = model['normal'](x_test)

            res.append(dict(prediction=model['model'].predict(x_test), domain=ad))
        return res

{'fraction_non_alphanumeric': 0.13220451527224436, 'fraction_numerical': 0.008100929614873837, 'mean_word_length': 1.9295856837191208, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176432', 'n_tokens_mistral': 4755, 'n_tokens_neox': 4542, 'n_words': 1152}