# Path: core/algo/features.py
# class FeatureGenerator(object):
#     def __init__(self, normal_vectorizer=None, clean_vectorizer=None):
#         self.mf_generator = MetaFeatureGenerator()
#         self.fit_complete = False
#         if normal_vectorizer and clean_vectorizer:
#             self.fit_complete = True
# 
#         if normal_vectorizer:
#             self.normal_vectorizer = normal_vectorizer
#         else:
#             self.normal_vectorizer = Vectorizer()
# 
#         if clean_vectorizer:
#             self.clean_vectorizer = clean_vectorizer
#         else:
#             self.clean_vectorizer = Vectorizer()
# 
#     def fit(self, input_text, input_scores):
#         self.normal_vectorizer.fit(input_text, input_scores)
#         clean_text = [self.mf_generator.generate_clean_stem_text(t) for t in input_text]
#         self.clean_vectorizer.fit(clean_text, input_scores)
# 
#     def get_features(self, text):
#         vec_feats = self.generate_vectorizer_features(text)
#         vec_keys = self.normal_vectorizer.vocab + self.clean_vectorizer.vocab
# 
#         meta_feats = self.generate_meta_features(text)
#         meta_keys = meta_feats.keys()
#         meta_keys.sort()
#         meta_feat_arr = np.matrix([meta_feats[k] for k in meta_keys])
# 
#         self.colnames = vec_keys + meta_keys
# 
#         return np.hstack([vec_feats, meta_feat_arr])
# 
#     def generate_meta_features(self, text):
#         feats = self.mf_generator.generate_meta_features(text)
#         return feats
# 
#     def generate_vectorizer_features(self, text):
#         clean_text = self.mf_generator.generate_clean_stem_text(text)
#         feats = self.normal_vectorizer.get_features([text])
#         clean_feats = self.clean_vectorizer.get_features([clean_text])
#         return np.hstack([feats, clean_feats])
# 
# Path: core/database/models.py
# class Model(db.Model):
#     __tablename__ = 'models'
# 
#     id = db.Column(db.Integer, primary_key=True)
#     path = db.Column(db.Text)
# 
#     question_id = db.Column(db.Integer, db.ForeignKey('questions.id'))
#     question = db.relationship("Question", backref=db.backref('models', order_by=id))
# 
#     version = db.Column(db.Integer(), default=settings.MODEL_VERSION)
#     error = db.Column(db.Float)
# 
#     created = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
#     updated = db.Column(db.DateTime(timezone=True), onupdate=datetime.utcnow)
# 
# Path: app.py
# def make_celery(app):
#         def __call__(self, *args, **kwargs):
# def create_app():
# def create_test_app():
#     class ContextTask(TaskBase):
# 
# Path: scan/settings.py
# DB_URL = 'sqlite:///scan.db'
# DEBUG = True
# ROOT_PATH = path(__file__).dirname()
# REPO_PATH = ROOT_PATH.dirname()
# ENV_ROOT = REPO_PATH.dirname()
# DATA_PATH = os.path.join(REPO_PATH, "data")
# TEST_DATA_PATH = os.path.join(DATA_PATH, "test")
# MODEL_PATH = os.path.join(DATA_PATH, "models")
# SECRET_KEY = "30y^$e7henbp@#_w+z9)o9f8tovjrhoq(%vw=%#*(1-1esrh!_"
# USERNAME = "test"
# EMAIL = "test@test.com"
# PASSWORD = "test"
# SQLALCHEMY_DATABASE_URI = DB_URL
# SECURITY_REGISTERABLE = True
# SECURITY_RECOVERABLE = True
# SECURITY_SEND_REGISTER_EMAIL = False
# SECURITY_PASSWORD_HASH = "bcrypt"
# SECURITY_PASSWORD_SALT = SECRET_KEY
# BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600}
# BROKER_URL = 'redis://localhost:6379/3'
# CELERY_RESULT_BACKEND = 'redis://localhost:6379/3'
# CELERY_IMPORTS = ('core.tasks.tasks',)
# CACHE_TYPE = 'redis'
# CACHE_REDIS_URL = 'redis://localhost:6379/4'
# CACHE_REDIS_DB = 4
# MAX_FEATURES = 500
# MODEL_VERSION = 1


import calendar
import numpy as np
import os
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from core.algo.features import FeatureGenerator
from sklearn import cross_validation
from sklearn.externals import joblib
from core.database.models import Model
from app import db
from datetime import datetime
from scan import settings

    def score_essay(self, essay):
        text = essay.text
        model = self.get_latest_model()
        model_obj = joblib.load(os.path.join(settings.MODEL_PATH, model.path))
        return model_obj.predict(text)

    def get_latest_model(self):
        models = self.question.models

        if len(models) == 0:
            raise NoModelException

        model = models[-1]
        return model

    def create_model(self):
        text = [e.text for e in self.question.essays if e.actual_score is not None]
        scores = [e.actual_score for e in self.question.essays if e.actual_score is not None]
        scorer = Scorer(text, scores)
        scorer.train()
        time = datetime.utcnow()
        timestamp = calendar.timegm(time.utctimetuple())
        path_string = "{0}_{1}.pickle".format(self.question.id, timestamp)
        model = Model(
            question=self.question,
            error=scorer.cv_score,
            path=path_string
        )


# Path: core/algo/features.py
# class FeatureGenerator(object):
#     def __init__(self, normal_vectorizer=None, clean_vectorizer=None):
#         self.mf_generator = MetaFeatureGenerator()
#         self.fit_complete = False
#         if normal_vectorizer and clean_vectorizer:
#             self.fit_complete = True
# 
#         if normal_vectorizer:
#             self.normal_vectorizer = normal_vectorizer
#         else:
#             self.normal_vectorizer = Vectorizer()
# 
#         if clean_vectorizer:
#             self.clean_vectorizer = clean_vectorizer
#         else:
#             self.clean_vectorizer = Vectorizer()
# 
#     def fit(self, input_text, input_scores):
#         self.normal_vectorizer.fit(input_text, input_scores)
#         clean_text = [self.mf_generator.generate_clean_stem_text(t) for t in input_text]
#         self.clean_vectorizer.fit(clean_text, input_scores)
# 
#     def get_features(self, text):
#         vec_feats = self.generate_vectorizer_features(text)
#         vec_keys = self.normal_vectorizer.vocab + self.clean_vectorizer.vocab
# 
#         meta_feats = self.generate_meta_features(text)
#         meta_keys = meta_feats.keys()
#         meta_keys.sort()
#         meta_feat_arr = np.matrix([meta_feats[k] for k in meta_keys])
# 
#         self.colnames = vec_keys + meta_keys
# 
#         return np.hstack([vec_feats, meta_feat_arr])
# 
#     def generate_meta_features(self, text):
#         feats = self.mf_generator.generate_meta_features(text)
#         return feats
# 
#     def generate_vectorizer_features(self, text):
#         clean_text = self.mf_generator.generate_clean_stem_text(text)
#         feats = self.normal_vectorizer.get_features([text])
#         clean_feats = self.clean_vectorizer.get_features([clean_text])
#         return np.hstack([feats, clean_feats])
# 
# Path: core/database/models.py
# class Model(db.Model):
#     __tablename__ = 'models'
# 
#     id = db.Column(db.Integer, primary_key=True)
#     path = db.Column(db.Text)
# 
#     question_id = db.Column(db.Integer, db.ForeignKey('questions.id'))
#     question = db.relationship("Question", backref=db.backref('models', order_by=id))
# 
#     version = db.Column(db.Integer(), default=settings.MODEL_VERSION)
#     error = db.Column(db.Float)
# 
#     created = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
#     updated = db.Column(db.DateTime(timezone=True), onupdate=datetime.utcnow)
# 
# Path: app.py
# def make_celery(app):
#         def __call__(self, *args, **kwargs):
# def create_app():
# def create_test_app():
#     class ContextTask(TaskBase):
# 
# Path: scan/settings.py
# DB_URL = 'sqlite:///scan.db'
# DEBUG = True
# ROOT_PATH = path(__file__).dirname()
# REPO_PATH = ROOT_PATH.dirname()
# ENV_ROOT = REPO_PATH.dirname()
# DATA_PATH = os.path.join(REPO_PATH, "data")
# TEST_DATA_PATH = os.path.join(DATA_PATH, "test")
# MODEL_PATH = os.path.join(DATA_PATH, "models")
# SECRET_KEY = "30y^$e7henbp@#_w+z9)o9f8tovjrhoq(%vw=%#*(1-1esrh!_"
# USERNAME = "test"
# EMAIL = "test@test.com"
# PASSWORD = "test"
# SQLALCHEMY_DATABASE_URI = DB_URL
# SECURITY_REGISTERABLE = True
# SECURITY_RECOVERABLE = True
# SECURITY_SEND_REGISTER_EMAIL = False
# SECURITY_PASSWORD_HASH = "bcrypt"
# SECURITY_PASSWORD_SALT = SECRET_KEY
# BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600}
# BROKER_URL = 'redis://localhost:6379/3'
# CELERY_RESULT_BACKEND = 'redis://localhost:6379/3'
# CELERY_IMPORTS = ('core.tasks.tasks',)
# CACHE_TYPE = 'redis'
# CACHE_REDIS_URL = 'redis://localhost:6379/4'
# CACHE_REDIS_DB = 4
# MAX_FEATURES = 500
# MODEL_VERSION = 1


# Path: core/algo/scorer.py
import calendar
import numpy as np
import os
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from core.algo.features import FeatureGenerator
from sklearn import cross_validation
from sklearn.externals import joblib
from core.database.models import Model
from app import db
from datetime import datetime
from scan import settings


    def score_essay(self, essay):
        text = essay.text
        model = self.get_latest_model()
        model_obj = joblib.load(os.path.join(settings.MODEL_PATH, model.path))
        return model_obj.predict(text)

    def get_latest_model(self):
        models = self.question.models

        if len(models) == 0:
            raise NoModelException

        model = models[-1]
        return model

    def create_model(self):
        text = [e.text for e in self.question.essays if e.actual_score is not None]
        scores = [e.actual_score for e in self.question.essays if e.actual_score is not None]
        scorer = Scorer(text, scores)
        scorer.train()
        time = datetime.utcnow()
        timestamp = calendar.timegm(time.utctimetuple())
        path_string = "{0}_{1}.pickle".format(self.question.id, timestamp)
        model = Model(
            question=self.question,
            error=scorer.cv_score,
            path=path_string
        )


        db.session.add(model)