<commit_before># -*- coding: utf-8 -*-
"""
learnWikipediaPageQuality.py
Created on Wed Jun 11 17:16:22 2014
Learn to predict the quality of Wikipedia pages

@author: ahna
"""

import pandas as pd
import numpy as np
import sklearn
import sklearn.linear_model
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.pipeline import Pipeline
from sklearn.metrics import recall_score, precision_score
import pickle
from qualPred import qualPred
#from app.helpers.qualPred import qualPred
#from qualityPredictor import qualPred
from database import *

def optimizeRegConstant(logres):
    # determine regularization constant. TO DO: make it so it returns best C. also might need to save and restore original C
    Cs = np.linspace(0.01,0.6,20)
    scores = []
    for c in Cs:
        logres.set_params(clf__C=c).fit(X_train, y_train)
        scores.append(logres.score(X_test,y_test))
    print(np.array([Cs,scores]))
    plt.plot(Cs,scores)


def main():
    qp = qualPred() 
    
    ##############################################
    # file names
    #featured_csvfilename = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/datasets/featured.csv'
    #flagged_csvfilename = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/datasets/flagged.csv'
    #qualityPredictorFile = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/data/qualityPredictor.p'
    qualityPredictorFile = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/datasets/qualityPredictorFile.p'
        
    ##############################################
    # load data from CSV file
#    featuredDF = pd.DataFrame().from_csv(featured_csvfilename) # load existing dataframe from file and append to it
#    flaggedDF = pd.DataFrame().from_csv(flagged_csvfilename) # load existing dataframe from file and append to it
#    flaggedDF['featured']=False
#    featuredDF['flags']='NA'
#    featuredDF['flagged']=0
#    featuredDF['featured']=True
#    DF = pd.concat([featuredDF,flaggedDF])

    ##############################################
    # load data from database
    import pandas.io.sql as psql
    conn = conDB()
    DF = psql.frame_query("SELECT * FROM training2", conn)
    closeDB(conn)

    ##############################################
    # set up data matrices
    #useFeatures = [ 'meanSentLen','meanWordLength','medianSentLen','medianWordLength','nChars','nImages','nLinks','nRefs',\
    #'nSections', 'nSents', 'nWordsSummary']
#    useFeatures = ['meanWordLength','nImages','nLinks','nRefs','nSections','nSents', 'nWordsSummary']
    DF = DF[~np.isnan(DF.meanWordLength)] # remove rows with NaN in meanWordLength
    DF = DF[~np.isnan(DF.reading_ease)] # remove rows with NaN in reading_ease  <commit_msg>accounted for new conDB() parameters<commit_after># -*- coding: utf-8 -*-
"""
learnWikipediaPageQuality.py
Created on Wed Jun 11 17:16:22 2014
Learn to predict the quality of Wikipedia pages

@author: ahna
"""

import pandas as pd
import numpy as np
import sklearn
import sklearn.linear_model
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.pipeline import Pipeline
from sklearn.metrics import recall_score, precision_score
import pickle
from qualPred import qualPred
#from app.helpers.qualPred import qualPred
#from qualityPredictor import qualPred
from database import *
configFileName = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/app/settings/development.cfg'
debug, host, port, user, passwd, dbname = grabDatabaseSettingsFromCfgFile(configFileName)

def optimizeRegConstant(logres):
    # determine regularization constant. TO DO: make it so it returns best C. also might need to save and restore original C
    Cs = np.linspace(0.01,0.6,20)
    scores = []
    for c in Cs:
        logres.set_params(clf__C=c).fit(X_train, y_train)
        scores.append(logres.score(X_test,y_test))
    print(np.array([Cs,scores]))
    plt.plot(Cs,scores)


def main():
    qp = qualPred() 
    
    ##############################################
    # file names
    #featured_csvfilename = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/datasets/featured.csv'
    #flagged_csvfilename = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/datasets/flagged.csv'
    #qualityPredictorFile = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/data/qualityPredictor.p'
    qualityPredictorFile = '/Users/ahna/Documents/Work/insightdatascience/project/wikiphilia/webapp/datasets/qualityPredictorFile.p'
        
    ##############################################
    # load data from CSV file
#    featuredDF = pd.DataFrame().from_csv(featured_csvfilename) # load existing dataframe from file and append to it
#    flaggedDF = pd.DataFrame().from_csv(flagged_csvfilename) # load existing dataframe from file and append to it
#    flaggedDF['featured']=False
#    featuredDF['flags']='NA'
#    featuredDF['flagged']=0
#    featuredDF['featured']=True
#    DF = pd.concat([featuredDF,flaggedDF])

    ##############################################
    # load data from database
    import pandas.io.sql as psql
    conn = conDB(host,dbname,passwd=<PASSWORD>,port=port, user=user)
#    conn = conDB()
    DF = psql.frame_query("SELECT * FROM training2", conn)
    closeDB(conn)

    ##############################################
    # set up data matrices
    #useFeatures = [ 'meanSentLen','meanWordLength','medianSentLen','medianWordLength','nChars','nImages','nLinks','nRefs',\
    #'nSections', 'nSents', 'nWordsSummary']
#    useFeatures = ['meanWordLength','nImages','nLinks','nRefs','nSections','nSents', 'nWordsSummary']
    DF = DF[~np.isnan(DF.meanWordLength)] # remove rows with NaN in meanWordLength
    DF = DF[~np.isnan(DF.reading_ease)] # remove rows with NaN in reading_ease  
{'fraction_non_alphanumeric': 0.16481879860996193, 'fraction_numerical': 0.0072811517458216115, 'mean_word_length': 4.913894324853229, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1872715', 'n_tokens_mistral': 1934, 'n_tokens_neox': 1824, 'n_words': 410}