<jupyter_start><jupyter_text>Note: this example is inferior and has been removed from the paper and replace by a much better demonstration foundhere: https://github.com/WinVector/pyvtreat/blob/master/Examples/OrderedCrossPlan/OrderedExample.ipynbTranslating the R sequences from https://arxiv.org/abs/1611.09477 into Python vtreat https://github.com/WinVector/pyvtreat R original<jupyter_code>## ----LookFTData, echo=TRUE, warning=FALSE, message=FALSE-----------------
library("Amelia")
data("freetrade")
str(freetrade)<jupyter_output><empty_output><jupyter_text>Python translation<jupyter_code>import pandas
import numpy
import numpy.random
import sklearn.linear_model
import vtreat # https://github.com/WinVector/pyvtreat
import vtreat.cross_plan
import vtreat.util
freetrade = pandas.read_csv('freetrade.csv')
freetrade.head()
freetrade.dtypes<jupyter_output><empty_output><jupyter_text>R original<jupyter_code>## ----LookFTData2, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
library("ggplot2")
compRsq <- function(d, x, y) {
  meany <- mean(d[[y]])
  rsq <- 1-sum((d[[y]]-d[[x]])^2)/sum((d[[y]]-meany)^2)
  rsq
}
plotFit <- function(d, x, y, title) {
  rsq <- compRsq(d, x, y)
  ggplot(data=d, aes_string(x=x, y=y)) +
    geom_abline(color='blue') +
    geom_point() +
    ggtitle(paste(title, '\n', 'R-squared:', 
                  format(rsq, scientific=FALSE, digits=3)))
}
plotFrame <- freetrade
sortFrame <- plotFrame[plotFrame$year==1989, c('country', 'gdp.pc')]
orderedLevels <- sortFrame$country[order(-sortFrame$gdp.pc)]
plotFrame$country <- factor(plotFrame$country, orderedLevels)
ggplot(data=plotFrame, aes(x=year, y=gdp.pc, color=country, linetype=country)) +
     geom_point() + geom_line()<jupyter_output><empty_output><jupyter_text>Python version<jupyter_code>import seaborn

def compRsq(d, x, y):
    meany = numpy.mean(d[y])
    rsq = 1-numpy.sum((d[y]-d[x])**2)/numpy.sum((d[y]-meany)**2)
    return rsq


def plotFit(d, x, y, title):
    rsq = compRsq(d, x, y)
    title = title + "\nR-squared: " + '{:.3f}'.format(rsq)
    seaborn.scatterplot(x=x, y=y, data=d).set_title(title)
plotFrame = freetrade
sortFrame = plotFrame.loc[plotFrame['year']==1989, ]
country_order = [c for c in sortFrame.groupby('country')[['gdp.pc']].mean().sort_values(by=['gdp.pc']).index]
country_order.reverse()
seaborn.lineplot(x='year', y='gdp.pc', data=plotFrame, hue='country',
                hue_order=country_order)
plotFrame.groupby('country')[['gdp.pc']].mean().sort_values(by=['gdp.pc']).index<jupyter_output><empty_output><jupyter_text>R original<jupyter_code>## ----LookFTDefGoal1------------------------------------------------------
trainData <- freetrade[freetrade$year<1990, ]
testData <- freetrade[freetrade$year>=1990, ]
origVars <- c('tariff', 'polity', 'pop', 'year', 'country')


## ----ModelPast2, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
paired <- merge(freetrade[freetrade$year==1989, c('country', 'year', 'gdp.pc')], 
               freetrade[freetrade$year>=1990, c('country', 'year', 'gdp.pc')], 
               by='country', suffixes=c('.1989', '.1990.and.after'))
plotFit(paired, 'gdp.pc.1989', 'gdp.pc.1990.and.after', 
                     '1990 and later gdp as a function of 1989 gdp, grouped by country')<jupyter_output><empty_output><jupyter_text>Python version<jupyter_code>## ----LookFTDefGoal1------------------------------------------------------
trainData = freetrade.loc[freetrade['year']<1990, :].copy()
trainData.reset_index(inplace=True, drop=True)
testData = freetrade.loc[freetrade['year']>=1990, :].copy()
testData.reset_index(inplace=True, drop=True)
origVars = ['tariff', 'polity', 'pop', 'year', 'country']
## ----ModelPast2, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
paired = freetrade.loc[freetrade['year']==1989, ['country', 'year', 'gdp.pc']].merge(
               freetrade.loc[freetrade['year']>=1990, ['country', 'year', 'gdp.pc']], 
               on='country', 
               suffixes=['.1989', '.1990.and.after'])
plotFit(paired, 'gdp.pc.1989', 'gdp.pc.1990.and.after', 
                     '1990 and later gdp as a function of 1989 gdp, grouped by country')<jupyter_output><empty_output><jupyter_text>R original<jupyter_code>## ----ModelFTTry1, warning=FALSE, message=FALSE, results='hide'-----------
library("vtreat")
treatments <- designTreatmentsN(trainData, origVars, 'gdp.pc')
scoreFrame <- treatments$scoreFrame


## ----ModelFTTry1p2, warning=FALSE, message=FALSE-------------------------
modelingVars <- unique(c('year', 
   scoreFrame$varName[(scoreFrame$sig < 1 / nrow(scoreFrame)) &
    !(scoreFrame$code %in% c('lev'))]))
print(modelingVars)
trainTreated <- prepare(treatments, trainData, 
                       pruneSig=NULL, varRestriction=modelingVars)
testTreated <- prepare(treatments, testData, 
                       pruneSig=NULL, varRestriction=modelingVars)
formula <- paste('gdp.pc', paste(modelingVars, collapse=' + '), sep=' ~ ')
print(strwrap(formula))
model <- lm(as.formula(formula), data=trainTreated)
testTreated$pred <- predict(model, newdata=testTreated)


## ----ModelFTTry1plot, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
plotFit(testTreated, 'pred', 'gdp.pc', 
        'test gdp naively reusing training data')<jupyter_output><empty_output><jupyter_text>Python version<jupyter_code>## ----ModelFTTry1, warning=FALSE, message=FALSE, results='hide'-----------
treatments = vtreat.NumericOutcomeTreatment(
    outcome_name='gdp.pc',
    params=vtreat.vtreat_parameters({'filter_to_recommended': False}))
trainTreated = treatments.fit_transform(trainData[origVars + ['gdp.pc']], trainData['gdp.pc'])
scoreFrame = treatments.score_frame_
scoreFrame
trainTreated.head()
## ----ModelFTTry1p2, warning=FALSE, message=FALSE-------------------------
modelingVars = [ scoreFrame['variable'][i] for i in range(scoreFrame.shape[0]) if 
                (scoreFrame['recommended'][i] and (not scoreFrame['treatment'][i]=='indicator_code'))]
if not 'year' in set(modelingVars):
    modelingVars = modelingVars + ['year']
print(modelingVars)
trainTreated = trainTreated[modelingVars + ['gdp.pc']].copy()
trainTreated.reset_index(inplace=True, drop=True)
testTreated = treatments.transform(testData)[modelingVars + ['gdp.pc']].copy()
model = sklearn.linear_model.LinearRegression()
testTreated.reset_index(inplace=True, drop=True)
model.fit(trainTreated[modelingVars], trainTreated['gdp.pc'])
testTreated['pred'] = model.predict(testTreated[modelingVars])
testTreated.head()
## ----ModelFTTry1plot, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
plotFit(testTreated, 'pred', 'gdp.pc', 
        'test gdp naively reusing training data (on test)')<jupyter_output><empty_output><jupyter_text>R original<jupyter_code>## ----ModelFTTry3, warning=FALSE, message=FALSE---------------------------
timeOrderedSplitter <- function(nRows, nSplits, dframe, y) {
  years <- sort(unique(dframe$year))
  splits <- lapply(years, function(y) {
    list(train=which(dframe$year<y), 
         app=which(dframe$year>=y))
  })
  Filter(function(si) {
          (length(si$train)>0) && (length(si$app)>0)
         }, 
         splits)
}


## ----ModelFTTry3b, warning=FALSE, message=FALSE--------------------------
cfe <- mkCrossFrameNExperiment(trainData, origVars, 'gdp.pc', 
                               splitFunction=timeOrderedSplitter)
print(cfe$method)
treatments <- cfe$treatments
scoreFrame <- treatments$scoreFrame
modelingVars <- unique(c('year', 
   scoreFrame$varName[(scoreFrame$sig < 1 / nrow(scoreFrame)) &
    !(scoreFrame$code %in% c('lev'))]))
print(modelingVars)
trainTreated <- cfe$crossFrame
testTreated <- prepare(treatments, testData, 
                       pruneSig=NULL, varRestriction=modelingVars)
formula <- paste('gdp.pc', paste(modelingVars, collapse=' + '), sep=' ~ ')
print(strwrap(formula))
model <- lm(as.formula(formula), data=trainTreated)
testTreated$pred <- predict(model, newdata=testTreated)


## ----ModelFTTry3bplot, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
plotFit(testTreated, 'pred', 'gdp.pc', 
        'test gdp using time ordered training split')<jupyter_output><empty_output><jupyter_text>Python version<jupyter_code>## ----ModelFTTry3, warning=FALSE, message=FALSE---------------------------
# a deliberately incomplete cross-plan (no app group for first year)
# need to drop those rows before working with them
class TimeOrderedSplitter(vtreat.cross_plan.CrossValidationPlan):
    """time oriented cross validation plan"""

    def __init__(self):
        vtreat.cross_plan.CrossValidationPlan.__init__(self)

    def split_plan(self, *, n_rows=None, k_folds=None, data=None, y=None):
       years = [y for y in set(data['year'])]
       years.sort()
       y0 = years[0]
       splits = [{'train':[i for i in range(data.shape[0]) if (data['year'][i]<yr)],
                  'app':[i for i in range(data.shape[0]) if data['year'][i]==yr]} for yr in years if yr>y0] 
       return splits
## ----ModelFTTry3b, warning=FALSE, message=FALSE--------------------------
treatments = vtreat.NumericOutcomeTreatment(
    outcome_name='gdp.pc',
    params=vtreat.vtreat_parameters({'filter_to_recommended': False,
                                     'cross_validation_plan': TimeOrderedSplitter()}))
trainTreated = treatments.fit_transform(trainData[origVars + ['gdp.pc']], trainData['gdp.pc'])
scoreFrame = treatments.score_frame_
modelingVars = [ scoreFrame['variable'][i] for i in range(scoreFrame.shape[0]) if 
                (scoreFrame['recommended'][i] and (not scoreFrame['treatment'][i]=='indicator_code'))]
if not 'year' in set(modelingVars):
    modelingVars = modelingVars + ['year']
print(modelingVars)

trainTreated = trainTreated[modelingVars + ['gdp.pc']].copy()
supported_rows = vtreat.cross_plan.support_indicator(trainTreated.shape[0], 
                                               treatments.cross_plan_)
print(numpy.mean(supported_rows))
trainTreated = trainTreated.loc[supported_rows, :].copy()
trainTreated.reset_index(inplace=True, drop=True)

testTreated = treatments.transform(testData)[modelingVars + ['gdp.pc']].copy()
model = sklearn.linear_model.LinearRegression()
model.fit(trainTreated[modelingVars], trainTreated['gdp.pc'])
testTreated['pred'] = model.predict(testTreated[modelingVars])
testTreated.head()
## ----ModelFTTry3bplot, echo=FALSE, warning=FALSE, message=FALSE, fig.width=6, fig.height=3----
plotFit(testTreated, 'pred', 'gdp.pc', 
        'test gdp using time ordered training split')
scoreFrame<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1900291052483335, 'fraction_numerical': 0.014177072575345038, 'mean_word_length': 4.2732673267326735, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 49, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '429794', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3782, 'n_tokens_neox': 3576, 'n_words': 615, 'path': 'Examples/vtreat_paper1/Sec04ExampleApplication.ipynb', 'repo_name': 'sdrees/pyvtreat', 'size': '10651'}