<jupyter_start><jupyter_text>To avoid or detect overfitting to groundstations conditions (too few ground stations)Let s shuffle Training and test data based on Groundstations idOnly 13 stations, so let's use 10 for training and 3 for testing.<jupyter_code>import random

def tts_gs(X, y, gs, test_size):
    stations = gs["station_id"].unique()
    nb_stations = len(stations) * test_size
    my_randoms = random.sample(list(stations), int(nb_stations))
    filters = gs["station_id"].isin(my_randoms)
    print('Substations selected for testing are : ')
    print(my_randoms)
    return X[~filters], X[filters], y[~filters], y[filters]


def visualize_model_gs(X_train, X_test, y_train, y_test, estimator):
    """
    Test various estimators.
    """ 
    #y = LabelEncoder().fit_transform(y)
    y_train = LabelEncoder().fit_transform(y_train)
    y_test = LabelEncoder().fit_transform(y_test)
    
    model = Pipeline([
     ("ColumnTransformer", ColumnTransformer(
        transformers=[
        ('num', numeric_transformer, numericColumns),
        ('cat', categorical_transformer, categoricalColumns)])),
     ('estimator', estimator)
    ])


    # Instantiate the classification model and visualizer
    visualizer = ClassificationReport(
        model, 
        classes=['Good', 'Not good'], 
        cmap="Reds", 
        support=True,
        size=(600, 360)
    )
    
    #X_train, X_test, y_train, y_test = tts_gs(X, y, gs, test_size=0.20)
    #import pdb; pdb.set_trace()
    visualizer.fit(X_train, y_train)  
    score = visualizer.score(X_test, y_test)
    visualizer.show()
    return visualizer.scores_

X_train, X_test, y_train, y_test = tts_gs(X, y, gs, test_size=0.20)
    
for model in models:
    s= visualize_model_gs(X_train, X_test, y_train, y_test, model)
#Remove all x/y features and keep just "postion in decimal degrees"
#We consider a dimension reduction from location dependant features inside X and Y
#Make also possible to predict in one step the pollution
#That, s not optimal for having good prediction because GIS feature are the most related to our AQI value
# location + weather --> air qual
# versus Location --> GIS variable + weather --> Air Qual
features_noGIS = df[[
    'x',
    'y',
    'dayofweek', 
    'sin_day', 
    'cos_day',
    'sin_year', 
    'cos_year', 
    'TEMP', 
    'cos_wind', 
    'sin_wind', 
    'Wind-Rate', 
    'DEW', 
    'SKY', 
    'VIS', 
    'ATM'
]].astype(np.float64)

features.loc[:,'dayofweek'] = features['dayofweek'].astype('category')

numericColumns = ['x','y',
    'dayofweek', 'sin_day', 'cos_day', 'sin_year', 'cos_year', 
    'TEMP', 'cos_wind', 'sin_wind', 'Wind-Rate', 'DEW', 'SKY', 'VIS',  'ATM'
    ]
categoricalColumns = ['dayofweek']

#features_noGIS.loc[:,'dayofweek'] = features['dayofweek'].astype('category')

X=features_noGIS
#numeric_features = X.select_dtypes(include=['int64', 'float64']).columns
#categorical_features = X.select_dtypes(include=['object']).columns


models = [
    #SVC(gamma='auto'),
    # NuSVC(gamma='auto'), LinearSVC(), 
    #SGDClassifier(max_iter=100, tol=1e-3), 
    #KNeighborsClassifier(), 
    LogisticRegression(solver='lbfgs'), #LogisticRegressionCV(cv=3), 
    BaggingClassifier(), 
    ExtraTreesClassifier(n_estimators=100), 
    RandomForestClassifier(n_estimators=100),
    DecisionTreeClassifier()
]
X_train, X_test, y_train, y_test = tts_gs(X, y, gs, test_size=0.20)
    
for model in models:
    visualize_model_gs(X_train, X_test, y_train, y_test, model)
s
import random

X=features_noGIS
y=labels["polluted"]


def tts_grps(X, y, groups):
    filters = gs["station_id"].isin(groups)
    print('Test substations selected: ')
    print(groups)
    return X[~filters], X[filters], y[~filters], y[filters]
    

stations = list(gs["station_id"].unique())
stations_groups = [[], [], [], [], []]

totStations = len(stations)
for i in range(totStations):
    random_item_from_list = random.choice(stations)
    stations.remove(random_item_from_list)
    stations_groups[i % 5].append(random_item_from_list)

dfScores = pd.DataFrame(
    columns = ['model', 'group1', 'group2', 'group3', 'group4', 'group5', 'Var', 'Mean']
)

for model in models:
    scores = []
    for groups in stations_groups:
        X_train, X_test, y_train, y_test = tts_grps(X, y, groups)
        scores_ = visualize_model_gs(X_train, X_test, y_train, y_test, model)
        score = scores_['f1']['Not good']
        print(" > Score {}".format(score))
        scores.append(score)
    print(" >> Mean score {}".format(sum(scores) / len(stations_groups)))
    mean = sum(scores) / len(stations_groups)
    scores.append(sum((i - mean) ** 2 for i in scores) / len(scores))
    scores.append(mean)
    scores.insert(0, str(model).split("(")[0])
    print(scores)
    dfScores.loc[len(dfScores)] = scores
dfScores.head(7)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14262193867050835, 'fraction_numerical': 0.010084379501955135, 'mean_word_length': 4.036269430051814, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '241903', 'license': 'MIT', 'n_tokens_mistral': 1765, 'n_tokens_neox': 1680, 'n_words': 378, 'path': 'model/exploration/model classif selection.ipynb', 'repo_name': 'georgetown-analytics/Air-Pollution', 'size': '4859'}