<jupyter_start><jupyter_text>CLASSIFICATION with SOFT LABELS<jupyter_code>### CREATE SYNTHETIC DATA ###

X, y = make_classification(n_samples=3000, n_features=20, n_classes=2, 
                           shuffle=True, random_state=33)

X.shape, y.shape
### UTILITY FUNCTION FOR SOFT RIDGE ###

class SoftRidge(BaseEstimator, ClassifierMixin):
    
    def __init__(
        self,
        alpha = 1.,
        solver = 'auto',
        fit_intercept = True,
        max_iter = None,
        random_state = None
    ):
        self.alpha = alpha
        self.solver = solver
        self.fit_intercept = fit_intercept
        self.max_iter = max_iter
        self.random_state = random_state
        
    def _sigmoid(
        self, y
    ): 
        # expit (from scipy)
        return np.exp(y) / (1 + np.exp(y)) 
    
    def _inv_sigmoid(
        self, y
    ): 
        # logit (from scipy)
        y = y.clip(1e-3, 1 - 1e-3)
        return np.log(y / (1 - y)) 
    
    def fit(
        self, X, y,
        sample_weights = None
    ):
        
        self.model = Ridge(
            alpha = self.alpha, solver = self.solver,
            max_iter = self.max_iter, fit_intercept = self.fit_intercept,
            random_state = self.random_state
        )
        self.model.fit(X, self._inv_sigmoid(y), sample_weights)
        self.classes_ = np.unique(y)
        self.coef_ = self.model.coef_
        return self
        
    def predict(
        self, X
    ):
        pred = self.model.predict(X)  
        return (self._sigmoid(pred) > 0.5).astype(int)
    
    def predict_proba(
        self, X
    ):
        pred = self._sigmoid(self.model.predict(X)) 
        proba = np.zeros((X.shape[0],2))
        proba[:,0] = 1 - pred
        proba[:,1] = pred
        return proba
### CV FIT SOFTRIDGE ###

scores = cross_val_score(SoftRidge(), X, y, cv=5)
probs = cross_val_predict(SoftRidge(), X, y, cv=5, method='predict_proba')

print("CV SCORES => mean: {:.4f} std: {:.4f}".format(scores.mean(), scores.std()))
### PLOT PROBABILITY CALIBRATION CURVE ###

fraction_of_positives, mean_predicted_value = calibration_curve(y, probs[:,1], n_bins=10)

fig = plt.figure(figsize=(10, 10))
ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
ax2 = plt.subplot2grid((3, 1), (2, 0))

ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated")
ax1.plot(mean_predicted_value, fraction_of_positives, "s-")
ax1.set_ylabel("Fraction of positives")
ax1.set_ylim([-0.05, 1.05])
ax1.set_title('Calibration plots  (reliability curve)')

ax2.hist(probs[:,1], range=(0, 1), bins=10, histtype="step", lw=2)
ax2.set_xlabel("Mean predicted value")
ax2.set_ylabel("Count")<jupyter_output><empty_output><jupyter_text>CLASSIFICATION with ARIMA and SOFT LABELS<jupyter_code>### READ DATA ###

air = pd.read_csv('AustraliaIntAirPassengers.csv')
air['change_passengers'] = air.Passengers / air.Passengers.shift()
air['Month'] = pd.to_datetime(air.Month)
air.set_index('Month', inplace=True)
air.dropna(inplace=True)

print(air.shape)
air.head()
### PLOT RAW PASSENGER DATA ###

air['Passengers'].plot(figsize=(15,6))
plt.ylabel('passengers'); plt.title('Monthly passengers')
### PLOT PASSENGER CHANGES ###

air['change_passengers'].plot(figsize=(15,6))
plt.ylabel('changes'); plt.title('Monthly passengers')
plt.axhline(1, linestyle='--', c='black')
### PLOT AUTOCORRELATION IN PASSENGER CHANGES ###

plt.figure(figsize=(15,6))
pd.plotting.autocorrelation_plot(air['change_passengers'])
### PLOT BINARY LABEL DISTRIBUTION ###

air['change_labels'] = (air['change_passengers'] > 1).astype(int)

air['change_labels'].value_counts().plot.pie(figsize=(6,6))
### PLOT BINARY PASSENGER CHANGES ###

air['change_labels'].plot(alpha=0.5, kind='area', figsize=(15,6))
plt.axhline(0.5, linestyle='--', c='black')
plt.ylabel('probabilities'); plt.title('true label')
### TRAIN TEST SPLIT ###

train = air.head(air.shape[0]//2).copy()
test = air[~air.index.isin(train.index)].copy()

train.shape, test.shape
### FIT AND FIND BEST MODEL ###

all_res = {}
best_model = None
best_aic, best_order = np.inf, 0

for p in range(1,15):
    for q in range(0,6):
        
        m = SARIMAX(logit(train.change_labels.clip(1e-3, 1 - 1e-3)), 
                    order=(p,0,q), enforce_invertibility=False) 
        try:
            model = m.fit(disp=False)
            all_res[(p,q)] = model.aic
        except:
            all_res[(p,q)] = np.inf
            
        if all_res[(p,q)] < best_aic:
            best_aic = all_res[(p,q)]
            best_order = (p,q)
            best_model = model
            
print('BEST ORDER', best_order, 'BEST AIC:', best_aic)
best_model.plot_diagnostics(figsize=(12,8))
plt.show()
### PLOT PREDICTED PROBABILITIES VS ACTUAL ###

n_forecast = test.shape[0]
predict = best_model.get_prediction(end=best_model.nobs + n_forecast)
proba = expit(predict.predicted_mean)

air.change_labels.plot(alpha=0.5, kind='area', figsize=(15,6), label='true label')
proba.plot(alpha=0.7, kind='area', label='train prob')
proba[-n_forecast:].plot(alpha=0.7, kind='area', label='test prob')
plt.axhline(0.5, linestyle='--', c='black')
plt.ylabel('probabilities'); plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16270990156340476, 'fraction_numerical': 0.02335456475583864, 'mean_word_length': 3.230204081632653, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '122787', 'license': 'MIT', 'n_tokens_mistral': 2008, 'n_tokens_neox': 1932, 'n_words': 353, 'path': 'Classification_ARIMA/Classification_ARIMA.ipynb', 'repo_name': 'MugiPham/MEDIUM_NoteBook', 'size': '5181'}