<jupyter_start><jupyter_text>For Locations and Cuisines<jupyter_code>#A function to find the maximun number of features in a single cell
def max_features_in_single_row(train_df, test_df, delimiter):
    max_info = 0 
    item_lis = list(train_df.append(test_df))
    for i in item_lis:
        if len(i.split("{}".format(delimiter))) > max_info:
            max_info = len(i.split("{}".format(delimiter)))
            print("\n","-"*35)    
            print("Max_Features in One Observation = ", max_info)
        return max_info
#This function splits a column in to n features where n is the maximum number of features in a single cell
def feature_splitter(feat, name, delimiter, max_info):
    item_lis = list()
    extracted_features = {}
    for i in range(max_info):
        extracted_features['{}_Feature_{}'.format(name, i+1)] = []
        print("-"*35)
        print("Features Dictionary : ", extracted_features)
  #tqdm is a graphics module that helps us see the progress bar
    for i in tqdm(range(len(item_lis))):feat
        for j in range(max_info):  
            try:
                extracted_features['{}_Feature_{}'.format(name,j+1)].append(item_lis[i].split("{}".format(delimiter))[j].lower().strip())
            except: 
                extracted_features['{}_Feature_{}'.format(name, j+1)].append(np.nan)

    return extracted_features 
max_features_in_single_row(train_df['Location'], test_df['Location'],',')
max_features_in_single_row(train_df['Cuisines'], test_df['Cuisines'],',')<jupyter_output>-----------------------------------
Max_Features in One Observation =  5<jupyter_text>Ratings, Votes and ‘Reviews<jupyter_code>#A function to find all the non numeric values
def non_numerals(series):
    non_numerals = []
    for i in series.unique():
        try :
            i = float(i)
        except:
            non_numerals.append(i)
    return non_numerals
non_numerals(train_df['Rating'])
non_numerals(test_df['Rating'])
non_numerals(train_df['Votes'])
non_numerals(test_df['Votes'])
non_numerals(train_df['Reviews'])
non_numerals(test_df['Reviews'])
# A function to replace the non-numeric values
def replace_nn_with(series, type_, fill_with = None, method = 'mean'):
    nn = non_numerals(series)
    print('-'*30)
    print('-'*30)
    print("Non Numerals in column ",series.name," : ",nn)
    series = series.replace(nn, np.nan, inplace = False)
    nulls = series.isnull().sum()
    if fill_with:
        series.fillna(fill_with, inplace = True)
        print("Filling Non Numerals with {}".format(fill_with))
    else:
        series = series.replace(nn, np.nan, inplace = False)

    if method == 'mean' :
        rep = series.astype(float).mean()
        print("Filling Non Numerals with MEAN = ", rep)

    elif method =='median' :
        rep = series.astype(float).median()
        print("Filling Non Numerals with MEDIAN = ", rep)

    elif method =='min' :
        rep = series.astype(float).min()
        print("Filling Non Numerals with MINIMUM = ", rep)
    else:
        print('Please pass a valid method as a string -- ("mean" or "median" or "min")')
    return 0

    series.fillna(rep, inplace = True)
    try:
        series = series.astype(type_)
        print(nulls, ": observations replaced")
        return series
    except:
        series =  series.astype(float)
        print(nulls, ": observations replaced")
        series =  series.astype(type_)
    return series
replace_nn_with(train_df['Rating'], float, fill_with = 3.611078, method = 'mean')
replace_nn_with(train_df['Votes'], float, fill_with = '244', method = 'mean')
replace_nn_with(train_df['Reviews'], float, fill_with = 123, method = 'mean')
train_df['Rating'] = train_df['Rating'].apply(lambda x: x.replace('-', '3.6'))
train_df['Rating'] = train_df['Rating'].apply(lambda x: x.replace('NEW', '3.6'))
train_df['Rating'] = train_df['Rating'].apply(lambda x: x.replace('Opening Soon', '3.6'))
train_df['Rating'] = train_df['Rating'].apply(lambda x: x.replace('Temporarily Closed', '3.6'))
train_df['Votes'] = train_df['Votes'].apply(lambda x: x.replace('-', '244'))
train_df['Reviews'] = train_df['Reviews'].apply(lambda x: x.replace('-', '123'))
replace_nn_with(test_df['Rating'], float, fill_with = None, method = 'mean')
replace_nn_with(test_df['Votes'], float, fill_with =None, method = 'mean')
replace_nn_with(test_df['Reviews'], float, fill_with =None, method = 'mean')
test_df['Rating'] = test_df['Rating'].apply(lambda x: x.replace('-', '3.6'))
test_df['Rating'] = test_df['Rating'].apply(lambda x: x.replace('NEW', '3.6'))
test_df['Rating'] = test_df['Rating'].apply(lambda x: x.replace('Opening Soon', '3.6'))
test_df['Votes'] = test_df['Votes'].apply(lambda x: x.replace('-', '226'))
test_df['Reviews'] = test_df['Reviews'].apply(lambda x: x.replace('-', '111'))
train_df.head()
train_df['Average_Cost'] = train_df['Average_Cost'].apply(lambda x: x.replace('₹', ''))
test_df['Average_Cost'] = test_df['Average_Cost'].apply(lambda x: x.replace('₹', ''))
import re

trim_function = lambda x : re.findall("^\s*(.*?)\s*$",str(x))[0]

remove_commas = lambda x: re.sub("[^\d]", "", str(x))


train_df['Average_Cost']= train_df['Average_Cost'].apply(trim_function).apply(remove_commas)
train_df['Average_Cost']=train_df['Average_Cost'].replace(r'^\s*$', np.nan, regex=True)
train_df['Average_Cost']= train_df['Average_Cost'].fillna(train_df['Average_Cost'].value_counts().idxmax())
train_df['Average_Cost']= train_df['Average_Cost'].astype(int)
test_df['Average_Cost']= test_df['Average_Cost'].apply(trim_function).apply(remove_commas).astype(int)
train_df['Minimum_Order'] = train_df['Minimum_Order'].apply(lambda x: x.replace('₹', ''))
test_df['Minimum_Order'] = test_df['Minimum_Order'].apply(lambda x: x.replace('₹', ''))
import re

trim_function = lambda x : re.findall("^\s*(.*?)\s*$",str(x))[0]

remove_commas = lambda x: re.sub("[^\d]", "", str(x))


train_df['Minimum_Order']= train_df['Minimum_Order'].apply(trim_function).apply(remove_commas).astype(int)
test_df['Minimum_Order']= test_df['Minimum_Order'].apply(trim_function).apply(remove_commas).astype(int)
train_df['Rating']= train_df['Rating'].astype(float)
train_df['Votes']= train_df['Votes'].astype(float)
train_df['Reviews']= train_df['Reviews'].astype(float)
test_df['Rating']= test_df['Rating'].astype(float)
test_df['Votes']= test_df['Votes'].astype(float)
test_df['Reviews']= test_df['Reviews'].astype(float)
#Number of words in Location
train_df["sync_num_words"]=train_df['Location'].apply(lambda x :len(str(x).split()))
test_df["sync_num_words"]=test_df['Location'].apply(lambda x :len(str(x).split()))

#Number of unique words in Location
train_df["Syn_num_unique_words"] = train_df['Location'].apply(lambda x: len(set(str(x).split())))
test_df["Syn_num_unique_words"] = test_df['Location'].apply(lambda x: len(set(str(x).split())))

#Number of characters in Location

train_df["Syn_num_chars"] = train_df['Location'].apply(lambda x: len(str(x)))
test_df["Syn_num_chars"] = test_df['Location'].apply(lambda x: len(str(x)))

## Number of stopwords in the Location ##
train_df["Syn_num_stopwords"] = train_df['Location'].apply(lambda x: len([w for w in str(x).lower().split() if w in stop_words]))
test_df["Syn_num_stopwords"] = test_df['Location'].apply(lambda x: len([w for w in str(x).lower().split() if w in stop_words]))

## Number of punctuations in the Location ##
train_df["Syn_num_punctuations"] =train_df['Location'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )
test_df["Syn_num_punctuations"] =test_df['Location'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

## Number of title case words in the Location ##
train_df["Syn_num_words_upper"] = train_df['Location'].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))
test_df["Syn_num_words_upper"] = test_df['Location'].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

## Number of title case words in the Location ##
train_df["Syn_num_words_title"] = train_df['Location'].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))
test_df["Syn_num_words_title"] = test_df['Location'].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

## Average length of the words in the Location ##
train_df["mean_word_len"] = train_df['Location'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
test_df["mean_word_len"] = test_df['Location'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
def clean_text(text):
    text=text.lower()
    text = re.sub(r'@[a-zA-Z0-9_]+', '', text)
    text = re.sub(r'https?://[A-Za-z0-9./]+', '', text)
    text = re.sub(r'www.[^ ]+', '', text)
    text = re.sub(r'[a-zA-Z0-9]*www[a-zA-Z0-9]*com[a-zA-Z0-9]*', '', text)
    text = re.sub(r'[^a-zA-Z]', ' ', text)
    text = [token for token in text.split() if len(token) > 2]
    text = ' '.join(text)
    return text

train_df['Location'] = train_df['Location'].apply(clean_text)
test_df['Location'] = test_df['Location'].apply(clean_text)
def clean_text(text):
    text=text.lower()
    text = re.sub(r'@[a-zA-Z0-9_]+', '', text)
    text = re.sub(r'https?://[A-Za-z0-9./]+', '', text)
    text = re.sub(r'www.[^ ]+', '', text)
    text = re.sub(r'[a-zA-Z0-9]*www[a-zA-Z0-9]*com[a-zA-Z0-9]*', '', text)
    text = re.sub(r'[^a-zA-Z]', ' ', text)
    text = [token for token in text.split() if len(token) > 2]
    text = ' '.join(text)
    return text

train_df['Cuisines'] = train_df['Cuisines'].apply(clean_text)
test_df['Cuisines'] = test_df['Cuisines'].apply(clean_text)
## Number of words in the Cuisines ##
train_df["Cuisines_num_words"] = train_df["Cuisines"].apply(lambda x: len(str(x).split()))
test_df["Cuisines_num_words"] = test_df["Cuisines"].apply(lambda x: len(str(x).split()))

## Number of unique words in the Title ##
train_df["Cuisines_num_unique_words"] = train_df["Cuisines"].apply(lambda x: len(set(str(x).split())))
test_df["Cuisines_num_unique_words"] = test_df["Cuisines"].apply(lambda x: len(set(str(x).split())))

## Number of characters in the Title ##
train_df["Cuisines_num_chars"] = train_df["Cuisines"].apply(lambda x: len(str(x)))
test_df["Cuisines_num_chars"] = test_df["Cuisines"].apply(lambda x: len(str(x)))

## Number of stopwords in the Title ##
train_df["Cuisines_num_stopwords"] = train_df["Cuisines"].apply(lambda x: len([w for w in str(x).lower().split() if w in stop_words]))
test_df["Cuisines_num_stopwords"] = test_df["Cuisines"].apply(lambda x: len([w for w in str(x).lower().split() if w in stop_words]))

## Number of punctuations in the Title ##
train_df["Cuisines_num_punctuations"] =train_df['Cuisines'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )
test_df["Cuisines_num_punctuations"] =test_df['Cuisines'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

## Number of title case words in the Title ##
train_df["Cuisines_num_words_upper"] = train_df["Cuisines"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))
test_df["Cuisines_num_words_upper"] = test_df["Cuisines"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

## Number of title case words in the Title ##
train_df["Cuisines_num_words_title"] = train_df["Cuisines"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))
test_df["Cuisines_num_words_title"] = test_df["Cuisines"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

## Average length of the words in the Title ##
train_df["mean_word_len_Cuisines"] = train_df["Cuisines"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
test_df["mean_word_len_Cuisines"] = test_df["Cuisines"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
train_df.head(10)
test_df.head(10)<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
tf = TfidfVectorizer(ngram_range=(1, 1), lowercase=False)
train_route = tf.fit_transform(train_df['Cuisines'])
test_route = tf.transform(test_df['Cuisines'])
train_route = pd.DataFrame(data=train_route.toarray(), columns=tf.get_feature_names())
test_route = pd.DataFrame(data=test_route.toarray(), columns=tf.get_feature_names())
train_route.head()
train_df = pd.concat([train_df, train_route], axis=1) 
train_df.drop('Cuisines', axis=1, inplace=True)

test_df = pd.concat([test_df, test_route], axis=1) 
test_df.drop('Cuisines', axis=1, inplace=True)
train_df.head()
from sklearn.feature_extraction.text import TfidfVectorizer
tf = TfidfVectorizer(ngram_range=(1, 1), lowercase=False)
train_route = tf.fit_transform(train_df['Location'])
test_route = tf.transform(test_df['Location'])
train_route = pd.DataFrame(data=train_route.toarray(), columns=tf.get_feature_names())
test_route = pd.DataFrame(data=test_route.toarray(), columns=tf.get_feature_names())
train_route.head()
train_df = pd.concat([train_df, train_route], axis=1) 
train_df.drop('Location', axis=1, inplace=True)

test_df = pd.concat([test_df, test_route], axis=1) 
test_df.drop('Location', axis=1, inplace=True)
train_df.head()
train_df.drop('Restaurant', axis=1, inplace=True)
test_df.drop('Restaurant', axis=1, inplace=True)
duplicate_columns = test_df.columns[test_df.columns.duplicated()]
duplicate_columns_1 = train_df.columns[train_df.columns.duplicated()]

duplicate_columns_1
train_df.drop('north', axis=1, inplace=True)
test_df.drop('north', axis=1, inplace=True)
X = train_df.drop(labels=['Delivery_Time'], axis=1)
y = train_df['Delivery_Time'].values

from sklearn.model_selection import train_test_split
X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.25, random_state=1)
X_test = test_df
from xgboost import XGBClassifier
xgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, gamma=0,
              learning_rate=0.1, max_delta_step=0, max_depth=3,
              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,
              nthread=None, objective='multi:softprob', random_state=0,
              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
              silent=None, subsample=1, verbosity=1)

#Training the classifier
xgb.fit(X_train,y_train)

#Evaluating the score on validation set
xgb.score(X_cv,y_cv)
#Predicting for test set
Predictions = xgb.predict(X_test)
df_sub = pd.DataFrame(data=Predictions, columns=['Delivery_Time'])
writer = pd.ExcelWriter('Delivery_Time_1.xlsx', engine='xlsxwriter')
df_sub.to_excel(writer,sheet_name='Sheet1', index=False)
writer.save()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18025692106890157, 'fraction_numerical': 0.008724325066978086, 'mean_word_length': 5.602267573696145, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '45055', 'license': 'MIT', 'n_tokens_mistral': 5362, 'n_tokens_neox': 5225, 'n_words': 1015, 'path': 'brazil_ecommerce/working/food_delivery/working/Delivery_Timing.ipynb', 'repo_name': 'gonsoomoon-ml/predict-delivery-time', 'size': '14557'}