<jupyter_start><jupyter_text><jupyter_code>!sudo apt-get install build-essential swig
!pip install auto-sklearn==0.11.1
from google.colab import drive
drive.mount('/content/gdrive')
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
import time
df = pd.read_csv('/content/domain_tfinal.csv')
import sys
import os

py_file_location = ("/content/")
sys.path.append(os.path.abspath(py_file_location))
import model6
import nltk
nltk.download('punkt')
df = df[df['Valid Repo'] == True]
principal_language_frame = df['Language']
principal_language_frame = pd.get_dummies(principal_language_frame).add_prefix('lg: ')

license_frame = df['License']
license_frame = pd.get_dummies(license_frame).add_prefix('li: ')

releases = df['Releases Count']
stars = df['Stars']	
forks = df['Forks']

# Pre procesing columns to vectorize 
# Modify datatype from object to string
description = df['Description'].apply(str)
readme = df['Readme File'].apply(str)
topic = df['Topic List'].apply(str)
prog_lang = df['Prog. Languages'].apply(str)
labels = df['Labels'].apply(str)
contributors = df['Contributors'].apply(str)

# Obtain labels
y = df.Domain
norm_d = model6.normalize_corpus(description)
nlr = model6.remove_links(readme)
norm_rm = model6.normalize_corpus(nlr)
norm_cn = model6.remove_brackets(contributors)
norm_pl = model6.remove_brackets(prog_lang)
norm_tp = model6.remove_brackets(topic)
blb = model6.remove_brackets(labels)
norm_lb = model6.clean_char(blb)
from sklearn.feature_extraction.text import CountVectorizer


def vectorize(text, prefix):
    vect = CountVectorizer()
    text_vectorized = vect.fit_transform(text)
    return pd.DataFrame(text_vectorized.toarray(), columns=vect.get_feature_names()).add_prefix(prefix)

ds = vectorize(norm_d, 'd: ')
rm = vectorize(norm_rm, 'rm: ')
vc = vectorize(norm_cn, 'cn: ')
vpl = vectorize(norm_pl, 'pl: ')
vt = vectorize(norm_tp, 'tp: ')
vl = vectorize(norm_lb, 'lb: ')

# concatenate vectorized word counts:

X_word = pd.concat([ds,rm,vc,vpl,vt,vl],axis=1)
X_word.reset_index(drop=True, inplace=True)
X_ohe = pd.concat([principal_language_frame, license_frame, releases, stars, forks], axis=1)
X_ohe.reset_index(drop=True, inplace=True)

from sklearn.model_selection import train_test_split
# train 0.6 test 0.4 split
# # unbalanced dataset
X_train_word, X_test_word, y_train, y_test = train_test_split(X_word, y, random_state=42, test_size= 0.4, stratify=y)
X_train_ohe, X_test_ohe = train_test_split(X_ohe, random_state=42, test_size=0.4, stratify = y)

 
# reduce dimension
start_time = time.time()
X_train_word, X_test_word = model6.dimensionality_reduction(X_train_word, X_test_word, y_train)
end_time = time.time()
dim_time = end_time - start_time

# concatenate bow and dummies
X_train = pd.concat([X_train_word,X_train_ohe], axis=1)
X_test = pd.concat([X_test_word,X_test_ohe], axis=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12434036939313985, 'fraction_numerical': 0.009894459102902375, 'mean_word_length': 7.023809523809524, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '155946', 'license': 'MIT', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1116, 'n_words': 219, 'path': 'Untitled1.ipynb', 'repo_name': 'fzanartu/github-slideshow', 'size': '3032'}