<jupyter_start><jupyter_text>Dataset: Ground Cover Types<jupyter_code>import pandas
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import time

# globals
n_row, n_col = 2, 5
n_components = n_row * n_col

np.random.seed(42)
## This script is applied to the covtype dataset available at https://s3.amazonaws.com/h2o-public-test-data/bigdata/laptop/covtype/covtype.data
## It converts expanded categorical levels back to dense columns
f = open("data/covtype.data.csv","r")
lines = f.readlines()
data = [l.split(",") for l in lines]
def which_level(x, f, t):
  for i in range(f,t):
    if x[i] == '1':
      return i - f
      raise Exception()
cats1 = [which_level(l,10,14) for l in data]
cats2 = [which_level(l,14,54) for l in data]
data2 = [data[i][0:10] + [str(cats1[i]),str(cats2[i]),str(data[i][54])] for i in range(len(data))]
output_data = [','.join(x) for x in data2]
f = open('data/covtype_processed.data.csv','w')
f.write("Elevation, Aspect, Slope, Horizontal_Distance_To_Hydrology, Vertical_Distance_To_Hydrology, Horizontal_Distance_To_Roadways, Hillshade_9am, Hillshade_Noon, Hillshade_3pm, Horizontal_Distance_To_Fire_Points, Wilderness_Area, Soil_Type, Cover_Type\n")
f.write("".join(output_data) )
f.close()
def read_dataset(path, num_rows):
    #./data/covtype.data.csv
    # With 581,012 instances we need to reduce for computability (dropped one or two zeros)
    full_data = pandas.read_csv(path, sep=',')
    RandInd = np.random.choice(len(full_data),num_rows)
    data = full_data.iloc[RandInd,:].reset_index().drop(['index'], axis = 1)
    instances = len(data.values)
    features = len(data.values[0])-1
    X = data.values[:, 0:features]
    y = data.values[:, features]

    return X, y
# num_rows max 581012
X, y = read_dataset('data/covtype_processed.data.csv',5810)
len(X)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14308855291576675, 'fraction_numerical': 0.02591792656587473, 'mean_word_length': 5.156146179401993, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '663191', 'license': 'MIT', 'n_tokens_mistral': 724, 'n_tokens_neox': 686, 'n_words': 161, 'path': 'covtype_data_preparation.ipynb', 'repo_name': 'mcgarrah/cs7641-unsupervised-learning', 'size': '1852'}