<jupyter_start><jupyter_text>Home Depot Product Search RelevanceThe challenge is to predict a relevance score for the provided combinations of search terms and products. To create the ground truth labels, Home Depot has crowdsourced the search/product pairs to multiple human raters.  LabGraph CreateThis notebook uses the LabGraph create machine learning iPython module. You need a personal licence to run this code.<jupyter_code>import graphlab as gl<jupyter_output><empty_output><jupyter_text>Load data from CSV files<jupyter_code>train = gl.SFrame.read_csv("../data/train.csv")
test = gl.SFrame.read_csv("../data/test.csv")
desc = gl.SFrame.read_csv("../data/product_descriptions.csv")
attr = gl.SFrame.read_csv("../data/attributes.csv")<jupyter_output><empty_output><jupyter_text>Data merging and feature engineering<jupyter_code># merge train with description
train = train.join(desc, on = 'product_uid', how = 'left')
# merge test with description
test = test.join(desc, on = 'product_uid', how = 'left')
# if some attributes has no so we don't need them
print len(attr)
attr = attr[attr['value'] != "No"]
print len(attr)
# if some attributes has "yes" we compy the value so we can search in it
attr['value'] = attr.apply(lambda x: x['name'] if x['value'] == "Yes" else x['value'])<jupyter_output><empty_output><jupyter_text>Let's select brands<jupyter_code>brands = attr[attr['name'] == "MFG Brand Name"]
brands.head()<jupyter_output><empty_output><jupyter_text>Bullets too<jupyter_code>bullets = attr[attr['name'].contains("Bullet")]
# converting bullets to columns
bullets = bullets.unstack(column = ['name', 'value'], new_column_name = "bullets")
bullets = bullets.unpack("bullets")
bullets = bullets.sort("product_uid")
print len(bullets)
# merge train with brands and bullets
train = train.join(brands, on = 'product_uid', how = 'left')
train = train.join(bullets, on = 'product_uid', how = 'left')
# merge test with brands and bullets
test = test.join(brands, on = 'product_uid', how = 'left')
test = test.join(bullets, on = 'product_uid', how = 'left')<jupyter_output><empty_output><jupyter_text>TF-IDF with linear regression<jupyter_code>def calculateTfIdf(cols, data, searchColTfIdfName):
    for item in xrange(len(cols)):
        colName = cols[item]
        newColNameWordCount = colName + "_word_count"
        newColNameTfIdf = colName + "_tfidf"
        newColDistance = colName + "_distance"
        
        wordCount = gl.text_analytics.count_words(data[colName])
        data[newColNameWordCount] = wordCount
        
        tfidf = gl.text_analytics.tf_idf(data[newColNameWordCount])
        data[newColNameTfIdf] = tfidf
        #print colName
        if searchColTfIdfName != colName:
            data[newColDistance] = data.apply(lambda x: 0 if x[newColNameTfIdf] is None else gl.distances.cosine(x[searchColTfIdfName],x[newColNameTfIdf]))
        
    return data
# columns = ['search_term', 'product_title', 'product_description', 'value', 'bullets.Bullet01',
#          'bullets.Bullet02', 'bullets.Bullet03', 'bullets.Bullet04', 'bullets.Bullet05', 'bullets.Bullet06'
#          , 'bullets.Bullet07', 'bullets.Bullet08', 'bullets.Bullet09', 'bullets.Bullet10', 'bullets.Bullet11'
#          , 'bullets.Bullet12', 'bullets.Bullet13', 'bullets.Bullet14', 'bullets.Bullet15', 'bullets.Bullet16'
#          , 'bullets.Bullet17', 'bullets.Bullet18', 'bullets.Bullet19', 'bullets.Bullet20', 'bullets.Bullet21'
#          , 'bullets.Bullet22']

columns = ['search_term', 'product_title', 'product_description', 'value']

train = calculateTfIdf(columns, train, 'search_term_tfidf')
test = calculateTfIdf(columns, test, 'search_term_tfidf')
featuresDistance = [s for s in train.column_names() if "distance" in s]
print featuresDistance
#train = train.dropna('value_distance')
model1 = gl.linear_regression.create(train, target = 'relevance', features = featuresDistance)
#let's take a look at the weights before we plot
model1.get("coefficients")
'''
predictions_test = model1.predict(test)
test_errors = predictions_test - test['relevance']
RSS_test = sum(test_errors * test_errors)
print RSS_test
'''
predictions_test = model1.predict(test)
predictions_test
submission = gl.SFrame(test['id'])
submission.add_column(predictions_test)
submission.rename({'X1': 'id', 'X2':'relevance'})
submission['relevance'] = submission.apply(lambda x: 3.0 if x['relevance'] > 3.0 else x['relevance'])
submission['relevance'] = submission.apply(lambda x: 1.0 if x['relevance'] < 1.0 else x['relevance'])
submission['relevance'] = submission.apply(lambda x: str(x['relevance']))
submission.export_csv('../data/submission.csv', quote_level = 3)
#gl.canvas.set_target('ipynb')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13679745493107104, 'fraction_numerical': 0.012725344644750796, 'mean_word_length': 5.522821576763485, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '385812', 'license': 'MIT', 'n_tokens_mistral': 1625, 'n_tokens_neox': 1565, 'n_words': 360, 'path': 'ProjectSearchRelevance.Python/Home Depot Product Search Relevance Features.ipynb', 'repo_name': 'Ravirajadrangi/HomeDepot', 'size': '4715'}