<jupyter_start><jupyter_text>Building Libraries of Secondary/ Tertiary Comparison<jupyter_code>beauty_profile_secondary={}
beauty_profile_third={}<jupyter_output><empty_output><jupyter_text>Benefits<jupyter_code>feature = 'Benefits'
lib = {}
alt = {}

val = beauty_profile[feature].get('light', 3)
print(val)
beauty_profile[feature].pop('light', None)

# val = beauty_profile[feature].get('feature')
# print(val)
# beauty_profile[feature].update({'newfeature': val})
# lib['newfeature'] = val

# val = beauty_profile[feature].get('high pigmentation')
# lib['pigmentation'] = val

# val = beauty_profile[feature].get('natural')
# lib['alami'] = val

# val = 3
# val = beauty_profile[feature].get('light')
# lib['luminous'] = val
# lib['glow'] = val

# val = beauty_profile[feature].get('hydrating')
# lib['melembabkan'] = val
# lib['hydramild'] = val
# lib['hydration'] = val
# lib['moisturizing'] = val

val = beauty_profile[feature].get('durable')
lib['tahan lama'] = val
# lib['long wear'] = val

# val = beauty_profile[feature].get('oil control')
# lib['oil'] = val
# lib['kontrol minyak'] = val
# lib['mattifying'] = val

# val = beauty_profile[feature].get('spf')
# lib['spf50'] = val
# lib['spf30'] = val
# lib['spf15'] = val


beauty_profile_secondary[feature] = lib
beauty_profile_third[feature] = alt<jupyter_output><empty_output><jupyter_text>Brand<jupyter_code>feature = 'Brand'
lib = {}
alt = {}

for f, val in beauty_profile[feature].items():
    """ if text contains dot, replace with spaces """
    temp = f.replace(".", " ")
    lib[temp] = val

beauty_profile[feature].update(lib)


lib = {}
for f, val in beauty_profile[feature].items():
    """ if text contains spaces, combine them """
    num_spaces = len(f.split())
    for i in range(1, num_spaces):
        temp = f.replace(" ", '', i)
        lib[temp] = val

val = beauty_profile[feature].get('etude house')
print(val)
lib['etude'] = val

val = beauty_profile[feature].get('natrue republic')
print(val)
beauty_profile[feature].update({'nature republic': val})
lib['natural republic'] = val
lib['natural republik'] = val

val = beauty_profile[feature].get('kiss proof')
print(val)
beauty_profile[feature].update({'kiss beauty': val})

val = beauty_profile[feature].get('kylie cosmetics')
print(val)
beauty_profile[feature].update({'kylie': val})
lib['kylie+lip'] = val

val = beauty_profile[feature].get('l oreal paris')
print(val)
lib['l oreal'] = val
lib['loreal'] = val

val = beauty_profile[feature].get('saint laurent')
print(val)
lib['ysl'] = val
lib['yves'] = val

val = beauty_profile[feature].get('tonymoly')
print(val)
lib['tony moly'] = val

val = beauty_profile[feature].get('burberry')
print(val)
lib['b erl'] = val

val = beauty_profile[feature].get('maybelline')
print(val)
lib['maybeline'] = val
lib['maybelin'] = val
lib['meybelline'] = val
lib['meybeline'] = val

val = beauty_profile[feature].get('aprilskin')
print(val)
lib['april skin magic snow'] = val
lib['april skin magic cushion'] = val

# val = beauty_profile[feature].get('wet n wild')
# print(val)
# lib['wet and wild'] = val
# lib['wet wild'] = val

# val = beauty_profile[feature].get('l.a.girl')
# print(val)
# lib['l.a girl'] = val

# val = beauty_profile[feature].get('theraskin')
# print(val)
# lib['thera skin'] = val
# lib['thera'] = val

# val = beauty_profile[feature].get('ultra jaya')
# print(val)
# lib['ultra'] = val

# val = beauty_profile[feature].get('hadalabo')
# print(val)
# lib['hada labo'] = val

val = beauty_profile[feature].get('oriflame')
print(val)
lib['giordani gold'] = val
lib['giordanigold'] = val
# lib['the one'] = val

# val = beauty_profile[feature].get('sop<NAME>')
# print(val)
# lib['<NAME>'] = val
# lib['sop<NAME>'] = val

# val = beauty_profile[feature].get('crystal')
# print(val)
# lib['cristal'] = val
# lib['kristal'] = val

val = beauty_profile[feature].get('wardah')
# print(val)
lib['bedakwardah'] = val
lib['murahwardah'] = val
lib['warda'] = val

# val = beauty_profile[feature].get('chanel')
# print(val)
# lib['channel'] = val

# val = beauty_profile[feature].get('hn original')
# print(val)
# lib['hn'] = val

# val = beauty_profile[feature].get('bioaqua')
# print(val)
# lib['bio aqua'] = val
# lib['biioaqua'] = val

val = beauty_profile[feature].get('victoria s secret')
print(val)
lib['victoria secret'] = val
lib['vs'] = val


beauty_profile_secondary[feature] = lib
beauty_profile_third[feature] = alt<jupyter_output><empty_output><jupyter_text>Colour_group<jupyter_code>feature = 'Colour_group'
lib = {}
alt = {}

val = beauty_profile[feature].get('krem')
print(val)
lib['beige'] = val
# lib['cream'] = val

val = beauty_profile[feature].get('merah muda')
print(val)
lib['pink'] = val

val = beauty_profile[feature].get('putih')
print(val)
lib['white'] = val

val = beauty_profile[feature].get('hitam')
print(val)
lib['black'] = val

val = beauty_profile[feature].get('mawar')
print(val)
lib['rose'] = val

val = beauty_profile[feature].get('bening')
print(val)
lib['clear'] = val

val = beauty_profile[feature].get('merah')
print(val)
lib['red'] = val

val = beauty_profile[feature].get('hijau')
print(val)
lib['green'] = val

val = beauty_profile[feature].get('sawo matang')
print(val)
lib['brown'] = val

val = beauty_profile[feature].get('netral')
print(val)
lib['neutral'] = val

val = beauty_profile[feature].get('kuning')
print(val)
lib['yellow'] = val
# lib['kemuning'] = val

val = beauty_profile[feature].get('biru')
print(val)
lib['blue'] = val

val = beauty_profile[feature].get('perak')
print(val)
lib['silver'] = val

val = beauty_profile[feature].get('jeruk')
print(val)
lib['orange'] = val
# lib['jingga'] = val

val = beauty_profile[feature].get('ceri')
print(val)
lib['cherry'] = val

beauty_profile_secondary[feature] = lib
beauty_profile_third[feature] = alt<jupyter_output><empty_output><jupyter_text>Product_texture<jupyter_code>feature = 'Product_texture'
lib = {}
alt = {}


val = beauty_profile[feature].get('crayon pensiln')
print(val)
beauty_profile[feature].update({'pencil': val})
lib['crayon'] = val
lib['pensil'] = val
# lib['pencil'] = val

val = beauty_profile[feature].get('solid powder')
print(val)
beauty_profile[feature].update({'bedak': val})
beauty_profile[feature].update({'powder': val})
# lib['loose'] = val
# lib['padat'] = val
# lib['solid'] = val
# lib['pwd'] = val
lib['two way cake'] = val
lib['twc'] = val
# lib['lipstick'] = val


val = beauty_profile[feature].get('cushion', 7)
print(val)
lib['cushion+refill'] = val
lib['cushion'] = val
lib['cuhsion'] = val
beauty_profile[feature].pop('cushion', None)


val = beauty_profile[feature].get('formula mousse')
print(val)
lib['mousse'] = val

val = beauty_profile[feature].get('cream')
print(val)
lib['krim'] = val
lib['cc+cream'] = val
# lib['bb'] = val


val = beauty_profile[feature].get('liquid')
print(val)
beauty_profile[feature].update({'cair': val})
# lib['foundation'] = val

val = beauty_profile[feature].get('stick')
print(val)
beauty_profile[feature].update({'stik': val})


""" THIRD """
# val = beauty_profile[feature].get('gel')
# print(val)
# beauty_profile[feature].update({'jelly': val})


beauty_profile_secondary[feature] = lib
beauty_profile_third[feature] = alt<jupyter_output><empty_output><jupyter_text>Skin_type<jupyter_code>feature = 'Skin_type'
lib = {}
alt = {}

# beauty_profile[feature].update({'newfeature': val})

val = beauty_profile[feature].get('signs of aging')
print(val)
beauty_profile[feature].update({'anti aging': val})
lib['aging'] = val
lib['penuaan'] = val
lib['wrinkle'] = val


val = beauty_profile[feature].get('acne')
print(val)
beauty_profile[feature].update({'acnes': val})
lib['jerawat'] = val
lib['full cover'] = val

val = beauty_profile[feature].get('greasy')
print(val)
beauty_profile[feature].update({'berminyak': val})
lib['oil control'] = val
lib['no sebum'] = val

# lib['matte'] = val
# alt['matte'] = val

val = beauty_profile[feature].get('sensitive')
print(val)
beauty_profile[feature].update({'sensitif': val})


val = beauty_profile[feature].get('dry')
print(val)
lib['kering'] = val



# alt['melembabkan'] = val  # doesn't work well
# alt['hydrating'] = val  # doesn't work well


beauty_profile_secondary[feature] = lib
beauty_profile_third[feature] = alt<jupyter_output><empty_output><jupyter_text>Storing Library<jupyter_code>beauty_library = {
    "primary": beauty_profile,
    "secondary": beauty_profile_secondary,
    "third": beauty_profile_third,
    "length": beauty_max_length,
}
with open('model/heuristic/beauty_library.json', 'w+') as file:
    json.dump(beauty_library, file)<jupyter_output><empty_output><jupyter_text>heuristic functions<jupyter_code>""" heuristic function :) """
def _word_compare(texts_lib, library):
    for text in texts_lib:
        value = library.get(text, None)
        if not value is None:
            return value

def word_compare(title, libraries, level=4):
    texts = title.split()
    texts_lib = texts
    texts_temp = texts
    for i in range(1, level):
        texts_temp = list(zip(texts_temp, texts[i:]))
        texts_temp = [' '.join(text) for text in texts_temp]
        texts_lib += texts_temp

    if isinstance(libraries, list):
        for lib in libraries:
            value = _word_compare(texts_lib, lib)
            if not value is None:
                return value
        return None
    else:
        return _word_compare(texts_lib, libraries)

    
def get_feature(title, library, level=4):
    res = word_compare(
        title,
        libraries=library,
        level=level
    )
    return res

def _word_compare_ans(texts_lib, library):
    for text in texts_lib:
        value = library.get(text, None)
        if not value is None:
            return value, text
    return None, None

def word_compare_ans(title, libraries, level=4):
    texts = title.split()
    texts_lib = texts
    texts_temp = texts
    for i in range(1, level):
        texts_temp = list(zip(texts_temp, texts[i:]))
        texts_temp = [' '.join(text) for text in texts_temp]
        texts_lib += texts_temp

    if isinstance(libraries, list):
        for lib in libraries:
            value, text = _word_compare_ans(texts_lib, lib)
            if not value is None:
                return value, text
        return None, None
    else:
        return _word_compare_ans(texts_lib, libraries)

    
def get_feature_ans(title, library, level=4):
    res, keyword = word_compare_ans(
        title,
        libraries=library,
        level=level
    )
    return res, keyword


def get_feature_strip(title, library, level=4):
    res, key = get_feature_ans(title, library, level)
    if res is not None:
        title = title.replace(key, "-")
        return [title, res]
    return [title, None]


import re
regex = re.compile('[^a-zA-Z ]')

def get_feature_wo_num(title, library, level=4):
    title = regex.sub('', title)
    res, keyword = word_compare_ans(
        title,
        libraries=library,
        level=level
    )
    return [title, res]
""" get text details of all the features"""
beauty_lib = {}

for feature in beauty_profile.keys():
    print(feature)
    key_to_text = {v: k for k, v in beauty_profile.get(feature).items()}
    beauty_lib[feature] = key_to_text
    beauty_train[feature+'_info'] = beauty_train[feature].map(key_to_text)<jupyter_output><empty_output><jupyter_text>Run heuristic function<jupyter_code>def get_first(mylist):
    return mylist[0]


def get_second(mylist):
    return mylist[1]
from model.heuristic.utilities import *


def run_beauty_heuristic(beauty_train, beauty_profile, beauty_profile_secondary, beauty_profile_third, beauty_max_length):
    beauty_pred = beauty_train.filter(['itemid', 'title'])

    feature = 'Product_texture'
    libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
    level = beauty_max_length[feature]
    beauty_pred[feature] = beauty_train['title'].apply(get_feature, args=(libraries,level,))

    feature = 'Brand'
    libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
    level = beauty_max_length[feature]
    temp = beauty_train['title'].apply(get_feature_strip, args=(libraries,level,))
    beauty_pred['title_strip'] = temp.map(get_first)
    beauty_pred['Brand'] = temp.map(get_second)

    for feature in ['Colour_group', 'Benefits']:
        libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
        level = beauty_max_length[feature]
        beauty_pred[feature] = beauty_pred.title_strip.apply(get_feature, args=(libraries,level,))

    feature = 'Skin_type'
    libraries = [beauty_profile[feature], beauty_profile_secondary[feature], beauty_profile_third[feature]]
    level = beauty_max_length[feature]
    beauty_pred[feature] = beauty_pred.title_strip.apply(get_feature, args=(libraries,level,))

    return beauty_pred

""" run heuristic - trim title """
from datetime import datetime
start_time = datetime.now()


beauty_library = load_library('model/heuristic/beauty_library.json')

beauty_profile = beauty_library.get('primary')
beauty_profile_secondary = beauty_library.get('secondary')
beauty_profile_third = beauty_library.get('third')
beauty_max_length = beauty_library.get('length')

beauty_pred = run_beauty_heuristic(
    beauty_train, beauty_profile, beauty_profile_secondary, beauty_profile_third, beauty_max_length
)


duration = datetime.now() - start_time
print(duration)

beauty_pred.head()
""" single feature prediction - trim"""
start_time = datetime.now()

# feature = 'Skin_type'
print(feature)

libraries = [beauty_profile[feature], beauty_profile_secondary[feature], beauty_profile_third[feature]]
level = beauty_max_length[feature]
beauty_pred[feature] = beauty_pred.title_strip.apply(get_feature, args=(libraries,level,))

duration = datetime.now() - start_time
print(duration)
""" single feature prediction"""
start_time = datetime.now()

# feature = 'Skin_type'
print(feature)

libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
level = beauty_max_length[feature]
beauty_pred[feature] = beauty_train.title.apply(get_feature, args=(libraries,level,))

duration = datetime.now() - start_time
print(duration)<jupyter_output><empty_output><jupyter_text>Review result<jupyter_code>result = {}
for feature in beauty_profile.keys():
    print(feature)
    """ compare actual brand and prediction on insight DF"""
    insight = beauty_train.filter(['itemid', 'title', feature, feature+'_info'])
    insight[feature+'_pred'] = beauty_pred[feature]
    insight[feature+'_info_pred'] = beauty_pred[feature].map(beauty_lib[feature])
    insight['title_strip'] = beauty_pred['title_strip']
    total = len(insight)

    """ drop brand NA of actual brand"""
    insight = insight.dropna(subset=[feature])
    print("  DF  ~NaN len:", len(insight), '({})%'.format(len(insight)/total*100))

    insight_accuracy = insight.dropna(subset=[feature, feature+'_pred'])
    print("  DF known len:",len(insight_accuracy), '({})%'.format(len(insight_accuracy)/len(insight)*100))
    print(insight_accuracy[feature].eq(insight_accuracy[feature+'_pred']).value_counts(normalize=True))

    """ get unequal insight_error DF"""
    insight_error = insight_accuracy.loc[insight_accuracy[feature] != insight_accuracy[feature+'_pred'], :]
    print(insight_error[feature].value_counts())

    result[feature] = insight
    print()
""" view top unknown/ error """
idx = 4
feature = list(beauty_profile.keys())[idx]
insight = result[feature]

res = insight[feature].eq(insight[feature+'_pred'])
wrong_index = res[res == False].index
insight_error = insight.loc[wrong_index, :]
print(insight_error[feature].value_counts().head(10))

for idx in insight_error[feature].value_counts().index[:10]:
    print( idx, beauty_lib[feature].get(int(idx)) )
""" view error title for : bkey """
# feature = 'Pattern'
bkey = 7

print(beauty_lib[feature].get(bkey))
feature_error = insight_error.loc[insight_error[feature] == bkey,:]

print("diff:", len(feature_error))
print("err :",len(feature_error.dropna(subset=[feature+'_pred'])))

# check errors
feature_error.dropna(subset=[feature+'_pred'])

# check undetected
feature_error.loc[feature_error[feature+'_pred'].isna(), :].title_strip.sample(60)<jupyter_output><empty_output><jupyter_text>Run heuristics and export submission<jupyter_code>""" beauty_val, run all heuristic functions - prediction on separate df """
from datetime import datetime
start_time = datetime.now()

beauty_val_pred = beauty_val.filter(['itemid', 'title'])
for feature in beauty_profile.keys():
    print(feature)
    libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
    level = beauty_max_length[feature]

    beauty_val_pred[feature] = beauty_val.title.apply(get_feature, args=(libraries,level,))

duration = datetime.now() - start_time
print(duration)

beauty_val_pred.info()
""" run heuristic - trim title """
from datetime import datetime
start_time = datetime.now()

beauty_val_pred = beauty_val.filter(['itemid', 'title'])

feature = 'Product_texture'
libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
level = beauty_max_length[feature]
beauty_val_pred[feature] = beauty_val['title'].apply(get_feature, args=(libraries,level,))


feature = 'Brand'
libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
level = beauty_max_length[feature]

temp = beauty_val['title'].apply(get_feature_strip, args=(libraries,level,))
beauty_val_pred['title_strip'] = temp.map(get_first)
beauty_val_pred['Brand'] = temp.map(get_second)


for feature in ['Benefits', 'Colour_group']:
    libraries = [beauty_profile[feature], beauty_profile_secondary[feature]]
    level = beauty_max_length[feature]
    beauty_val_pred[feature] = beauty_val_pred.title_strip.apply(get_feature, args=(libraries,level,))


feature = 'Skin_type'
libraries = [beauty_profile[feature], beauty_profile_secondary[feature], beauty_profile_third[feature]]
level = beauty_max_length[feature]
beauty_val_pred[feature] = beauty_val_pred.title_strip.apply(get_feature, args=(libraries,level,))

    
duration = datetime.now() - start_time
print(duration)

beauty_val_pred.info()
# format answer
beauty_val_submission = pd.DataFrame(columns=['id', 'tagging'])

for feature in beauty_profile.keys():
    temp = pd.DataFrame()
    temp['id'] = beauty_val_pred['itemid'].map(str)+"_"+feature
    temp['tagging'] = beauty_val_pred[feature]
    
    if len(beauty_val_submission) == 0:
        beauty_val_submission = temp
    else:
        beauty_val_submission = beauty_val_submission.append(temp, ignore_index=True)
    print(feature, len(beauty_val_submission))

beauty_val_submission.head()
beauty_val_submission.info()

filename = 'beauty_info_val_submission_'+str(datetime.now().date()).replace('-', '')+'.csv'
print(filename)
beauty_val_submission.to_csv(filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15469114837614095, 'fraction_numerical': 0.0018573551263001485, 'mean_word_length': 5.689740859069933, 'pattern_counts': {'":': 4, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '639535', 'license': 'MIT', 'n_tokens_mistral': 6575, 'n_tokens_neox': 6492, 'n_words': 1206, 'path': 'notebooks/ida/beauty_heuristics_generate_review_heuristics_library.ipynb', 'repo_name': 'AdityaSidharta/shopee_data_science', 'size': '18864'}