<jupyter_start><jupyter_text>phash embedding<jupyter_code>test["phash_matches"] = mylib.phash_matches(test, threshold=0.3)<jupyter_output><empty_output><jupyter_text>SBERT sentence embedding<jupyter_code>st_name = "stsb-distilbert-base"
#st_name = "paraphrase-xlm-r-multilingual-v1"
test["title_p"] = test.apply(mylib.preprocess, axis=1)
posting_ids = test["posting_id"].tolist()
sentences = test["title_p"].tolist()
test["sbert_matches"] = mylib.sbert_matches(
    model_path=f"{RESOURCE_DIR}/pretrained/sentence-transformers/{st_name}",
    sentences=sentences,
    posting_ids=posting_ids,
    threshold=0.5
)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>cols = ["phash_matches", "sbert_matches"]
test["matches"] = test.apply(mylib.combine_as_string(cols), axis=1)
sub = test[["posting_id", "matches"]]
sub.head()
sub.to_csv("submission.csv", index = False)<jupyter_output><empty_output><jupyter_text>Debug<jupyter_code>!pip list<jupyter_output>Package                        Version             Location
------------------------------ ------------------- --------------
absl-py                        0.12.0
adal                           1.2.6
affine                         2.3.0
aiobotocore                    1.2.2
aiohttp                        3.7.3
aiohttp-cors                   0.7.0
aioitertools                   0.7.1
aioredis                       1.3.1
albumentations                 0.5.2
alembic                        1.5.8
allennlp                       2.2.0
altair                         4.1.0
annoy                          1.17.0
ansiwrap                       0.8.4
appdirs                        1.4.4
argon2-cffi                    20.1.0
arrow                          0.17.0
arviz                          0.11.2
asn1crypto                     1.4.0
astropy                        4.2
astunparse                     1.6.3
async-generator                1.10
async-timeout                  3.0[...]
{'fraction_non_alphanumeric': 0.13569620253164558, 'fraction_numerical': 0.0410126582278481, 'mean_word_length': 1.4638403990024937, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '547748', 'license': 'MIT', 'n_tokens_mistral': 682, 'n_tokens_neox': 652, 'n_words': 84, 'path': 'shopee-product-matching.ipynb', 'repo_name': 'seahrh/kaggle-shopee-product-matching', 'size': '1975'}