<jupyter_start><jupyter_text>code2vec_vectorization> This module uses Code2Vec model to perform vectorization on source code> @Alvaro 14 April 2021  Load model<jupyter_code>config = {
    'code2vec_model_path': '/tf/main/dvc-ds4se/models/cv/java-large-release/saved_model_iter3.release',
    'code2vec_predicter': {
        'SHOW_TOP_CONTEXTS': 10,
        'MAX_PATH_LENGTH': 8,
        'MAX_PATH_WIDTH': 2,
        'JAR_PATH': 'JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar',
    },
    'codesearchnet_java_csv_path': '/tf/main/dvc-ds4se/code/searchnet/[codesearchnet-java-1597073966.81902].csv',
    'spm_model_path': '/tf/main/my_model/spm',
    'spm_vocab_size': 16384,
    'spm_sentence_length': 256,
    'ae_checkpoint_path': '/tf/main/my_model/ae_checkpoint',
    'ae_embedding_dim': 256,
    'ae_batch_size': 16,
}
import code2vec
import common as code2vec_common
import config as code2vec_config
import extractor as code2vec_extrator

code2vec_cfg = code2vec_config.Config(set_defaults=True)
code2vec_cfg.PREDICT = True
code2vec_cfg.MODEL_LOAD_PATH = config['code2vec_model_path']
code2vec_cfg.DL_FRAMEWORK = 'tensorflow'
code2vec_cfg.EXPORT_CODE_VECTORS = True
code2vec_cfg.verify()
help(code2vec)
code2vec_model = code2vec.load_model_dynamically(code2vec_cfg)
code2vec_path_extractor = code2vec_extrator.Extractor(
    code2vec_cfg,
    jar_path=config['code2vec_predicter']['JAR_PATH'],
    max_path_length=config['code2vec_predicter']['MAX_PATH_LENGTH'],
    max_path_width=config['code2vec_predicter']['MAX_PATH_WIDTH']
)
import os
import tempfile
import numpy as np

def code2vec_predict(code: str) -> List[np.ndarray]:
    os.chdir('/tf/main/notebooks/code2vec/code2vec')
    with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', suffix='.java') as input_file:
        input_file.write(code)
        input_file.flush()
        input_filename = input_file.name
        try:
            predict_lines, hash_to_string_dict = code2vec_path_extractor.extract_paths(input_filename)
        except ValueError as e:
            raise
    assert len(predict_lines) == 1
    raw_prediction_results = code2vec_model.predict(predict_lines)
    assert len(raw_prediction_results) == 1
    raw_prediction = raw_prediction_results[0]
    return raw_prediction.code_vector<jupyter_output><empty_output><jupyter_text>Load data (Searchnet - Java)<jupyter_code>java_df = pd.read_csv(config['codesearchnet_java_csv_path'], header=0, index_col=0, sep='~')
java_df.head()<jupyter_output><empty_output><jupyter_text>Use code2vec with Searchnet data<jupyter_code>for i, row in java_df.iterrows():
    if i == 4:
        break
    print('\n'.join(repr(code2vec_predict(row['code'])).split('\n')[:2]))
samples = java_df.sample(10)<jupyter_output><empty_output><jupyter_text>LimitationsApparently Code2vec is not able to code snippets with errors, because it is unable to generate the required AST structure.<jupyter_code>c2v_vecs = samples['code'].apply(lambda snippet: code2vec_predict(snippet))
c2v_vecs
individual_vec = code2vec_predict(java_df['code'].values[345])
print(individual_vec.shape)
repr(individual_vec).split('\n')[:2]
pprint(individual_vec)
def get_code2vec_vectors(df: pd.DataFrame, code_column: str):
    """
    """

    c2v_vectors = df[code_column]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11711439782674313, 'fraction_numerical': 0.02958044068819801, 'mean_word_length': 5.64128256513026, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '268180', 'license': 'Apache-2.0', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1204, 'n_words': 179, 'path': 'notebooks/code2vec-usage.ipynb', 'repo_name': 'WM-CSCI-435-F19/data-science-4-software-engineering', 'size': '3313'}