<jupyter_start><jupyter_text>----  Method Naming<jupyter_code>count_labels = Counter()
with open('../magret/java/methodname/java-train-labels.txt', 'r') as f:
    for l in f:
        count_labels.update([l.strip('\n')])
count_labels.most_common(100)
filename = "data/java/train_java.jsonl"
global_id = 0
n_edges = 0
snippet_count = 0
method_labels = []
method_labels_train = [];method_labels_test = []
name = 'java'
suffix = '_val'
test_count = 0; train_count = 0
max_len = 128
if not os.path.exists(out_path):
    os.makedirs(out_path)
#vocab = set()

with open(filename, 'r', encoding="ISO-8859-1") as f:
    for idx, line in enumerate(f):
        is_funcdef = False
        data = json.loads(line)
        G=nx.Graph()
        row = ["[CLS]"]; row_order = ["[CLS]"]
        for r in data:
            #print(r)
            if type(r) == int:
                break
            from_node = r['uid']
            if r['type'] in ['FunctionDeclaration', 'MethodDeclaration']:
                if r.get('value', None) is not None:
                    w = r['type']
                    label = str(r['value'])
                    #print(label)
                    is_funcdef = True
            if r['type'] in ['ClassDeclaration', 'FormalParameter', 'VariableDeclarator', 'SuperMethodInvocation', 'CatchClauseParameter', 'ConstructorDeclaration', 'MethodInvocation', 'MemberReference']:
                if r.get('value', None) is not None:
                    if (vocab[r['value']] >= 5):
                        w = str(r['value'])
                    else: 
                        w = r['type']
            else:
                w = r['type']
            #if is_funcdef:
            row.append(w)

            for to_node in r.get('children', []):
                n_edges += 1
                G.add_edge(from_node, to_node)
            if G.number_of_nodes() > max_len:
                break
                
        if (G.number_of_nodes() < max_len) and (G.number_of_nodes() > 10) and (is_funcdef == True) and (count_method_labels[label] > 1):
            # Save tk
            # print(row)
            if np.random.uniform() < 0.2:
                is_train = False
                suffix = '_val'
                prefix = str(test_count)
                method_labels_test.append(label)

            else:
                is_train = True
                suffix = ''
                prefix = str(train_count)
                method_labels_train.append(label)
            
            with open(os.path.join(out_path, name+'_tk'+suffix+'.txt'), 'a') as f:
                f.write(' '.join(row))
                sep='\n'
                f.write(sep)
                    
            # Save adj
            G_u = G.to_undirected()
            adj = nx.adj_matrix(G_u).todense()
            
            final = np.zeros((max_len,max_len), dtype=int)
            final[1:adj.shape[0]+1, 1:adj.shape[1]+1] = adj
            final += np.eye(max_len, dtype=int)
            final[:,0] = np.ones(max_len)
            final[0,:] = np.ones(max_len)
            
            m = sparse.csr_matrix(final)
            sparsedir = os.path.join(out_path, 'adj')
            if not os.path.exists(sparsedir):
                os.makedirs(sparsedir)
            io.mmwrite(os.path.join(sparsedir, prefix+"_"+name+"_adj"+suffix+".mtx"), m)
            if is_train:
                train_count += 1
            else:
                test_count += 1
            snippet_count += 1
        #if idx > 10000:
        #    break
with open(os.path.join(out_path, 'java-val-labels.txt'), 'a') as f:
    for v in method_labels_test:
        f.write(v.lower()+'\n')
with open(os.path.join(out_path, 'java-train-labels.txt'), 'a') as f:
    for v in method_labels_train:
        f.write(v.lower()+'\n')
with open(os.path.join(out_path, 'java-vocab-labels-val.txt'), 'a') as f:
    for v in list(set(method_labels)):
        f.write(v.lower())
        f.write('\n')
count_method_labels = Counter()
with open('../magret/java/methodname/java-train-labels.txt', 'r') as f:
    for l in f:
        method_labels_all.update([l.strip('\n')])
        count_method_labels.update([l.strip('\n')])
with open('../magret/java/methodname/data-val/java-vocab-labels-val.txt', 'r') as f:
    for l in f:
        method_labels_all.update([l.strip('\n')])
        count_method_labels.update([l.strip('\n')])
with open(os.path.join(out_path, 'java-vocab-labels-all.txt'), 'a') as f:
    for v in list(method_labels_all.keys()):
        f.write(v.lower())
        f.write('\n')

len(Counter(method_labels_all).keys())
count_labels
count_method_labels = Counter()
with open('../magret/java-subset/methodname/java-train-labels.txt', 'r') as f:
    for l in f:
        count_method_labels.update([l.strip('\n')])
with open('../magret/java-subset/methodname/java-val-labels.txt', 'r') as f:
    for l in f:
        print(l)
        count_method_labels.update([l.strip('\n')])
count_method_labels.keys()
len({x : count_method_labels[x] for x in count_method_labels if count_method_labels[x] > 1}.keys())
thresh_labels = {x : count_method_labels[x] for x in count_method_labels if count_method_labels[x] > 1}
with open(os.path.join(out_path, 'java-vocab-labels-thresh.txt'), 'a') as f:
    for v in list(thresh_labels.keys()):
        f.write(v.lower())
        f.write('\n')
filename = "data/java/train_java.jsonl"
global_id = 0
n_edges = 0
snippet_count = 0
method_labels = []
method_labels_train = [];method_labels_test = []
name = 'java'
suffix = '_val'
test_count = 0; train_count = 0
max_len = 128
if not os.path.exists(out_path):
    os.makedirs(out_path)
#vocab = set()
funcdef_counts = []

with open(filename, 'r', encoding="ISO-8859-1") as f:
    for idx, line in enumerate(f):
        nb_funcdef = 0
        is_funcdef = False
        data = json.loads(line)
        G=nx.Graph()
        row = ["[CLS]"]; row_order = ["[CLS]"]
        funcdef_ids = []
        row_dict = {}
        for r in data:
            #print(r)
            if type(r) == int:
                break
            from_node = r['uid']
            if r['type'] in ['FunctionDeclaration', 'MethodDeclaration']:
                if r.get('value', None) is not None:
                    w = r['type']
                    label = str(r['value'])
                    #print(label)
                    is_funcdef = True
            if r['type'] in ['ClassDeclaration', 'FormalParameter', 'VariableDeclarator', 'SuperMethodInvocation', 'CatchClauseParameter', 'ConstructorDeclaration', 'MethodInvocation', 'MemberReference']:
                if r.get('value', None) is not None:
                    if (vocab[r['value']] >= 5):
                        w = str(r['value'])
                    else: 
                        w = r['type']
            else:
                w = r['type']
            if is_funcdef:
                nb_funcdef += 1
            row.append(w)

            for to_node in r.get('children', []):
                n_edges += 1
                G.add_edge(from_node, to_node)
            if G.number_of_nodes() > max_len:
                break
                
        if (G.number_of_nodes() < max_len) and (G.number_of_nodes() > 10) and (is_funcdef == True) and (count_method_labels[label] > 1):
            # Save tk
            # print(row)
            if np.random.uniform() < 0.2:
                is_train = False
                suffix = '_val'
                prefix = str(test_count)
                method_labels_test.append(label)

            else:
                is_train = True
                suffix = ''
                prefix = str(train_count)
                method_labels_train.append(label)
                    
            # Save adj
            G_u = G.to_undirected()
            adj = nx.adj_matrix(G_u).todense()
            
            final = np.zeros((max_len,max_len), dtype=int)
            final[1:adj.shape[0]+1, 1:adj.shape[1]+1] = adj
            final += np.eye(max_len, dtype=int)
            final[:,0] = np.ones(max_len)
            final[0,:] = np.ones(max_len)
            

            if is_train:
                train_count += 1
            else:
                test_count += 1
            snippet_count += 1
        funcdef_counts.append(nb_funcdef)
        #if idx > 10000:
        #    break
funcdef_counts
import matplotlib.pyplot as plt
plt.hist(funcdef_counts)
plt.title("Number of FuncDef per file")
np.sum(funcdef_counts)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14009947891994315, 'fraction_numerical': 0.009355755566082426, 'mean_word_length': 1.7972838688307387, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '427852', 'license': 'Apache-2.0', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2708, 'n_words': 550, 'path': 'notebook/Java.ipynb', 'repo_name': 'dtsbourg/codegraph-fmt', 'size': '8444'}