<commit_before>import json
import networkx as nx
from eden import util

def gspan_to_eden(input = None, options = dict()):
    """
    Takes a string list in the extended gSpan format and yields networkx graphs.

    Parameters
    ----------
    input : string
        A pointer to the data source.

    """

    string_list = []
    for line in util.read( input ):
        if line.strip():
            if line[0] in ['g','t']:
                if string_list:
                    yield gspan_to_networkx(string_list)
                string_list = []
            string_list += [line]

    if string_list:
        yield gspan_to_networkx(string_list)


def gspan_to_networkx(string_list):
    G = nx.Graph()
    for line in string_list:
        if line.strip():
            line_list = line.split()
            fc = line_list[0]
            #process vertices
            if fc in ['v','V'] : 
                vid = int(line_list[1])
                vlabel = line_list[2]
                #lowercase v indicates active viewpoint
                if fc == 'v':
                    weight = 1
                else: #uppercase v indicates no-viewpoint
                    weight = 0.1
                G.add_node(vid, label = vlabel, weight = weight)
                #abstract vertices
                if vlabel[0] == '^':
                    G.node[vid]['nesting'] = True
                #extract the rest of the line  as a JSON string that contains all attributes
                attribute_str=' '.join(line_list[3:])
                if attribute_str.strip():
                    attribute_dict = json.loads(attribute_str)
                    G.node[vid].update(attribute_dict)<commit_msg>cleaned gspan<commit_after>import json
import networkx as nx
from eden import util

def gspan_to_eden(input = None, options = dict()):
    """
    Takes a string list in the extended gSpan format and yields networkx graphs.

    Parameters
    ----------
    input : string
        A pointer to the data source.

    """

    string_list = []
    for line in util.read( input ):
        if line.strip():
            if line[0] in ['g','t']:
                if string_list:
                    yield gspan_to_networkx(string_list)
                string_list = []
            string_list += [line]

    if string_list:
        yield gspan_to_networkx(string_list)


def gspan_to_networkx(string_list):
    G = nx.Graph()
    for line in string_list:
        if line.strip():
            line_list = line.split()
            fc = line_list[0]
            #process vertices
            if fc in ['v','V'] : 
                vid = int(line_list[1])
                vlabel = line_list[2]
                #lowercase v indicates active viewpoint
                if fc == 'v':
                    weight = 1
                else: #uppercase v indicates no-viewpoint, in the new EDeN this is simulated via a smaller weight
                    weight = 0.1
                G.add_node(vid, label = vlabel, weight = weight)
                #extract the rest of the line  as a JSON string that contains all attributes
                attribute_str=' '.join(line_list[3:])
                if attribute_str.strip():
                    attribute_dict = json.loads(attribute_str)
                    G.node[vid].update(attribute_dict)
{'fraction_non_alphanumeric': 0.09206541490006057, 'fraction_numerical': 0.00514839491217444, 'mean_word_length': 1.5584817970565452, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4712864', 'n_tokens_mistral': 943, 'n_tokens_neox': 888, 'n_words': 271}