<commit_before>import numpy as np
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from scipy.io import loadmat, savemat
from scipy.sparse import dok_matrix
from collections import defaultdict
from struct import pack
from sys import exit


def mat_to_bcsr(fname, mat):
    with open(fname, 'wb') as outf:
        nv = mat.shape[0]
        ne = mat.nnz
        print('nv', nv, 'ne', ne)
        outf.write(str.encode('XGFS'))
        outf.write(pack('q', nv))
        outf.write(pack('q', ne))
        outf.write(pack(str(nv) + 'i', *mat.indptr[:-1]))
        outf.write(pack(str(ne) + 'i', *mat.indices))


def is_numbers_only(nodes):
    for node in nodes:
        try:
            int(node)
        except ValueError:
            return False
    return True


def process(args):
    if args.format == "mat":
        mtx = loadmat(args.input)[args.matfile_variable_name].tocsr()
        if args.undirected:
            mtx = mtx + mtx.T  # we dont care about weights anyway
        mat_to_bcsr(args.output, mtx)
    elif args.format in ['edgelist', 'adjlist']:
        pass
    else:
        raise Exception(
            "Unknown file format: '%s'.  Valid formats: 'adjlist', 'edgelist', 'mat'" % args.format)
    nodes = set()
    with open(args.input, 'r') as inf:
        for line in inf:
            if line.startswith('#'):
                continue
            if args.sep is None:
                splt = line.split()
            else:
                splt = line.split(args.sep)
            if args.format == "edgelist":
                if len(splt) == 3:
                    if abs(float(splt[2]) - 1) >= 1e-4:
                        raise ValueError("Weighted graphs are not supported")
                    else:
                        splt = splt[:-1]
                else:
                    raise ValueError("Incorrect graph format")        
            for node in splt:
                nodes.add(node)
    number_of_nodes = len(nodes)
    isnumbers = is_numbers_only(nodes)
    print('Node IDs are numbers: ', isnumbers)
    if isnumbers:
        node2id = dict(zip(sorted(map(int, nodes)), range(number_of_nodes)))
    else:
        node2id = dict(zip(nodes), range(number_of_nodes))
    graph = defaultdict(set)
    with open(args.input, 'r') as inf:
        for line in inf:
            if line.startswith('#'):
                continue
            if args.sep is None:
                splt = line.split()
            else:
                splt = line.split(args.sep)
            if isnumbers:
                src = node2id[int(splt[0])]
            else:
                src = node2id[splt[0]]
            if args.format == "edgelist" and len(splt) == 3:
                splt = splt[:-1]
            for node in splt[1:]:
                if isnumbers:
                    tgt = node2id[int(node)]
                else:
                    tgt = node2id[node]
                graph[src].add(tgt)
                if args.undirected:
                    graph[tgt].add(src)
    indptr = np.zeros(number_of_nodes + 1, dtype=np.int32)
    indptr[0] = 0<commit_msg>strip the lines in the convert script<commit_after>import numpy as np
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from scipy.io import loadmat, savemat
from scipy.sparse import dok_matrix
from collections import defaultdict
from struct import pack
from sys import exit


def mat_to_bcsr(fname, mat):
    with open(fname, 'wb') as outf:
        nv = mat.shape[0]
        ne = mat.nnz
        print('nv', nv, 'ne', ne)
        outf.write(str.encode('XGFS'))
        outf.write(pack('q', nv))
        outf.write(pack('q', ne))
        outf.write(pack(str(nv) + 'i', *mat.indptr[:-1]))
        outf.write(pack(str(ne) + 'i', *mat.indices))


def is_numbers_only(nodes):
    for node in nodes:
        try:
            int(node)
        except ValueError:
            return False
    return True


def process(args):
    if args.format == "mat":
        mtx = loadmat(args.input)[args.matfile_variable_name].tocsr()
        if args.undirected:
            mtx = mtx + mtx.T  # we dont care about weights anyway
        mat_to_bcsr(args.output, mtx)
    elif args.format in ['edgelist', 'adjlist']:
        pass
    else:
        raise Exception(
            "Unknown file format: '%s'.  Valid formats: 'adjlist', 'edgelist', 'mat'" % args.format)
    nodes = set()
    with open(args.input, 'r') as inf:
        for line in inf:
            if line.startswith('#'):
                continue
            line = line.strip()
            if args.sep is None:
                splt = line.split()
            else:
                splt = line.split(args.sep)
            if args.format == "edgelist":
                if len(splt) == 3:
                    if abs(float(splt[2]) - 1) >= 1e-4:
                        raise ValueError("Weighted graphs are not supported")
                    else:
                        splt = splt[:-1]
                else:
                    raise ValueError("Incorrect graph format")        
            for node in splt:
                nodes.add(node)
    number_of_nodes = len(nodes)
    isnumbers = is_numbers_only(nodes)
    print('Node IDs are numbers: ', isnumbers)
    if isnumbers:
        node2id = dict(zip(sorted(map(int, nodes)), range(number_of_nodes)))
    else:
        node2id = dict(zip(nodes), range(number_of_nodes))
    graph = defaultdict(set)
    with open(args.input, 'r') as inf:
        for line in inf:
            if line.startswith('#'):
                continue
            line = line.strip()
            if args.sep is None:
                splt = line.split()
            else:
                splt = line.split(args.sep)
            if isnumbers:
                src = node2id[int(splt[0])]
            else:
                src = node2id[splt[0]]
            if args.format == "edgelist" and len(splt) == 3:
                splt = splt[:-1]
            for node in splt[1:]:
                if isnumbers:
                    tgt = node2id[int(node)]
                else:
                    tgt = node2id[node]
                graph[src].add(tgt)
                if args.undirected:
                    graph[tgt].add(src)
    indptr = np.zeros(number_of_nodes + 1, dtype=np.int32)
    indptr[0] = 0
{'fraction_non_alphanumeric': 0.11422208081450844, 'fraction_numerical': 0.007636016544702514, 'mean_word_length': 1.6338500209467952, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3986375', 'n_tokens_mistral': 1985, 'n_tokens_neox': 1908, 'n_words': 489}