<commit_before>##
#
# cmap_parse.py
# An attempt to parse concept maps, exported from cmap tools...take one
#
# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and limitations under the
# License.
#
##

import glob
import itertools
import networkx as nx


def CmapParse (cmap_files, result, filenames, root_concept):
                
        # open the result file to write output
        rfile = open(result, 'w')
        rfile.write('Filename\t\t NC\t NH\t HH\t NCL\t\n\n')

        # iterate over all the files and start doing stuffs
        for index, cmap_file in enumerate(cmap_files):

                # create an empty Multi-directed graph
                G = nx.MultiDiGraph ()

                # open a cmap text file and begin writing results
                f = open (cmap_file)
                rfile.write(filenames[index] + '\t')

                # split the lines in to a list
                lines = ((f.read ()).splitlines ())

                # iterate over the list and split each line
                # in to individual lists, delimited by tab
                textFormatCorrect = True
                for line in lines:
                        edge = line.split ('\t')

                        # break if not 3 items per line
                        textFormatCorrect = True
                        if len(edge) != 3:
                                rfile.write('>> Text file not formatted correctly.\n\n')
                                textFormatCorrect = False
                                break

                        G.add_edge (edge[0].lower(), edge[2].lower(), link=edge[1].lower())

                # if the file had a line without 3 items, break completely
                if not textFormatCorrect:
                        continue

                # if 'sustainability' isn't a concept, fail
                if root_concept.lower() not in G:
                        rfile.write('>> ' + root_concept.lower() + ' not a concept in the map.\n\n')
                        continue
                        
                # number of concepts is the number of nodes
                # minus the root node
                num_concepts = G.order () - 1

                # hierarchy is the out degree of the root node
                # we assume the root is 'sustainabiliy'
                hierarchy = G.out_degree (root_concept.lower())

                # look at all paths from sustainability to all 
                # other nodes. no repeated nodes (cycles)
                paths_list = []
                for n in G.nodes ():
                        for path in nx.all_simple_paths (G, source=root_concept.lower(), target=n):
                                paths_list.append (path)

                # highest hierarchy defined here as the max path length
                highest_hier = max (len (x) for x in paths_list) - 1
                

                # let's make subgraphs of all hierarchies
                # we can use these subgraphs to do some
                # operations and check out cross links
                hierarchy_list = G.successors (root_concept.lower())
                subgraph_list = []
                for x in hierarchy_list:
                        subgraph = nx.MultiDiGraph ()
                        connected_nodes = []
                        for y in G.nodes ():
                                if nx.has_path (G, x, y):
                                        connected_nodes.append (y)
                                        
                        subgraph = G.subgraph(connected_nodes).copy ()  
                        subgraph.graph['name'] = x
                        subgraph_list.append (subgraph)


                # iterate through the main graph and set hierarchy to zero for now
                for x in G:
                        G.node[x]['hier'] = 0

                                
                # iterate through the subgraph_list and set all hier attr to zero
                for x in subgraph_list:
                        # for all the nodes in a subgraph, set to zero for now
                        for y in x:
                                x.node[y]['hier'] = 0


                # iterate through the top hierarchy in the main graph and set these first-level hierarchy
                # concepts to an incrementing integer
                hierIter = 1
                for x in hierarchy_list:
                        G.node[x]['hier'] = hierIter
                        hierIter += 1

                # iterate through the hierarchy list
                # look at its number through the original graph
                # iterate through the subgraph list
                # pick the subgraph that was made with the chosen hierarchy
                # label all concepts in that subgraph with the same number, with the exception of any concepts that are in the heirarchy list
                for x in hierarchy_list:
                        print 'Hier concept: ' + x
                        currentHierNumber = G.node[x]['hier']
                        print 'Hier number: ' + str(currentHierNumber)
                        for y in subgraph_list:
                                if y.name == x:
                                        for concept in y:
                                                if concept not in hierarchy_list:
                                                        if G.node[concept]['hier'] == 0:
                                                                print concept
                                                                y.node[concept]['hier'] = currentHierNumber
                                                                G.node[concept]['hier'] = currentHierNumber
                                                                print y.node[concept]['hier']


                # now i need to find all edges that have
                # two hier node attributes that don't match.
                # these are crosslinks
                total_crosslinks = 0
                for x in G.edges():
                        if ((G.node[x[0]]['hier']) != 0) and ((G.node[x[1]]['hier']) != 0):
                                if G.node[x[0]]['hier'] != G.node[x[1]]['hier']:
                                        print (str (x[0]) + ' ---- ' + str (x[1]) + ' hier: ' + str (G.node[x[0]]['hier']) + ' ---- ' + str (G.node[x[1]]['hier']))
                                        total_crosslinks += 1
                        

                # print out the stuffs<commit_msg>comment out prints<commit_after>##
#
# cmap_parse.py
# An attempt to parse concept maps, exported from cmap tools...take one
#
# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and limitations under the
# License.
#
##

import glob
import itertools
import networkx as nx


def CmapParse (cmap_files, result, filenames, root_concept):
                
        # open the result file to write output
        rfile = open(result, 'w')
        rfile.write('Filename\t\t NC\t NH\t HH\t NCL\t\n\n')

        # iterate over all the files and start doing stuffs
        for index, cmap_file in enumerate(cmap_files):

                # create an empty Multi-directed graph
                G = nx.MultiDiGraph ()

                # open a cmap text file and begin writing results
                f = open (cmap_file)
                rfile.write(filenames[index] + '\t')

                # split the lines in to a list
                lines = ((f.read ()).splitlines ())

                # iterate over the list and split each line
                # in to individual lists, delimited by tab
                textFormatCorrect = True
                for line in lines:
                        edge = line.split ('\t')

                        # break if not 3 items per line
                        textFormatCorrect = True
                        if len(edge) != 3:
                                rfile.write('>> Text file not formatted correctly.\n\n')
                                textFormatCorrect = False
                                break

                        G.add_edge (edge[0].lower(), edge[2].lower(), link=edge[1].lower())

                # if the file had a line without 3 items, break completely
                if not textFormatCorrect:
                        continue

                # if 'sustainability' isn't a concept, fail
                if root_concept.lower() not in G:
                        rfile.write('>> ' + root_concept.lower() + ' not a concept in the map.\n\n')
                        continue
                        
                # number of concepts is the number of nodes
                # minus the root node
                num_concepts = G.order () - 1

                # hierarchy is the out degree of the root node
                # we assume the root is 'sustainabiliy'
                hierarchy = G.out_degree (root_concept.lower())

                # look at all paths from sustainability to all 
                # other nodes. no repeated nodes (cycles)
                paths_list = []
                for n in G.nodes ():
                        for path in nx.all_simple_paths (G, source=root_concept.lower(), target=n):
                                paths_list.append (path)

                # highest hierarchy defined here as the max path length
                highest_hier = max (len (x) for x in paths_list) - 1
                

                # let's make subgraphs of all hierarchies
                # we can use these subgraphs to do some
                # operations and check out cross links
                hierarchy_list = G.successors (root_concept.lower())
                subgraph_list = []
                for x in hierarchy_list:
                        subgraph = nx.MultiDiGraph ()
                        connected_nodes = []
                        for y in G.nodes ():
                                if nx.has_path (G, x, y):
                                        connected_nodes.append (y)
                                        
                        subgraph = G.subgraph(connected_nodes).copy ()  
                        subgraph.graph['name'] = x
                        subgraph_list.append (subgraph)


                # iterate through the main graph and set hierarchy to zero for now
                for x in G:
                        G.node[x]['hier'] = 0

                                
                # iterate through the subgraph_list and set all hier attr to zero
                for x in subgraph_list:
                        # for all the nodes in a subgraph, set to zero for now
                        for y in x:
                                x.node[y]['hier'] = 0


                # iterate through the top hierarchy in the main graph and set these first-level hierarchy
                # concepts to an incrementing integer
                hierIter = 1
                for x in hierarchy_list:
                        G.node[x]['hier'] = hierIter
                        hierIter += 1

                # iterate through the hierarchy list
                # look at its number through the original graph
                # iterate through the subgraph list
                # pick the subgraph that was made with the chosen hierarchy
                # label all concepts in that subgraph with the same number, with the exception of any concepts that are in the heirarchy list
                for x in hierarchy_list:
                        #print 'Hier concept: ' + x
                        currentHierNumber = G.node[x]['hier']
                        #print 'Hier number: ' + str(currentHierNumber)
                        for y in subgraph_list:
                                if y.name == x:
                                        for concept in y:
                                                if concept not in hierarchy_list:
                                                        if G.node[concept]['hier'] == 0:
                                                                #print concept
                                                                y.node[concept]['hier'] = currentHierNumber
                                                                G.node[concept]['hier'] = currentHierNumber
                                                                #print y.node[concept]['hier']


                # now i need to find all edges that have
                # two hier node attributes that don't match.
                # these are crosslinks
                total_crosslinks = 0
                for x in G.edges():
                        if ((G.node[x[0]]['hier']) != 0) and ((G.node[x[1]]['hier']) != 0):
                                if G.node[x[0]]['hier'] != G.node[x[1]]['hier']:
                                        print (str (x[0]) + ' ---- ' + str (x[1]) + ' hier: ' + str (G.node[x[0]]['hier']) + ' ---- ' + str (G.node[x[1]]['hier']))
                                        total_crosslinks += 1
                        

                # print out the stuffs
{'fraction_non_alphanumeric': 0.07445516255805645, 'fraction_numerical': 0.004715969989281886, 'mean_word_length': 1.1443235789796231, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4897698', 'n_tokens_mistral': 3409, 'n_tokens_neox': 3074, 'n_words': 1201}