<commit_before>from pyinsect.documentModel.representations.DocumentNGramGraph import \
    DocumentNGramGraph


class DocumentNGramHSubGraph(DocumentNGramGraph):
    def __init__(self, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.lookup_table = []

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        super().buildGraph(verbose=verbose, d=d)

        for node in self._Graph.nodes:
            subset = [node] + list(self._Graph.neighbors(node))
            subgraph = self._Graph.subgraph(subset)
            self.lookup_table.append(subgraph)

        return self._Graph

    def addEdgeInc(self, a, b, w=1):
        # A = repr(a)#str(a)
        # B = repr(b)#str(b)
        # merging can also be done in other ways
        # add an extra class variable
        A, B = tuple(a), tuple(b)
        if (A, B) in self._edges:
            edata = self._Graph.get_edge_data(A, B)
            # DEBUG LINES
            # print "updating edge between (",A,B,")"
            # print "to weight",(edata['weight']+1)

            r = edata["weight"] + w
        else:
            # DEBUG LINES
            # print "adding edge between (",A,B,")"

            r = w
        # update/add edge weight
        self.setEdge(A, B, r)


class DocumentNGramHGraph(DocumentNGramHSubGraph):
    def __init__(self, levels, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.levels = levels

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        sequence = self._Data
        for level in range(1, self.levels):
            window_size = int(self._Dwin * level)

            document_n_gram_graph = DocumentNGramHSubGraph(
                n=self._n, Dwin=window_size, Data=sequence, GPrintVerbose=True
            )

            sequence = document_n_gram_graph.lookup_table

        self._Dwin = int(self._Dwin * level)
        super().buildGraph(verbose=verbose, d=sequence)

        return self._Graph<commit_msg>fix: Adding `+1` to level field<commit_after>from pyinsect.documentModel.representations.DocumentNGramGraph import DocumentNGramGraph


class DocumentNGramHSubGraph(DocumentNGramGraph):
    def __init__(self, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.lookup_table = []

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        super().buildGraph(verbose=verbose, d=d)

        for node in self._Graph.nodes:
            subset = [node] + list(self._Graph.neighbors(node))
            subgraph = self._Graph.subgraph(subset)
            self.lookup_table.append(subgraph)

        return self._Graph

    def addEdgeInc(self, a, b, w=1):
        # A = repr(a)#str(a)
        # B = repr(b)#str(b)
        # merging can also be done in other ways
        # add an extra class variable
        A, B = tuple(a), tuple(b)
        if (A, B) in self._edges:
            edata = self._Graph.get_edge_data(A, B)
            # DEBUG LINES
            # print "updating edge between (",A,B,")"
            # print "to weight",(edata['weight']+1)

            r = edata["weight"] + w
        else:
            # DEBUG LINES
            # print "adding edge between (",A,B,")"

            r = w
        # update/add edge weight
        self.setEdge(A, B, r)


class DocumentNGramHGraph(DocumentNGramHSubGraph):
    def __init__(self, levels, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.levels = levels
        self.original_data = Data

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        sequence = self._Data
        for level in range(1, self.levels):
            window_size = int(self._Dwin * level)

            document_n_gram_graph = DocumentNGramHSubGraph(
                n=self._n, Dwin=window_size, Data=sequence, GPrintVerbose=True
            )

            sequence = document_n_gram_graph.lookup_table

        self._Dwin = int(self._Dwin * (level + 1))
        super().buildGraph(verbose=verbose, d=sequence)

        return self._Graph
{'fraction_non_alphanumeric': 0.11971496437054632, 'fraction_numerical': 0.003800475059382423, 'mean_word_length': 2.412479740680713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '826951', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1332, 'n_words': 291}