<commit_before># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import app.background
import app.log
import app.selectable
import app.prefs
import third_party.pyperclip as clipboard
import curses.ascii
import os
import re
import sys
import time
import traceback


class ParserNode:
  """A parser node represents a span of grammar. i.e. from this point to that
      point is HTML. Another parser node would represent the next segment, of
      grammar (maybe JavaScript, CSS, comment, or quoted string for example."""
  def __init__(self):
    self.grammar = None
    self.prior = None  # Index of prior grammar (like a stack of grammars).
    self.begin = None  # Offset from start of file.

  def debugLog(self, out, indent, data):
    out('%sParserNode %26s prior %4s %4d %s' % (indent,
        self.grammar.get('name', 'None'),
        self.prior, self.begin, repr(data[self.begin:self.begin+15])[1:-1]))


class Parser:
  """A parser generates a set of grammar segments (ParserNode objects)."""
  def __init__(self):
    self.data = ""
    self.emptyNode = ParserNode()
    self.emptyNode.grammar = {}
    self.endNode = ParserNode()
    self.endNode.grammar = {}
    self.endNode.begin = sys.maxint
    self.endNode.col = sys.maxint
    self.parserNodes = []
    self.rows = []  # Row parserNodes index.
    app.log.parser('__init__')

  def grammarIndexFromRowCol(self, row, col):
    """
    returns index. |index| may then be passed to grammarNext().
    """
    if row + 1 >= len(self.rows): # or self.rows[row + 1] > len(self.parserNodes):
      # This file is too large. There's other ways to handle this, but for now
      # let's leave the tail un-highlighted.
      return 0
    gl = self.parserNodes[self.rows[row]:self.rows[row + 1]] + [self.endNode]
    offset = gl[0].begin + col
    # Binary search to find the node for the column.
    low = 0
    high = len(gl) - 1
    while True:
      index = (high + low) / 2
      if offset >= gl[index + 1].begin:
        low = index
      elif offset < gl[index].begin:
        high = index
      else:
        return index

  def grammarAtIndex(self, row, col, index):
    """
    Call grammarIndexFromRowCol() to get the index parameter.
    returns (node, preceding, remaining). |index| may then be passed to
        grammarNext(). |proceeding| and |remaining| are relative to the |col|
        parameter.
    """
    finalResult = (self.emptyNode, 0, sys.maxint)
    if row + 1 >= len(self.rows):
      return finalResult
    nextRowIndex = self.rows[row + 1]
    if col >= self.parserNodes[nextRowIndex].begin:
      return finalResult
    rowIndex = self.rows[row]<commit_msg>start final parse node start to begining of line; fix #81<commit_after># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import app.background
import app.log
import app.selectable
import app.prefs
import third_party.pyperclip as clipboard
import curses.ascii
import os
import re
import sys
import time
import traceback


class ParserNode:
  """A parser node represents a span of grammar. i.e. from this point to that
      point is HTML. Another parser node would represent the next segment, of
      grammar (maybe JavaScript, CSS, comment, or quoted string for example."""
  def __init__(self):
    self.grammar = None
    self.prior = None  # Index of prior grammar (like a stack of grammars).
    self.begin = None  # Offset from start of file.

  def debugLog(self, out, indent, data):
    out('%sParserNode %26s prior %4s %4d %s' % (indent,
        self.grammar.get('name', 'None'),
        self.prior, self.begin, repr(data[self.begin:self.begin+15])[1:-1]))


class Parser:
  """A parser generates a set of grammar segments (ParserNode objects)."""
  def __init__(self):
    self.data = ""
    self.emptyNode = ParserNode()
    self.emptyNode.grammar = {}
    self.endNode = ParserNode()
    self.endNode.grammar = {}
    self.endNode.begin = sys.maxint
    self.endNode.col = sys.maxint
    self.parserNodes = []
    self.rows = []  # Row parserNodes index.
    app.log.parser('__init__')

  def grammarIndexFromRowCol(self, row, col):
    """
    returns index. |index| may then be passed to grammarNext().
    """
    if row + 1 >= len(self.rows): # or self.rows[row + 1] > len(self.parserNodes):
      # This file is too large. There's other ways to handle this, but for now
      # let's leave the tail un-highlighted.
      return 0
    gl = self.parserNodes[self.rows[row]:self.rows[row + 1]] + [self.endNode]
    offset = gl[0].begin + col
    # Binary search to find the node for the column.
    low = 0
    high = len(gl) - 1
    while True:
      index = (high + low) / 2
      if offset >= gl[index + 1].begin:
        low = index
      elif offset < gl[index].begin:
        high = index
      else:
        return index

  def grammarAtIndex(self, row, col, index):
    """
    Call grammarIndexFromRowCol() to get the index parameter.
    returns (node, preceding, remaining). |index| may then be passed to
        grammarNext(). |proceeding| and |remaining| are relative to the |col|
        parameter.
    """
    finalResult = (self.emptyNode, -col, sys.maxint)
    if row + 1 >= len(self.rows):
      return finalResult
    nextRowIndex = self.rows[row + 1]
    if col >= self.parserNodes[nextRowIndex].begin:
      return finalResult
    rowIndex = self.rows[row]
{'fraction_non_alphanumeric': 0.10568341944574917, 'fraction_numerical': 0.008924377642085486, 'mean_word_length': 3.043037974683544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1818531', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1881, 'n_words': 709}