<commit_before>import os
import re

from StringIO import StringIO


class Tokens(object):
  TEXT = 1
  GROUP = 2
  INDENT = 4
  NEWLINE = 8


class GherkinParser(object):

  def __init__(self, gherkin_text):
    self._gherkin_text = StringIO(gherkin_text)
    self._tokens = []
    self._group = []

  def parse(self):
    example_re = re.compile(r'\s*\|[^|]+\|\s*')

    for line in self._gherkin_text.readlines():

      if example_re.match(line):
        self._collect_group_item(line)

      else:
        self._store_and_reset_group()
        line = self._strip_and_store_indent(line)
        self._add_text_token(line)

    self._finish()

    return self._tokens

  def _collect_group_item(self, line):
    elements = [item.strip() for item in line.split('|')[1:-1]]
    self._group.append(elements)

  def _new_group(self):
    self._group = []

  def _store_and_reset_group(self):
    if len(self._group) > 0:
      self._tokens.append((Tokens.GROUP, self._group))
      self._new_group()

  def _strip_and_store_indent(self, line):
    indent_re = re.compile(r'(\ +)')
    match = indent_re.match(line)

    if match:
      self._tokens.append((Tokens.INDENT, match.groups()[0]))
      return line.lstrip()

    return line

  def _add_text_token(self, text):
    if len(text):
      self._tokens.append((Tokens.TEXT, text.rstrip()))

    # TODO: detect exact line seperator used (for non Unix systems)
    self._tokens.append((Tokens.NEWLINE, '\n'))

  def _finish(self):
    self._store_and_reset_group()


class GherkinFormatter(object):

  def __init__(self):
    self._result = StringIO()

  def format(self, parsed):
    for token_type, token in parsed:

      if token_type in [Tokens.TEXT, Tokens.INDENT, Tokens.NEWLINE]:
        self._emit(token)

      elif token_type == Tokens.GROUP:
        self._format_group(token)

      else:
        raise Exception('unsupported token type %s' % token_type)

    result = self._result.getvalue()
    self._result.close()

    return result

  def _format_group(self, group):
    widths = self._determine_column_widths(group)

    for line in group:
      buf = StringIO()
      buf.write('|')

      for idx, col in enumerate(line):
        width = widths[idx]
        padding = width - len(col)
        buf.write(' ')
        buf.write(col)
        buf.write(' ' * padding)
        buf.write(' |')

      buf.write('\n')
      self._emit(buf.getvalue())
      buf.close()

  def _determine_column_widths(self, group):
    widths = []

    for i in range(len(group[0])):
      width = 0
      for j in range(len(group)):
        width = max(len(group[j][i]), width)
      widths.append(width)

    return widths

  def _emit(self, text):
    self._result.write(text)
<commit_msg>Inlined superfluous function<commit_after>import os
import re

from StringIO import StringIO


class Tokens(object):
  TEXT = 1
  GROUP = 2
  INDENT = 4
  NEWLINE = 8


class GherkinParser(object):

  def __init__(self, gherkin_text):
    self._gherkin_text = StringIO(gherkin_text)
    self._tokens = []
    self._group = []

  def parse(self):
    example_re = re.compile(r'\s*\|[^|]+\|\s*')

    for line in self._gherkin_text.readlines():

      if example_re.match(line):
        self._collect_group_item(line)

      else:
        self._store_and_reset_group()
        line = self._strip_and_store_indent(line)
        self._add_text_token(line)

    self._finish()

    return self._tokens

  def _collect_group_item(self, line):
    elements = [item.strip() for item in line.split('|')[1:-1]]
    self._group.append(elements)

  def _store_and_reset_group(self):
    if len(self._group) > 0:
      self._tokens.append((Tokens.GROUP, self._group))
      self._group = []

  def _strip_and_store_indent(self, line):
    indent_re = re.compile(r'(\ +)')
    match = indent_re.match(line)

    if match:
      self._tokens.append((Tokens.INDENT, match.groups()[0]))
      return line.lstrip()

    return line

  def _add_text_token(self, text):
    if len(text):
      self._tokens.append((Tokens.TEXT, text.rstrip()))

    # TODO: detect exact line seperator used (for non Unix systems)
    self._tokens.append((Tokens.NEWLINE, '\n'))

  def _finish(self):
    self._store_and_reset_group()


class GherkinFormatter(object):

  def __init__(self):
    self._result = StringIO()

  def format(self, parsed):
    for token_type, token in parsed:

      if token_type in [Tokens.TEXT, Tokens.INDENT, Tokens.NEWLINE]:
        self._emit(token)

      elif token_type == Tokens.GROUP:
        self._format_group(token)

      else:
        raise Exception('unsupported token type %s' % token_type)

    result = self._result.getvalue()
    self._result.close()

    return result

  def _format_group(self, group):
    widths = self._determine_column_widths(group)

    for line in group:
      buf = StringIO()
      buf.write('|')

      for idx, col in enumerate(line):
        width = widths[idx]
        padding = width - len(col)
        buf.write(' ')
        buf.write(col)
        buf.write(' ' * padding)
        buf.write(' |')

      buf.write('\n')
      self._emit(buf.getvalue())
      buf.close()

  def _determine_column_widths(self, group):
    widths = []

    for i in range(len(group[0])):
      width = 0
      for j in range(len(group)):
        width = max(len(group[j][i]), width)
      widths.append(width)

    return widths

  def _emit(self, text):
    self._result.write(text)

{'fraction_non_alphanumeric': 0.13032305433186492, 'fraction_numerical': 0.003671071953010279, 'mean_word_length': 2.799860529986053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '41045', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1805, 'n_words': 389}