<commit_before>"""Vimdoc blocks: encapsulated chunks of documentation."""
from collections import OrderedDict
import warnings

import vimdoc
from vimdoc import error
from vimdoc import paragraph
from vimdoc import regex


class Block(object):
  """Blocks are encapsulated chunks of documentation.

  They consist of a number of paragraphs and an optional header. The paragraphs
  can come in many types, including text, lists, or code. The block may also
  contain metadata statements specifying things like the plugin author, etc.
  """

  def __init__(self, is_secondary=False):
    # May include:
    # deprecated (boolean)
    # dict (name)
    # private (boolean, in function)
    # name (of section)
    # id (of section, in section or backmatter)
    # namespace (of function)
    # attribute (of function in dict)
    self.locals = {}
    # Merged into module. May include:
    # author (string)
    # library (boolean)
    # order (list of strings)
    # standalone (boolean)
    # stylization (string)
    # tagline (string)
    self.globals = {}
    self.header = None
    self.paragraphs = paragraph.Paragraphs()
    self._required_args = []
    self._optional_args = []
    self._closed = False
    self._is_secondary = is_secondary

  def AddLine(self, line):
    """Adds a line of text to the block. Paragraph type is auto-determined."""
    # Always grab the required/optional args.
    self._ParseArgs(line)
    # Code blocks are treated differently:
    # Newlines aren't joined and blanklines aren't special.
    # See :help help-writing for specification.
    if self.paragraphs.IsType(paragraph.CodeBlock):
      # '<' exits code blocks.
      if line.startswith('<'):
        self.paragraphs.Close()
        line = line[1:].lstrip()
        if line:
          self.AddLine(line)
        return
      # Lines starting in column 0 exit code lines.
      if line[:1] not in ' \t':
        self.paragraphs.Close()
        self.AddLine(line)
        return
      self.paragraphs.AddLine(line)
      return
    # Blank lines divide paragraphs.
    if not line.strip():
      self.paragraphs.SetType(paragraph.BlankLine)
      return
    # Start lists if you get a list item.
    match = regex.list_item.match(line or '')
    if match:
      leader = match.group(1)
      self.paragraphs.Close()
      line = regex.list_item.sub('', line)
      self.paragraphs.SetType(paragraph.ListItem, leader)
      self.paragraphs.AddLine(line)
      return
    if line and line[:1] in ' \t':
      # Continue lists by indenting.
      if self.paragraphs.IsType(paragraph.ListItem):
        self.paragraphs.AddLine(line.lstrip())
        return
    elif self.paragraphs.IsType(paragraph.ListItem):
      self.paragraphs.Close()
    # Everything else is text.
    self.paragraphs.SetType(paragraph.TextParagraph)
    # Lines ending in '>' enter code blocks.
    if line.endswith('>'):
      line = line[:-1].rstrip()<commit_msg>Stop erroneously detecting arguments within code blocks<commit_after>"""Vimdoc blocks: encapsulated chunks of documentation."""
from collections import OrderedDict
import warnings

import vimdoc
from vimdoc import error
from vimdoc import paragraph
from vimdoc import regex


class Block(object):
  """Blocks are encapsulated chunks of documentation.

  They consist of a number of paragraphs and an optional header. The paragraphs
  can come in many types, including text, lists, or code. The block may also
  contain metadata statements specifying things like the plugin author, etc.
  """

  def __init__(self, is_secondary=False):
    # May include:
    # deprecated (boolean)
    # dict (name)
    # private (boolean, in function)
    # name (of section)
    # id (of section, in section or backmatter)
    # namespace (of function)
    # attribute (of function in dict)
    self.locals = {}
    # Merged into module. May include:
    # author (string)
    # library (boolean)
    # order (list of strings)
    # standalone (boolean)
    # stylization (string)
    # tagline (string)
    self.globals = {}
    self.header = None
    self.paragraphs = paragraph.Paragraphs()
    self._required_args = []
    self._optional_args = []
    self._closed = False
    self._is_secondary = is_secondary

  def AddLine(self, line):
    """Adds a line of text to the block. Paragraph type is auto-determined."""
    # Code blocks are treated differently:
    # Newlines aren't joined and blanklines aren't special.
    # See :help help-writing for specification.
    if self.paragraphs.IsType(paragraph.CodeBlock):
      # '<' exits code blocks.
      if line.startswith('<'):
        self.paragraphs.Close()
        line = line[1:].lstrip()
        if line:
          self.AddLine(line)
        return
      # Lines starting in column 0 exit code lines.
      if line[:1] not in ' \t':
        self.paragraphs.Close()
        self.AddLine(line)
        return
      self.paragraphs.AddLine(line)
      return
    # Always grab the required/optional args.
    self._ParseArgs(line)
    # Blank lines divide paragraphs.
    if not line.strip():
      self.paragraphs.SetType(paragraph.BlankLine)
      return
    # Start lists if you get a list item.
    match = regex.list_item.match(line or '')
    if match:
      leader = match.group(1)
      self.paragraphs.Close()
      line = regex.list_item.sub('', line)
      self.paragraphs.SetType(paragraph.ListItem, leader)
      self.paragraphs.AddLine(line)
      return
    if line and line[:1] in ' \t':
      # Continue lists by indenting.
      if self.paragraphs.IsType(paragraph.ListItem):
        self.paragraphs.AddLine(line.lstrip())
        return
    elif self.paragraphs.IsType(paragraph.ListItem):
      self.paragraphs.Close()
    # Everything else is text.
    self.paragraphs.SetType(paragraph.TextParagraph)
    # Lines ending in '>' enter code blocks.
    if line.endswith('>'):
      line = line[:-1].rstrip()
{'fraction_non_alphanumeric': 0.09962724500169434, 'fraction_numerical': 0.0020332090816672314, 'mean_word_length': 2.8256642903434868, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7412841', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1674, 'n_words': 539}