<commit_before>import private.pcre as pcre
import private.util
import tables
import unsigned
from future import lc, `[]`
from strutils import toLower, `%`
from math import ceil
import optional_t

# Type definitions {{{
type
  Regex* = ref object
    pattern: string  # not nil
    pcreObj: ptr pcre.Pcre  # not nil
    pcreExtra: ptr pcre.ExtraData  ## nil

    captureNameToId: Table[string, int]

  RegexMatch* = ref object
    pattern*: Regex  ## The regex doing the matching.
                     ## Not nil.
    str*: string  ## The string that was matched against.
                  ## Not nil.
    pcreMatchBounds: seq[Slice[cint]] ## First item is the bounds of the match
                                      ## Other items are the captures
                                      ## `a` is inclusive start, `b` is exclusive end
    matchCache: seq[string]

  Captures* = distinct RegexMatch
  CaptureBounds* = distinct RegexMatch

  SyntaxError* = ref object of Exception
    pos*: int  ## the location of the syntax error in bytes
    pattern*: string  ## the pattern that caused the problem

  StudyError* = ref object of Exception
# }}}

proc getinfo[T](self: Regex, opt: cint): T =
  let retcode = pcre.fullinfo(self.pcreObj, self.pcreExtra, opt, addr result)

  if retcode < 0:
    # XXX Error message that doesn't expose implementation details
    raise newException(FieldError, "Invalid getinfo for $1, errno $2" % [$opt, $retcode])

# Capture accessors {{{
proc captureCount(self: Regex): int =
  ## get the maximum number of captures
  ##
  ## Does not return the number of captured captures
  return getinfo[int](self, pcre.INFO_CAPTURECOUNT)

proc captureNames*(self: Regex): seq[string] =
  result = @[]
  for key in self.captureNameToId.keys:
    result.add(key)

proc captureBounds*(self: RegexMatch): CaptureBounds = return CaptureBounds(self)

proc captures*(self: RegexMatch): Captures = return Captures(self)

proc `[]`*(self: CaptureBounds, i: int): Option[Slice[int]] =
  ## Gets the bounds of the `i`th capture.
  ## Undefined behavior if `i` is out of bounds
  ## If `i` is a failed optional capture, returns None
  ## If `i == -1`, returns the whole match
  let self = RegexMatch(self)
  if self.pcreMatchBounds[i + 1].a != -1:
    let bounds = self.pcreMatchBounds[i + 1]
    return Some(int(bounds.a) .. int(bounds.b))
  else:
    return None[Slice[int]]()

proc `[]`*(self: Captures, i: int): string =
  ## gets the `i`th capture
  ## Undefined behavior if `i` is out of bounds
  ## If `i` is a failed optional capture, returns nil
  ## If `i == -1`, returns the whole match
  let self = RegexMatch(self)
  let bounds = self.captureBounds[i]

  if bounds:
    let bounds = bounds.get
    if self.matchCache == nil:
      # capture count, plus the entire string
      self.matchCache = newSeq[string](self.pattern.captureCount + 1)
    if self.matchCache[i + 1] == nil:
      self.matchCache[i + 1] = self.str[bounds.a .. bounds.b-1]
    return self.matchCache[i + 1]
  else:
    return nil

proc match*(self: RegexMatch): string =
  return self.captures[-1]

proc matchBounds*(self: RegexMatch): Slice[int] =
  return self.captureBounds[-1].get

proc `[]`*(self: CaptureBounds, name: string): Option[Slice[int]] =
  ## Will fail with KeyError if `name` is not a real named capture
  let self = RegexMatch(self)
  return self.captureBounds[self.pattern.captureNameToId.fget(name)]

proc `[]`*(self: Captures, name: string): string =
  ## Will fail with KeyError if `name` is not a real named capture
  let self = RegexMatch(self)
  return self.captures[self.pattern.captureNameToId.fget(name)]

template asTableImpl(cond: bool): stmt {.immediate, dirty.} =
  for key in RegexMatch(self).pattern.captureNames:
    let nextVal = self[key]
    if cond:
      result[key] = default<commit_msg>Expose the pattern string, rename some procs<commit_after>import private.pcre as pcre
import private.util
import tables
import unsigned
from future import lc, `[]`
from strutils import toLower, `%`
from math import ceil
import optional_t

# Type definitions {{{
type
  Regex* = ref object
    pattern*: string  # not nil
    pcreObj: ptr pcre.Pcre  # not nil
    pcreExtra: ptr pcre.ExtraData  ## nil

    captureNameToId: Table[string, int]

  RegexMatch* = ref object
    pattern*: Regex  ## The regex doing the matching.
                     ## Not nil.
    str*: string  ## The string that was matched against.
                  ## Not nil.
    pcreMatchBounds: seq[Slice[cint]] ## First item is the bounds of the match
                                      ## Other items are the captures
                                      ## `a` is inclusive start, `b` is exclusive end
    matchCache: seq[string]

  Captures* = distinct RegexMatch
  CaptureBounds* = distinct RegexMatch

  SyntaxError* = ref object of Exception
    pos*: int  ## the location of the syntax error in bytes
    pattern*: string  ## the pattern that caused the problem

  StudyError* = ref object of Exception
# }}}

proc getinfo[T](self: Regex, opt: cint): T =
  let retcode = pcre.fullinfo(self.pcreObj, self.pcreExtra, opt, addr result)

  if retcode < 0:
    # XXX Error message that doesn't expose implementation details
    raise newException(FieldError, "Invalid getinfo for $1, errno $2" % [$opt, $retcode])

# Regex accessors {{{
proc captureCount*(self: Regex): int =
  ## Get the maximum number of captures
  ##
  ## Does not return the number of captured captures
  return getinfo[int](self, pcre.INFO_CAPTURECOUNT)

proc captureNameId*(self: Regex): Table[string, int] =
  ## Returns a map from named capture groups to their numerical
  ## identifier
  return self.captureNameToId
# }}}

# Capture accessors {{{
proc captureBounds*(self: RegexMatch): CaptureBounds = return CaptureBounds(self)

proc captures*(self: RegexMatch): Captures = return Captures(self)

proc `[]`*(self: CaptureBounds, i: int): Option[Slice[int]] =
  ## Gets the bounds of the `i`th capture.
  ## Undefined behavior if `i` is out of bounds
  ## If `i` is a failed optional capture, returns None
  ## If `i == -1`, returns the whole match
  let self = RegexMatch(self)
  if self.pcreMatchBounds[i + 1].a != -1:
    let bounds = self.pcreMatchBounds[i + 1]
    return Some(int(bounds.a) .. int(bounds.b))
  else:
    return None[Slice[int]]()

proc `[]`*(self: Captures, i: int): string =
  ## gets the `i`th capture
  ## Undefined behavior if `i` is out of bounds
  ## If `i` is a failed optional capture, returns nil
  ## If `i == -1`, returns the whole match
  let self = RegexMatch(self)
  let bounds = self.captureBounds[i]

  if bounds:
    let bounds = bounds.get
    if self.matchCache == nil:
      # capture count, plus the entire string
      self.matchCache = newSeq[string](self.pattern.captureCount + 1)
    if self.matchCache[i + 1] == nil:
      self.matchCache[i + 1] = self.str[bounds.a .. bounds.b-1]
    return self.matchCache[i + 1]
  else:
    return nil

proc match*(self: RegexMatch): string =
  return self.captures[-1]

proc matchBounds*(self: RegexMatch): Slice[int] =
  return self.captureBounds[-1].get

proc `[]`*(self: CaptureBounds, name: string): Option[Slice[int]] =
  ## Will fail with KeyError if `name` is not a real named capture
  let self = RegexMatch(self)
  return self.captureBounds[self.pattern.captureNameToId.fget(name)]

proc `[]`*(self: Captures, name: string): string =
  ## Will fail with KeyError if `name` is not a real named capture
  let self = RegexMatch(self)
  return self.captures[self.pattern.captureNameToId.fget(name)]

template asTableImpl(cond: bool): stmt {.immediate, dirty.} =
  for key in RegexMatch(self).pattern.captureNameId.keys:
    let nextVal = self[key]
    if cond:
      result[key] = default
{'fraction_non_alphanumeric': 0.1142636058808357, 'fraction_numerical': 0.003868970853752902, 'mean_word_length': 3.366554054054054, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2297574', 'n_tokens_mistral': 2447, 'n_tokens_neox': 2383, 'n_words': 827}