<commit_before>#!/bin/bash
#
# Experiment: When is .*? necessary?
#
# Conjecture: almost nowhere in regular languages.  It's a performance
# optimization for backtracking engines.  You can trivially rewrite most
# regexes without it.
#
# By definition .* vs. .*? doesn't affect matching.  It only affects submatch
# extraction.  (e.g. the "recognition problem" vs. the "parse problem")
#
# Usage:
#   ./greedy.sh <function name>

set -o nounset
set -o pipefail
set -o errexit

csv() {
  cat <<EOF
foo,bar
spam,eggs
EOF
}

html() {
  cat <<EOF
<div foo="bar>
</div>
EOF
}


demo() {
  csv | gawk 'match($0, "(.*),(.*)", a) { print "0=" a[0] "\t1=" a[1] "\t2=" a[2] }'

  echo ---

  html | gawk 'match($0, "<(.*)>", a) { print "0=" a[0] "\t1=" a[1] }'

  html | python2 -c '
import re, sys

pat = re.compile("<(.*)>")

for line in sys.stdin:
  m = pat.match(line)
  if m:
    print(m.groups())
'
}

submatch-demo() {
  csv | ./pygrep.py '(.+)(.+)'

  csv | ./pygrep.py '(.+?)(.+?)'

  csv | ./pygrep.py '(.+?)(.+)'

  csv | ./pygrep.py '(.+)(.+?)'
}

survey() {
  local out=$PWD/survey.txt
  cd ~/git/oilshell/oil
  fgrep '.*?' */*.py | tee $out
}
<commit_msg>[regular-languages] Test out nongreedy submatch with ripgrep too<commit_after>#!/bin/bash
#
# Experiment: When is .*? necessary?
#
# Conjecture: almost nowhere in regular languages.  It's a performance
# optimization for backtracking engines.  You can trivially rewrite most
# regexes without it.
#
# By definition .* vs. .*? doesn't affect matching.  It only affects submatch
# extraction.  (e.g. the "recognition problem" vs. the "parse problem")
#
# Usage:
#   ./greedy.sh <function name>

set -o nounset
set -o pipefail
set -o errexit

csv() {
  cat <<EOF
foo,bar
spam,eggs
EOF
}

html() {
  cat <<EOF
<div foo="bar>
</div>
EOF
}


demo() {
  csv | gawk 'match($0, "(.*),(.*)", a) { print "0=" a[0] "\t1=" a[1] "\t2=" a[2] }'

  echo ---

  html | gawk 'match($0, "<(.*)>", a) { print "0=" a[0] "\t1=" a[1] }'

  html | python2 -c '
import re, sys

pat = re.compile("<(.*)>")

for line in sys.stdin:
  m = pat.match(line)
  if m:
    print(m.groups())
'
}

lines() {
  echo spam
  echo eggs
}


RG=~/install/ripgrep-0.10.0-x86_64-unknown-linux-musl/rg

submatch-demo() {
  # note:  POSIX regexes don't have this construct, so we use ripgrep.
  for g in ./pygrep.py $RG; do
    echo $g
    echo ===

    for pat in \
      '(.+)(.+)' \
      '(.+?)(.+?)' \
      '(.+)(.+?)' \
      '(.+?)(.+)'
    do
      # woah ripgrep does multiple matches on a line

      lines | $g $pat -r '[ 1: $1 2: $2 ]'
    done
  done
}

survey() {
  local out=$PWD/survey.txt
  cd ~/git/oilshell/oil
  fgrep '.*?' */*.py | tee $out
}

{'fraction_non_alphanumeric': 0.21292491595069107, 'fraction_numerical': 0.014194994396712738, 'mean_word_length': 2.72461752433936, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6690074', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1090, 'n_words': 303}