<commit_before>import os
import sys
import vim
import subprocess
import bisect

import vimbufsync
vimbufsync.check_version([0,1,0], who="merlindent")

inline_process = None
saved_states = []
saved_sync = None
ocp_indent_path = "ocp-indent"

def sync():
  global saved_sync, saved_states
  curr_sync = vimbufsync.sync()
  if not saved_sync or curr_sync.buf() != saved_sync.buf():
    saved_states = []
  else:
    line = saved_sync.line()
    i = bisect.bisect_left(saved_states, (line,""))
    if i != len(saved_states):
      saved_states = saved_states[:i]
  saved_sync = curr_sync


#########################################################
# Process management, not really related to indentation #
#########################################################

def ocp_indent(content,state=None,lines=None):
  global ocp_indent_path
  if lines:
    if type(lines) == int:
      lines = "%d-%d" % (lines,lines)
    else:
      lines = "%d-%d" % lines
  else:
    lines = "0-0"
  if not state:
    state = ""
  content = "\n".join([state] + content + [""])
  process = subprocess.Popen(
      [ocp_indent_path,"--lines",lines,"--numeric","--rest"],
      stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  process.stdin.write(content)
  process.stdin.close()
  answer = process.stdout.readlines()
  state = answer.pop()
  return (state,answer)

########################################
# Start from here to tweak indentation #
########################################

def emptyline(s):
  return s == "" or s.isspace()

def ocpindentline(l):
  global saved_states

  # Drop invalid states
  if l <= 2:
    saved_states = []
  else:
    i = bisect.bisect_left(saved_states, (l,""))
    if i != len(saved_states):
      saved_states = saved_states[:i]

  # Get or generate state to resume from
  if saved_states:
    l0, state = saved_states[-1]
  else:
    l0, state = 0, ""
  if l - l0 > 10:
    content = vim.current.buffer[l0:l-4]
    (state, _) = ocp_indent(content,state)
    l0 = l-4
    saved_states.append((l0,state))

  # Indent
  content = list(vim.current.buffer[l0:l])
  if emptyline(content[-1]):
    content[-1] = "X"

  _, content = ocp_indent(content,state,lines=l)
  if len(content):
    return int(content[-1])
  else:
    return -1

def vim_sync():
  sync()

def vim_indentline():
  return ocpindentline(int(vim.eval("v:lnum")))

<commit_msg>Switched to "new" ocp-indent.<commit_after>import os
import sys
import vim
import subprocess
import bisect

import vimbufsync
vimbufsync.check_version([0,1,0], who="merlindent")

inline_process = None
saved_states = []
saved_sync = None
ocp_indent_path = "ocp-indent"

def sync():
  global saved_sync, saved_states
  curr_sync = vimbufsync.sync()
  if not saved_sync or curr_sync.buf() != saved_sync.buf():
    saved_states = []
  else:
    line = saved_sync.line()
    i = bisect.bisect_left(saved_states, (line,""))
    if i != len(saved_states):
      saved_states = saved_states[:i]
  saved_sync = curr_sync


#########################################################
# Process management, not really related to indentation #
#########################################################

def ocp_indent(content,state=None,lines=None):
  global ocp_indent_path
  if lines:
    if type(lines) == int:
      lines = "%d-%d" % (lines,lines)
    else:
      lines = "%d-%d" % lines
  else:
    lines = "0-0"
  if not state:
    state = ""
  content = "\n".join([state] + content + [""])
  process = subprocess.Popen(
      [ocp_indent_path,"--lines",lines,"--numeric","--marshal-state"],
      stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  process.stdin.write(content)
  process.stdin.close()
  answer = process.stdout.readlines()
  state = answer.pop()
  print state
  [line,col] = map(int,state.split(",")[:2])
  return ((line-1,col-1,state),answer)

########################################
# Start from here to tweak indentation #
########################################

def emptyline(s):
  return s == "" or s.isspace()

def ocpindentline(l):
  global saved_states

  # Drop invalid states
  if l <= 2:
    saved_states = []
  else:
    i = bisect.bisect_left(saved_states, (l,0,""))
    if i != len(saved_states):
      saved_states = saved_states[:i]

  # Get or generate state to resume from
  if saved_states:
    l0, c0, state = saved_states[-1]
  else:
    l0, c0, state = 0, 0, None
  if l - l0 > 10:
    content = vim.current.buffer[l0:l-4]
    ((l0,c0,state), _) = ocp_indent(content,state)
    saved_states.append((l0,c0,state))

  # Indent
  content = list(vim.current.buffer[l0:l])
  content[0] = content[0][c0:]
  if emptyline(content[-1]):
    content[-1] = "X"

  _, content = ocp_indent(content,state,lines=l)
  if len(content):
    return int(content[-1])
  else:
    return -1

def vim_sync():
  sync()

def vim_indentline():
  return ocpindentline(int(vim.eval("v:lnum")))


{'fraction_non_alphanumeric': 0.21585993820803295, 'fraction_numerical': 0.011740473738414007, 'mean_word_length': 3.970317297850563, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2720881', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1674, 'n_words': 363}