<commit_before>import subprocess
import signal
import json
import vim
import re
import os
import sys
from itertools import groupby

import vimbufsync
vimbufsync.check_version("0.1.0",who="merlin")

flags = []
enclosing_types = [] # nothing to see here
current_enclosing = -1
atom_bound = re.compile('[a-z_0-9A-Z\'`.]')

######## ERROR MANAGEMENT

class MerlinExc(Exception):
  def __init__(self, value):
      self.value = value
  def __str__(self):
    return repr(self.value)

class Failure(MerlinExc):
  pass

class Error(MerlinExc):
  pass

class MerlinException(MerlinExc):
  pass

def try_print_error(e, msg=None):
  try:
    raise e
  except Error as e:
    if msg: print(msg)
    else:
      print(e.value['message'])
  except Exception as e:
    if msg: sys.stderr.write(msg)
    else:
      msg = str(e)
      if re.search('Not_found',msg):
        print ("error: Not found")
        return None
      elif re.search('Cmi_format.Error', msg):
        sys.stderr.write ("error: The version of merlin you're using doesn't support this version of ocaml")
        return None
      sys.stderr.write(msg)

def catch_and_print(f, msg=None):
  try:
    return f()
  except MerlinExc as e:
    try_print_error(e, msg=msg)

######## PROCESS MANAGEMENT

class MerlinProcess:
  def __init__(self):
    self.mainpipe = None
    self.saved_sync = None

  def restart(self):
    if self.mainpipe:
      try:
        try:
          self.mainpipe.terminate()
        except OSError:
          pass
        self.mainpipe.communicate()
      except OSError:
        pass
    try:
      cmd = [vim.eval("merlin#FindOcamlMerlin()"),"-ignore-sigint"]
      cmd.extend(flags)
      self.mainpipe = subprocess.Popen(
              cmd,
              stdin=subprocess.PIPE,
              stdout=subprocess.PIPE,
              stderr=None,
          )
    except OSError as e:
      print("Failed starting ocamlmerlin. Please ensure that ocamlmerlin binary\
              is executable.")
      raise e

  def command(self, *cmd):
    if self.mainpipe == None or self.mainpipe.returncode != None:
      self.restart()
    json.dump(cmd, self.mainpipe.stdin)
    line = self.mainpipe.stdout.readline()
    result = json.loads(line)
    content = None
    if len(result) == 2:
      content = result[1]

    if result[0] == "return":
      return content
    elif result[0] == "failure":
      raise Failure(content)
    elif result[0] == "error":
      raise Error(content)
    elif result[0] == "exception":
      raise MerlinException(content)

## MULTI-PROCESS
#merlin_processes = {}
#def merlin_process():
#  global merlin_processes
#  name = vim.eval("exists('b:merlin_project') ? b:merlin_project : ''")
#  if not name in merlin_processes:
#    merlin_processes[name] = MerlinProcess()
#  return merlin_processes[name]

# MONO-PROCESS
merlin_processes = None
def merlin_process():
  global merlin_processes
  if not merlin_processes:
    merlin_processes = MerlinProcess()
  return merlin_processes

def command(*cmd):
  return merlin_process().command(*cmd)

def dump(*cmd):
  print(command('dump', *cmd))

######## PATH MANIPULATION

def path_is_recursive(path):
  for component in path:
    if isinstance(component,list):
      for x in component:
        if x == "rec":
          return True
    elif component == "rec":
      return True
  return False

def path_common(p1,p2):
  l = min(len(p1),len(p2))
  p = []
  for i in range(l):
    if p1[i] == p2[i]:
      p.append(p1[i])
    else:
      c1, c2 = p1[i], p2[i]
      l = min(len(c1),len(c2))
      c = []
      for i in range(l):
        if c1[i] == c2[i]:
          c.append(c1[i])
      p.append(c)
      return p
  return p

######## BASIC COMMANDS

def parse_position(pos):
  position = pos['pos']
  path = pos['path']
  return (position['line'], position['col'], path)

def display_load_failures(result):
  if 'failures' in result:
    print (result['failures'])
  return result['result']

def command_tell(content):
  if isinstance(content,list):
    content = "\n".join(content) + "\n"
  return parse_position(command("tell", "source", content))

def command_find_use(*packages):
  result = catch_and_print(lambda: command('find', 'use', packages))
  return display_load_failures(result)

def command_reset(kind="ml",name=None):
  global saved_sync
  if name: r = command("reset",kind,name)
  else:    r = command("reset",kind)
  if name == "myocamlbuild.ml":
    command_find_use("ocamlbuild")
  saved_sync = None
  return r

def command_seek(mtd,line,col):
  return parse_position(command("seek", mtd, {'line' : line, 'col': col}))

def command_complete_cursor(base,line,col):
  return command("complete", "prefix", base, "at", {'line' : line, 'col': col})

def command_locate(path, line, col):
  try:
    if line is None or col is None:
        return command("locate", path)
    else:
        pos_or_err = command("locate", path, "at", {'line': line, 'col': col})
    if not isinstance(pos_or_err, dict):
      print(pos_or_err)
    else:
      l = pos_or_err['pos']['line']
      c = pos_or_err['pos']['col']
      if "file" in pos_or_err:
        vim.command(":split %s" % pos_or_err['file'])
      vim.current.window.cursor = (l, c)
  except MerlinExc as e:
    try_print_error(e)

######## BUFFER SYNCHRONIZATION

def sync_buffer_to(to_line, to_col, load_project=True):
  process = merlin_process()
  saved_sync = process.saved_sync
  curr_sync = vimbufsync.sync()
  cb = vim.current.buffer
  max_line = len(cb)
  end_line = min(to_line, max_line)

  if saved_sync and curr_sync.bufnr() == saved_sync.bufnr():
    line, col = saved_sync.pos()
    command_seek("before", line, 0)
  else:
    if load_project:
      project = vim.eval("exists('b:dotmerlin') && len(b:dotmerlin) > 0 ? b:dotmerlin[0] : ''")
      command("project","load",project)
    command_reset(
            kind=(vim.eval("expand('%:e')") == "mli") and "mli" or "ml",
            name=vim.eval("expand('%:p')")
            )
  line, col, path = parse_position(command("tell", "start"))

  if line <= end_line:
    rest    = cb[line-1][col:]
    content = cb[line:end_line]
    content.insert(0, rest)
    process.saved_sync = curr_sync
  else:
    content = None

  # Send content
  if content:
    kind = "source"
    _, _, path = command_tell(content)
    while path_is_recursive(path):
      if end_line < max_line:
        next_end = min(max_line,end_line + 50)
        _, _, p2 = command_tell(cb[end_line:next_end])
        path = path_common(path,p2)
        end_line = next_end
      else:
        break

def sync_buffer():
  to_line, to_col = vim.current.window.cursor
  sync_buffer_to(to_line, to_col)

def sync_full_buffer():
  sync_buffer_to(len(vim.current.buffer),0)
  command_tell("") # Signal EOF

######## VIM FRONTEND
<commit_msg>vim: finish at least next item when telling<commit_after>import subprocess
import signal
import json
import vim
import re
import os
import sys
from itertools import groupby

import vimbufsync
vimbufsync.check_version("0.1.0",who="merlin")

flags = []
enclosing_types = [] # nothing to see here
current_enclosing = -1
atom_bound = re.compile('[a-z_0-9A-Z\'`.]')

######## ERROR MANAGEMENT

class MerlinExc(Exception):
  def __init__(self, value):
      self.value = value
  def __str__(self):
    return repr(self.value)

class Failure(MerlinExc):
  pass

class Error(MerlinExc):
  pass

class MerlinException(MerlinExc):
  pass

def try_print_error(e, msg=None):
  try:
    raise e
  except Error as e:
    if msg: print(msg)
    else:
      print(e.value['message'])
  except Exception as e:
    if msg: sys.stderr.write(msg)
    else:
      msg = str(e)
      if re.search('Not_found',msg):
        print ("error: Not found")
        return None
      elif re.search('Cmi_format.Error', msg):
        sys.stderr.write ("error: The version of merlin you're using doesn't support this version of ocaml")
        return None
      sys.stderr.write(msg)

def catch_and_print(f, msg=None):
  try:
    return f()
  except MerlinExc as e:
    try_print_error(e, msg=msg)

######## PROCESS MANAGEMENT

class MerlinProcess:
  def __init__(self):
    self.mainpipe = None
    self.saved_sync = None

  def restart(self):
    if self.mainpipe:
      try:
        try:
          self.mainpipe.terminate()
        except OSError:
          pass
        self.mainpipe.communicate()
      except OSError:
        pass
    try:
      cmd = [vim.eval("merlin#FindOcamlMerlin()"),"-ignore-sigint"]
      cmd.extend(flags)
      self.mainpipe = subprocess.Popen(
              cmd,
              stdin=subprocess.PIPE,
              stdout=subprocess.PIPE,
              stderr=None,
          )
    except OSError as e:
      print("Failed starting ocamlmerlin. Please ensure that ocamlmerlin binary\
              is executable.")
      raise e

  def command(self, *cmd):
    if self.mainpipe == None or self.mainpipe.returncode != None:
      self.restart()
    json.dump(cmd, self.mainpipe.stdin)
    line = self.mainpipe.stdout.readline()
    result = json.loads(line)
    content = None
    if len(result) == 2:
      content = result[1]

    if result[0] == "return":
      return content
    elif result[0] == "failure":
      raise Failure(content)
    elif result[0] == "error":
      raise Error(content)
    elif result[0] == "exception":
      raise MerlinException(content)

## MULTI-PROCESS
#merlin_processes = {}
#def merlin_process():
#  global merlin_processes
#  name = vim.eval("exists('b:merlin_project') ? b:merlin_project : ''")
#  if not name in merlin_processes:
#    merlin_processes[name] = MerlinProcess()
#  return merlin_processes[name]

# MONO-PROCESS
merlin_processes = None
def merlin_process():
  global merlin_processes
  if not merlin_processes:
    merlin_processes = MerlinProcess()
  return merlin_processes

def command(*cmd):
  return merlin_process().command(*cmd)

def dump(*cmd):
  print(command('dump', *cmd))

######## PATH MANIPULATION

def path_last_structure_or_sig(path):
  si = -1
  for i in range(len(path)):
    c = path[i]
    if len(c) > 0 and c[0] in ["struct","sig"]:
      si = i
  if si == -1:
      return path
  else:
      return path[:si+1]

def path_is_after(path, ref=[]):
  l = min(len(path),len(ref))
  for i in range(l):
    if path[i] != ref[i]:
      p, r = path[i], ref[i]
      l = min(len(p),len(r))
      for i in range(l):
        if p[i] != r[i] and isinstance(p[i],int) and isinstance(r[i],int):
          return p[i] > r[i]
        else:
          return True
      return True
  return False

def path_is_recursive(path):
  for component in path:
    if isinstance(component,list):
      for x in component:
        if x == "rec":
          return True
    elif component == "rec":
      return True
  return False

def path_common(p1,p2):
  l = min(len(p1),len(p2))
  p = []
  for i in range(l):
    if p1[i] == p2[i]:
      p.append(p1[i])
    else:
      c1, c2 = p1[i], p2[i]
      l = min(len(c1),len(c2))
      c = []
      for i in range(l):
        if c1[i] == c2[i]:
          c.append(c1[i])
      p.append(c)
      return p
  return p

######## BASIC COMMANDS

def parse_position(pos):
  position = pos['pos']
  path = pos['path']
  return (position['line'], position['col'], path)

def display_load_failures(result):
  if 'failures' in result:
    print (result['failures'])
  return result['result']

def command_tell(content):
  if isinstance(content,list):
    content = "\n".join(content) + "\n"
  return parse_position(command("tell", "source", content))

def command_find_use(*packages):
  result = catch_and_print(lambda: command('find', 'use', packages))
  return display_load_failures(result)

def command_reset(kind="ml",name=None):
  global saved_sync
  if name: r = command("reset",kind,name)
  else:    r = command("reset",kind)
  if name == "myocamlbuild.ml":
    command_find_use("ocamlbuild")
  saved_sync = None
  return r

def command_seek(mtd,line,col):
  return parse_position(command("seek", mtd, {'line' : line, 'col': col}))

def command_complete_cursor(base,line,col):
  return command("complete", "prefix", base, "at", {'line' : line, 'col': col})

def command_locate(path, line, col):
  try:
    if line is None or col is None:
        return command("locate", path)
    else:
        pos_or_err = command("locate", path, "at", {'line': line, 'col': col})
    if not isinstance(pos_or_err, dict):
      print(pos_or_err)
    else:
      l = pos_or_err['pos']['line']
      c = pos_or_err['pos']['col']
      if "file" in pos_or_err:
        vim.command(":split %s" % pos_or_err['file'])
      vim.current.window.cursor = (l, c)
  except MerlinExc as e:
    try_print_error(e)

######## BUFFER SYNCHRONIZATION

def sync_buffer_to(to_line, to_col, load_project=True):
  process = merlin_process()
  saved_sync = process.saved_sync
  curr_sync = vimbufsync.sync()
  cb = vim.current.buffer
  max_line = len(cb)
  end_line = min(to_line, max_line)

  if saved_sync and curr_sync.bufnr() == saved_sync.bufnr():
    line, col = saved_sync.pos()
    command_seek("before", line, 0)
  else:
    if load_project:
      project = vim.eval("exists('b:dotmerlin') && len(b:dotmerlin) > 0 ? b:dotmerlin[0] : ''")
      command("project","load",project)
    command_reset(
            kind=(vim.eval("expand('%:e')") == "mli") and "mli" or "ml",
            name=vim.eval("expand('%:p')")
            )
  line, col, path = parse_position(command("tell", "start"))

  if line <= end_line:
    rest    = cb[line-1][col:]
    content = cb[line:end_line]
    content.insert(0, rest)
    process.saved_sync = curr_sync
  else:
    content = None

  # Send content
  if content:
    kind = "source"
    _, _, path = command_tell(content)
    root = path_last_structure_or_sig(path)
    while path_is_recursive(path) or not path_is_after(path,ref=root):
      if end_line < max_line:
        next_end = min(max_line,end_line + 50)
        _, _, p2 = command_tell(cb[end_line:next_end])
        path = path_common(path,p2)
        end_line = next_end
      else:
        break

def sync_buffer():
  to_line, to_col = vim.current.window.cursor
  sync_buffer_to(to_line, to_col)

def sync_full_buffer():
  sync_buffer_to(len(vim.current.buffer),0)
  command_tell("") # Signal EOF

######## VIM FRONTEND

{'fraction_non_alphanumeric': 0.14023278642546627, 'fraction_numerical': 0.005679427850231384, 'mean_word_length': 3.1258316459357824, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4713684', 'n_tokens_mistral': 5069, 'n_tokens_neox': 4982, 'n_words': 1172}