<commit_before># Copyright 2013 Clemson University
#
# This file is part of python-anyvcs.
#
# python-anyvcs is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# python-anyvcs is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with python-anyvcs.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import re
import subprocess
from common import *

HG = 'hg'

manifest_rx = re.compile(r'^(?P<mode>[0-7]{3}) (?P<type>.) (?P<name>.+)$')
parse_heads_rx = re.compile(r'^(?P<name>.+?)\s+(?P<rev>-?\d+):(?P<nodeid>[0-9a-f]+)', re.I)
bookmarks_rx = re.compile(r'^\s+(?:\*\s+)?(?P<name>.+?)\s+(?P<rev>\d+):(?P<nodeid>[0-9a-f]+)', re.I)
annotate_rx = re.compile(r'^(?P<author>.*)\s+(?P<rev>\d+):\s')

def parent_dirs(path):
  ds = path.find('/')
  while ds != -1:
    yield path[:ds]
    ds = path.find('/', ds + 1)

def parse_hgdate(datestr):
  ts, tzoffset = datestr.split(None, 1)
  date = datetime.datetime.fromtimestamp(float(ts))
  return date.replace(tzinfo=UTCOffset(-int(tzoffset)/60))

class HgRepo(VCSRepo):
  @classmethod
  def create(cls, path):
    cmd = [HG, 'init', path]
    subprocess.check_call(cmd)
    return cls(path)

  @property
  def private_path(self):
    import os
    path = os.path.join(self.path, '.hg', '.private')
    try:
      os.mkdir(path)
    except OSError as e:
      import errno
      if e.errno != errno.EEXIST:
        raise
    return path

  def _ls(self, rev, path, recursive=False, recursive_dirs=False,
          directory=False):
    forcedir = False
    if path.endswith('/'):
      forcedir = True
      path = path.rstrip('/')
    if path == '':
      ltrim = 0
      prefix = ''
    else:
      ltrim = len(path) + 1
      prefix = path + '/'

    cmd = [HG, 'manifest', '-v', '-r', rev]
    output = self._command(cmd)
    dirs = set()
    exists = False
    for line in output.splitlines():
      m = manifest_rx.match(line)
      assert m, 'unexpected output: ' + line
      t, name = m.group('type', 'name')
      if name.startswith(prefix) or (not forcedir and name == path):
        if directory and name.startswith(prefix):
          yield ('d', '')
          return
        exists = True
        entry_name = name[ltrim:]
        if '/' in entry_name:
          p = parent_dirs(entry_name)
          if not recursive:
            d = p.next()
            if d not in dirs:
              dirs.add(d)
              yield ('d', d)
            continue
          if recursive_dirs:
            for d in p:
              if d not in dirs:
                dirs.add(d)
                yield ('d', d)
        yield (t, entry_name)
    if not exists:
      raise PathDoesNotExist(rev, path)

  def ls(self, rev, path, recursive=False, recursive_dirs=False,
         directory=False, report=()):
    revstr = str(rev)
    path = type(self).cleanPath(path)
    if path == '':
      if directory:
        return [{'type':'d'}]

    results = []
    for t, name in self._ls(revstr, path, recursive, recursive_dirs, directory):
      entry = attrdict()
      if name:
        entry.name = name
      if t == 'd':
        entry.type = 'd'
      elif t in ' *':
        entry.type = 'f'
        if 'executable' in report:
          entry.executable = t == '*'
        if 'size' in report:
          entry.size = len(self._cat(revstr, name))
      elif t == '@':
        entry.type = 'l'
        if 'target' in report:
          entry.target = self._cat(revstr, name)
      else:
        assert False, 'unexpected output: ' + line
      if 'commit' in report:
        p = type(self).cleanPath(path + '/' + name)
        cmd = [HG, 'log', '--template={node}', '-l1', '-r', 'ancestors('+revstr+')', '--', p]
        entry.commit = self._command(cmd)
      results.append(entry)
    return results

  def _cat(self, rev, path):
    cmd = [HG, 'cat', '-r', rev, path]
    return self._command(cmd)

  def cat(self, rev, path):
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].get('type') != 'f':
      raise BadFileType(rev, path)
    return self._cat(str(rev), path)

  def readlink(self, rev, path):
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].get('type') != 'l':<commit_msg>speed up reporting last commit by a bit - can do better<commit_after># Copyright 2013 Clemson University
#
# This file is part of python-anyvcs.
#
# python-anyvcs is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# python-anyvcs is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with python-anyvcs.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import re
import subprocess
from common import *

HG = 'hg'

manifest_rx = re.compile(r'^(?P<mode>[0-7]{3}) (?P<type>.) (?P<name>.+)$')
parse_heads_rx = re.compile(r'^(?P<name>.+?)\s+(?P<rev>-?\d+):(?P<nodeid>[0-9a-f]+)', re.I)
bookmarks_rx = re.compile(r'^\s+(?:\*\s+)?(?P<name>.+?)\s+(?P<rev>\d+):(?P<nodeid>[0-9a-f]+)', re.I)
annotate_rx = re.compile(r'^(?P<author>.*)\s+(?P<rev>\d+):\s')

def parent_dirs(path):
  ds = path.find('/')
  while ds != -1:
    yield path[:ds]
    ds = path.find('/', ds + 1)

def parse_hgdate(datestr):
  ts, tzoffset = datestr.split(None, 1)
  date = datetime.datetime.fromtimestamp(float(ts))
  return date.replace(tzinfo=UTCOffset(-int(tzoffset)/60))

class HgRepo(VCSRepo):
  @classmethod
  def create(cls, path):
    cmd = [HG, 'init', path]
    subprocess.check_call(cmd)
    return cls(path)

  @property
  def private_path(self):
    import os
    path = os.path.join(self.path, '.hg', '.private')
    try:
      os.mkdir(path)
    except OSError as e:
      import errno
      if e.errno != errno.EEXIST:
        raise
    return path

  def _ls(self, rev, path, recursive=False, recursive_dirs=False,
          directory=False):
    forcedir = False
    if path.endswith('/'):
      forcedir = True
      path = path.rstrip('/')
    if path == '':
      ltrim = 0
      prefix = ''
    else:
      ltrim = len(path) + 1
      prefix = path + '/'

    cmd = [HG, 'manifest', '-v', '-r', rev]
    output = self._command(cmd)
    dirs = set()
    exists = False
    for line in output.splitlines():
      m = manifest_rx.match(line)
      assert m, 'unexpected output: ' + line
      t, name = m.group('type', 'name')
      if name.startswith(prefix) or (not forcedir and name == path):
        if directory and name.startswith(prefix):
          yield ('d', '')
          return
        exists = True
        entry_name = name[ltrim:]
        if '/' in entry_name:
          p = parent_dirs(entry_name)
          if not recursive:
            d = p.next()
            if d not in dirs:
              dirs.add(d)
              yield ('d', d)
            continue
          if recursive_dirs:
            for d in p:
              if d not in dirs:
                dirs.add(d)
                yield ('d', d)
        yield (t, entry_name)
    if not exists:
      raise PathDoesNotExist(rev, path)

  def ls(self, rev, path, recursive=False, recursive_dirs=False,
         directory=False, report=()):
    revstr = str(rev)
    path = type(self).cleanPath(path)
    if path == '':
      if directory:
        return [{'type':'d'}]

    if True and 'commit' in report:
      import tempfile
      with tempfile.NamedTemporaryFile() as style:
        style.write(
          r"changeset = '{rev}\n{node}\n{files}\0'" '\n'
          r"file = '{file|escape}\n'" '\n'
        )
        style.flush()
        cmd = [HG, 'log', '--style', style.name, '-r', 'reverse(ancestors('+revstr+'))']
        log = self._command(cmd).split('\0')
    else:
      log = None

    results = []
    for t, name in self._ls(revstr, path, recursive, recursive_dirs, directory):
      entry = attrdict()
      if name:
        entry.name = name
      if t == 'd':
        entry.type = 'd'
      elif t in ' *':
        entry.type = 'f'
        if 'executable' in report:
          entry.executable = t == '*'
        if 'size' in report:
          entry.size = len(self._cat(revstr, name))
      elif t == '@':
        entry.type = 'l'
        if 'target' in report:
          entry.target = self._cat(revstr, name)
      else:
        assert False, 'unexpected output: ' + line
      if 'commit' in report:
        p = type(self).cleanPath(path + '/' + name)
        if log is None:
          cmd = [HG, 'log', '--template={node}', '-l1', '-r', 'reverse(ancestors('+revstr+'))', '--', p]
          entry.commit = self._command(cmd)
        else:
          for logentry in log:
            lines = logentry.splitlines()
            found = False
            for l in lines[2:]:
              if l == p or l.startswith(p+'/'):
                found = True
                break
            if found:
              entry.commit = lines[1]
              break
      results.append(entry)
    return results

  def _cat(self, rev, path):
    cmd = [HG, 'cat', '-r', rev, path]
    return self._command(cmd)

  def cat(self, rev, path):
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].get('type') != 'f':
      raise BadFileType(rev, path)
    return self._cat(str(rev), path)

  def readlink(self, rev, path):
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].get('type') != 'l':
{'fraction_non_alphanumeric': 0.1521464031928356, 'fraction_numerical': 0.005061812518251728, 'mean_word_length': 2.349853276817737, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '594922', 'n_tokens_mistral': 3530, 'n_tokens_neox': 3410, 'n_words': 975}