<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gclient-specific SCM-specific operations."""

from __future__ import print_function

import logging
import os
import posixpath
import re
import shlex
import sys
import tempfile
import traceback
import urlparse

import download_from_google_storage
import gclient_utils
import git_cache
import scm
import subprocess2


THIS_FILE_PATH = os.path.abspath(__file__)

GSUTIL_DEFAULT_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    'third_party', 'gsutil', 'gsutil')

CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
class DiffFiltererWrapper(object):
  """Simple base class which tracks which file is being diffed and
  replaces instances of its file name in the original and
  working copy lines of the svn/git diff output."""
  index_string = None
  original_prefix = "--- "
  working_prefix = "+++ "

  def __init__(self, relpath, print_func):
    # Note that we always use '/' as the path separator to be
    # consistent with svn's cygwin-style output on Windows
    self._relpath = relpath.replace("\\", "/")
    self._current_file = None
    self._print_func = print_func

  def SetCurrentFile(self, current_file):
    self._current_file = current_file

  @property
  def _replacement_file(self):
    return posixpath.join(self._relpath, self._current_file)

  def _Replace(self, line):
    return line.replace(self._current_file, self._replacement_file)

  def Filter(self, line):
    if (line.startswith(self.index_string)):
      self.SetCurrentFile(line[len(self.index_string):])
      line = self._Replace(line)
    else:
      if (line.startswith(self.original_prefix) or
          line.startswith(self.working_prefix)):
        line = self._Replace(line)
    self._print_func(line)


class SvnDiffFilterer(DiffFiltererWrapper):
  index_string = "Index: "


class GitDiffFilterer(DiffFiltererWrapper):
  index_string = "diff --git "

  def SetCurrentFile(self, current_file):
    # Get filename by parsing "a/<filename> b/<filename>"
    self._current_file = current_file[:(len(current_file)/2)][2:]

  def _Replace(self, line):
    return re.sub("[a|b]/" + self._current_file, self._replacement_file, line)


### SCM abstraction layer

# Factory Method for SCM wrapper creation

def GetScmName(url):
  if url:
    url, _ = gclient_utils.SplitUrlRevision(url)
    if (url.startswith('git://') or url.startswith('ssh://') or
        url.startswith('git+http://') or url.startswith('git+https://') or
        url.endswith('.git') or url.startswith('sso://')):
      return 'git'
    elif (url.startswith('http://') or url.startswith('https://') or
          url.startswith('svn://') or url.startswith('svn+ssh://')):
      return 'svn'
  return None


def CreateSCM(url, root_dir=None, relpath=None, out_fh=None, out_cb=None):
  SCM_MAP = {
    'svn' : SVNWrapper,
    'git' : GitWrapper,
  }

  scm_name = GetScmName(url)
  if not scm_name in SCM_MAP:
    raise gclient_utils.Error('No SCM found for url %s' % url)
  scm_class = SCM_MAP[scm_name]
  if not scm_class.BinaryExists():
    raise gclient_utils.Error('%s command not found' % scm_name)
  return scm_class(url, root_dir, relpath, out_fh, out_cb)


# SCMWrapper base class

class SCMWrapper(object):
  """Add necessary glue between all the supported SCM.

  This is the abstraction layer to bind to different SCM.
  """

  def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
               out_cb=None):
    self.url = url
    self._root_dir = root_dir
    if self._root_dir:
      self._root_dir = self._root_dir.replace('/', os.sep)
    self.relpath = relpath
    if self.relpath:
      self.relpath = self.relpath.replace('/', os.sep)
    if self.relpath and self._root_dir:
      self.checkout_path = os.path.join(self._root_dir, self.relpath)
    if out_fh is None:
      out_fh = sys.stdout
    self.out_fh = out_fh
    self.out_cb = out_cb

  def Print(self, *args, **kwargs):
    kwargs.setdefault('file', self.out_fh)
    if kwargs.pop('timestamp', True):
      self.out_fh.write('[%s] ' % gclient_utils.Elapsed())
    print(*args, **kwargs)

  def RunCommand(self, command, options, args, file_list=None):
    commands = ['cleanup', 'update', 'updatesingle', 'revert',
                'revinfo', 'status', 'diff', 'pack', 'runhooks']

    if not command in commands:
      raise gclient_utils.Error('Unknown command %s' % command)

    if not command in dir(self):
      raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
          command, self.__class__.__name__))

    return getattr(self, command)(options, args, file_list)

  def GetActualRemoteURL(self, options):
    """Attempt to determine the remote URL for this SCMWrapper."""
    # Git
    if os.path.exists(os.path.join(self.checkout_path, '.git')):
      actual_remote_url = shlex.split(self._Capture(
          ['config', '--local', '--get-regexp', r'remote.*.url'],
          cwd=self.checkout_path))[1]

      # If a cache_dir is used, obtain the actual remote URL from the cache.
      if getattr(self, 'cache_dir', None):
        mirror = git_cache.Mirror(self.url)
        if (mirror.exists() and mirror.mirror_path.replace('\\', '/') ==
            actual_remote_url.replace('\\', '/')):
          actual_remote_url = shlex.split(self._Capture(
              ['config', '--local', '--get-regexp', r'remote.*.url'],
              cwd=mirror.mirror_path))[1]
      return actual_remote_url

    # Svn
    if os.path.exists(os.path.join(self.checkout_path, '.svn')):
      return scm.SVN.CaptureLocalInfo([], self.checkout_path)['URL']
    return None
<commit_msg>SplitUrlRevision when obtaining the cache_dir in gclient_scm.GetActualRemoteURL<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gclient-specific SCM-specific operations."""

from __future__ import print_function

import logging
import os
import posixpath
import re
import shlex
import sys
import tempfile
import traceback
import urlparse

import download_from_google_storage
import gclient_utils
import git_cache
import scm
import subprocess2


THIS_FILE_PATH = os.path.abspath(__file__)

GSUTIL_DEFAULT_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    'third_party', 'gsutil', 'gsutil')

CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
class DiffFiltererWrapper(object):
  """Simple base class which tracks which file is being diffed and
  replaces instances of its file name in the original and
  working copy lines of the svn/git diff output."""
  index_string = None
  original_prefix = "--- "
  working_prefix = "+++ "

  def __init__(self, relpath, print_func):
    # Note that we always use '/' as the path separator to be
    # consistent with svn's cygwin-style output on Windows
    self._relpath = relpath.replace("\\", "/")
    self._current_file = None
    self._print_func = print_func

  def SetCurrentFile(self, current_file):
    self._current_file = current_file

  @property
  def _replacement_file(self):
    return posixpath.join(self._relpath, self._current_file)

  def _Replace(self, line):
    return line.replace(self._current_file, self._replacement_file)

  def Filter(self, line):
    if (line.startswith(self.index_string)):
      self.SetCurrentFile(line[len(self.index_string):])
      line = self._Replace(line)
    else:
      if (line.startswith(self.original_prefix) or
          line.startswith(self.working_prefix)):
        line = self._Replace(line)
    self._print_func(line)


class SvnDiffFilterer(DiffFiltererWrapper):
  index_string = "Index: "


class GitDiffFilterer(DiffFiltererWrapper):
  index_string = "diff --git "

  def SetCurrentFile(self, current_file):
    # Get filename by parsing "a/<filename> b/<filename>"
    self._current_file = current_file[:(len(current_file)/2)][2:]

  def _Replace(self, line):
    return re.sub("[a|b]/" + self._current_file, self._replacement_file, line)


### SCM abstraction layer

# Factory Method for SCM wrapper creation

def GetScmName(url):
  if url:
    url, _ = gclient_utils.SplitUrlRevision(url)
    if (url.startswith('git://') or url.startswith('ssh://') or
        url.startswith('git+http://') or url.startswith('git+https://') or
        url.endswith('.git') or url.startswith('sso://')):
      return 'git'
    elif (url.startswith('http://') or url.startswith('https://') or
          url.startswith('svn://') or url.startswith('svn+ssh://')):
      return 'svn'
  return None


def CreateSCM(url, root_dir=None, relpath=None, out_fh=None, out_cb=None):
  SCM_MAP = {
    'svn' : SVNWrapper,
    'git' : GitWrapper,
  }

  scm_name = GetScmName(url)
  if not scm_name in SCM_MAP:
    raise gclient_utils.Error('No SCM found for url %s' % url)
  scm_class = SCM_MAP[scm_name]
  if not scm_class.BinaryExists():
    raise gclient_utils.Error('%s command not found' % scm_name)
  return scm_class(url, root_dir, relpath, out_fh, out_cb)


# SCMWrapper base class

class SCMWrapper(object):
  """Add necessary glue between all the supported SCM.

  This is the abstraction layer to bind to different SCM.
  """

  def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
               out_cb=None):
    self.url = url
    self._root_dir = root_dir
    if self._root_dir:
      self._root_dir = self._root_dir.replace('/', os.sep)
    self.relpath = relpath
    if self.relpath:
      self.relpath = self.relpath.replace('/', os.sep)
    if self.relpath and self._root_dir:
      self.checkout_path = os.path.join(self._root_dir, self.relpath)
    if out_fh is None:
      out_fh = sys.stdout
    self.out_fh = out_fh
    self.out_cb = out_cb

  def Print(self, *args, **kwargs):
    kwargs.setdefault('file', self.out_fh)
    if kwargs.pop('timestamp', True):
      self.out_fh.write('[%s] ' % gclient_utils.Elapsed())
    print(*args, **kwargs)

  def RunCommand(self, command, options, args, file_list=None):
    commands = ['cleanup', 'update', 'updatesingle', 'revert',
                'revinfo', 'status', 'diff', 'pack', 'runhooks']

    if not command in commands:
      raise gclient_utils.Error('Unknown command %s' % command)

    if not command in dir(self):
      raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
          command, self.__class__.__name__))

    return getattr(self, command)(options, args, file_list)

  def GetActualRemoteURL(self, options):
    """Attempt to determine the remote URL for this SCMWrapper."""
    # Git
    if os.path.exists(os.path.join(self.checkout_path, '.git')):
      actual_remote_url = shlex.split(self._Capture(
          ['config', '--local', '--get-regexp', r'remote.*.url'],
          cwd=self.checkout_path))[1]

      # If a cache_dir is used, obtain the actual remote URL from the cache.
      if getattr(self, 'cache_dir', None):
        url, _ = gclient_utils.SplitUrlRevision(self.url)
        mirror = git_cache.Mirror(url)
        if (mirror.exists() and mirror.mirror_path.replace('\\', '/') ==
            actual_remote_url.replace('\\', '/')):
          actual_remote_url = shlex.split(self._Capture(
              ['config', '--local', '--get-regexp', r'remote.*.url'],
              cwd=mirror.mirror_path))[1]
      return actual_remote_url

    # Svn
    if os.path.exists(os.path.join(self.checkout_path, '.svn')):
      return scm.SVN.CaptureLocalInfo([], self.checkout_path)['URL']
    return None

{'fraction_non_alphanumeric': 0.12845765179412522, 'fraction_numerical': 0.0015414918215295025, 'mean_word_length': 3.5386708122813837, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5646180', 'n_tokens_mistral': 3958, 'n_tokens_neox': 3861, 'n_words': 946}