<commit_before>from __future__ import unicode_literals

import logging
import os
import re
import shutil
import tempfile

from django.core.exceptions import ValidationError
from django.utils import six
from django.utils.six.moves.urllib.parse import urlparse
from django.utils.translation import ugettext as _
from djblets.util.filesystem import is_exe_in_path

from reviewboard.scmtools.core import SCMTool, HEAD, PRE_CREATION
from reviewboard.scmtools.errors import (AuthenticationError,
                                         SCMError,
                                         FileNotFoundError,
                                         RepositoryNotFoundError)
from reviewboard.diffviewer.parser import DiffParser, DiffParserError
from reviewboard.ssh import utils as sshutils
from reviewboard.ssh.errors import SSHAuthenticationError, SSHError


sshutils.register_rbssh('CVS_RSH')


class CVSTool(SCMTool):
    name = "CVS"
    field_help_text = {
        'path': 'The CVSROOT used to access the repository.',
    }
    dependencies = {
        'executables': ['cvs'],
    }

    rev_re = re.compile(r'^.*?(\d+(\.\d+)+)\r?$')

    remote_cvsroot_re = re.compile(
        r'^(:(?P<protocol>[gkp]?server|ext|ssh|extssh):'
        r'((?P<username>[^:@]+)(:(?P<password>[^@]+))?@)?)?'
        r'(?P<hostname>[^:]+):(?P<port>\d+)?(?P<path>.*)')
    local_cvsroot_re = re.compile(r'^:(?P<protocol>local|fork):(?P<path>.+)')

    def __init__(self, repository):
        super(CVSTool, self).__init__(repository)

        credentials = repository.get_credentials()

        # Note that we're not validating the CVSROOT here, as the path may
        # contain a value that "worked" prior to us introducing validation.
        # In those cases, we'd have either parsed out the invalid data or
        # simply let CVS deal with it. We don't want to break those setups.
        self.cvsroot, self.repopath = \
            self.build_cvsroot(self.repository.path,
                               credentials['username'],
                               credentials['password'],
                               validate=False)

        local_site_name = None

        if repository.local_site:
            local_site_name = repository.local_site.name

        self.client = CVSClient(self.cvsroot, self.repopath, local_site_name)

    def get_file(self, path, revision=HEAD, **kwargs):
        if not path:
            raise FileNotFoundError(path, revision)

        return self.client.cat_file(path, revision)

    def parse_diff_revision(self, file_str, revision_str, *args, **kwargs):
        if revision_str == "PRE-CREATION":
            return file_str, PRE_CREATION

        m = self.rev_re.match(revision_str)
        if m:
            return file_str, m.group(1)
        else:
            # Newer versions of CVS stick the file revision after the filename,
            # separated by a colon. Check for that format too.
            colon_idx = file_str.rfind(":")
            if colon_idx == -1:
                raise SCMError("Unable to parse diff revision header "
                               "(file_str='%s', revision_str='%s')"
                               % (file_str, revision_str))
            return file_str[:colon_idx], file_str[colon_idx + 1:]

    def get_diffs_use_absolute_paths(self):
        return True

    def get_fields(self):
        return ['diff_path']

    def get_parser(self, data):
        return CVSDiffParser(data, self.repopath)

    def normalize_patch(self, patch, filename, revision=HEAD):
        """Normalizes the content of a patch.<commit_msg>Fix syntax highlighting for files on CVS.<commit_after>from __future__ import unicode_literals

import logging
import os
import re
import shutil
import tempfile

from django.core.exceptions import ValidationError
from django.utils import six
from django.utils.six.moves.urllib.parse import urlparse
from django.utils.translation import ugettext as _
from djblets.util.filesystem import is_exe_in_path

from reviewboard.scmtools.core import SCMTool, HEAD, PRE_CREATION
from reviewboard.scmtools.errors import (AuthenticationError,
                                         SCMError,
                                         FileNotFoundError,
                                         RepositoryNotFoundError)
from reviewboard.diffviewer.parser import DiffParser, DiffParserError
from reviewboard.ssh import utils as sshutils
from reviewboard.ssh.errors import SSHAuthenticationError, SSHError


sshutils.register_rbssh('CVS_RSH')


class CVSTool(SCMTool):
    name = "CVS"
    field_help_text = {
        'path': 'The CVSROOT used to access the repository.',
    }
    dependencies = {
        'executables': ['cvs'],
    }

    rev_re = re.compile(r'^.*?(\d+(\.\d+)+)\r?$')

    remote_cvsroot_re = re.compile(
        r'^(:(?P<protocol>[gkp]?server|ext|ssh|extssh):'
        r'((?P<username>[^:@]+)(:(?P<password>[^@]+))?@)?)?'
        r'(?P<hostname>[^:]+):(?P<port>\d+)?(?P<path>.*)')
    local_cvsroot_re = re.compile(r'^:(?P<protocol>local|fork):(?P<path>.+)')

    def __init__(self, repository):
        super(CVSTool, self).__init__(repository)

        credentials = repository.get_credentials()

        # Note that we're not validating the CVSROOT here, as the path may
        # contain a value that "worked" prior to us introducing validation.
        # In those cases, we'd have either parsed out the invalid data or
        # simply let CVS deal with it. We don't want to break those setups.
        self.cvsroot, self.repopath = \
            self.build_cvsroot(self.repository.path,
                               credentials['username'],
                               credentials['password'],
                               validate=False)

        local_site_name = None

        if repository.local_site:
            local_site_name = repository.local_site.name

        self.client = CVSClient(self.cvsroot, self.repopath, local_site_name)

    def get_file(self, path, revision=HEAD, **kwargs):
        if not path:
            raise FileNotFoundError(path, revision)

        return self.client.cat_file(path, revision)

    def parse_diff_revision(self, file_str, revision_str, *args, **kwargs):
        if revision_str == "PRE-CREATION":
            return file_str, PRE_CREATION

        m = self.rev_re.match(revision_str)
        if m:
            return file_str, m.group(1)
        else:
            # Newer versions of CVS stick the file revision after the filename,
            # separated by a colon. Check for that format too.
            colon_idx = file_str.rfind(":")
            if colon_idx == -1:
                raise SCMError("Unable to parse diff revision header "
                               "(file_str='%s', revision_str='%s')"
                               % (file_str, revision_str))
            return file_str[:colon_idx], file_str[colon_idx + 1:]

    def get_diffs_use_absolute_paths(self):
        return True

    def get_fields(self):
        return ['diff_path']

    def get_parser(self, data):
        return CVSDiffParser(data, self.repopath)

    def normalize_path_for_display(self, filename):
        """Normalize a path from a diff for display to the user.

        This can take a path/filename found in a diff and normalize it,
        stripping away unwanted information, so that it displays in a better
        way in the diff viewer.

        For CVS, this strips trailing ",v" from filenames.

        Args:
            filename (unicode):
                The filename/path to normalize.

        Returns:
            unicode:
            The resulting filename/path.
        """
        return re.sub(',v$', '', filename)

    def normalize_patch(self, patch, filename, revision=HEAD):
        """Normalizes the content of a patch.
{'fraction_non_alphanumeric': 0.10218228498074454, 'fraction_numerical': 0.0007702182284980744, 'mean_word_length': 2.365442764578834, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6245713', 'n_tokens_mistral': 2306, 'n_tokens_neox': 2188, 'n_words': 588}