<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# The GitRepoDriver provides the following:
#
# 1- A getpackages function based on finding all directories inside a specific
#    git repo, where each directory represents a package
#
# 2- A getinfo function based on a single-distgit repo paradigm
#
# 3- A preprocess function using renderspec on any *.spec.j2 file found in the
#    distgit.

import logging
import os
import re
import sh

from dlrn.db import Commit
from dlrn.drivers.pkginfo import PkgInfoDriver
from dlrn.repositories import getsourcebranch
from dlrn.repositories import refreshrepo
from dlrn.utils import run_external_preprocess
from pymod2pkg import module2package
from pymod2pkg import module2upstream
from rdopkg.utils import specfile
from six.moves import urllib
from six.moves.urllib.request import urlopen

logger = logging.getLogger("dlrn-gitrepo-driver")

version_match = re.compile(r'\W*set upstream_version\D+([\w.]+).*')
wrong_match = re.compile(r'\W*set upstream_version\D+\(\).*')

base_urls = ['https://opendev.org/openstack',
             'https://opendev.org/x',
             'https://opendev.org/opendev']


def check_url(url):
    logger.info("Checking url %s" % url)
    try:
        urlopen(url)
        # URL found
        return True
    except (urllib.error.HTTPError, urllib.error.URLError):
        # Trouble finding URL
        return False


class GitRepoDriver(PkgInfoDriver):
    DRIVER_CONFIG = {
        'gitrepo_driver': {
            'gitrepo_repo': {'name': 'repo'},
            'gitrepo_dir': {'name': 'directory'},
            'skip_dirs': {'name': 'skip', 'type': 'list'},
            'use_version_from_spec': {'type': 'boolean'},
            'keep_tarball': {'type': 'boolean'},
        }
    }

    def __init__(self, *args, **kwargs):
        super(GitRepoDriver, self).__init__(*args, **kwargs)

    def _get_version_from_pkg(self, packagepath, package):
        self.preprocess(package_name=package)
        pkgdir = os.path.join(packagepath, package)
        for pkgfile in os.listdir(pkgdir):
            if pkgfile.endswith('.spec'):
                spec = specfile.Spec(fn=os.path.join(packagepath, package,
                                                     pkgfile))
                version = spec.get_tag('Version').encode()
                release = spec.get_tag('Release').encode()

        if release.startswith('0'):
            # This is a pre-release version, so we are following trunk
            return None<commit_msg>[gitrepo] Do not encode version and release in gitrepo driver<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# The GitRepoDriver provides the following:
#
# 1- A getpackages function based on finding all directories inside a specific
#    git repo, where each directory represents a package
#
# 2- A getinfo function based on a single-distgit repo paradigm
#
# 3- A preprocess function using renderspec on any *.spec.j2 file found in the
#    distgit.

import logging
import os
import re
import sh

from dlrn.db import Commit
from dlrn.drivers.pkginfo import PkgInfoDriver
from dlrn.repositories import getsourcebranch
from dlrn.repositories import refreshrepo
from dlrn.utils import run_external_preprocess
from pymod2pkg import module2package
from pymod2pkg import module2upstream
from rdopkg.utils import specfile
from six.moves import urllib
from six.moves.urllib.request import urlopen

logger = logging.getLogger("dlrn-gitrepo-driver")

version_match = re.compile(r'\W*set upstream_version\D+([\w.]+).*')
wrong_match = re.compile(r'\W*set upstream_version\D+\(\).*')

base_urls = ['https://opendev.org/openstack',
             'https://opendev.org/x',
             'https://opendev.org/opendev']


def check_url(url):
    logger.info("Checking url %s" % url)
    try:
        urlopen(url)
        # URL found
        return True
    except (urllib.error.HTTPError, urllib.error.URLError):
        # Trouble finding URL
        return False


class GitRepoDriver(PkgInfoDriver):
    DRIVER_CONFIG = {
        'gitrepo_driver': {
            'gitrepo_repo': {'name': 'repo'},
            'gitrepo_dir': {'name': 'directory'},
            'skip_dirs': {'name': 'skip', 'type': 'list'},
            'use_version_from_spec': {'type': 'boolean'},
            'keep_tarball': {'type': 'boolean'},
        }
    }

    def __init__(self, *args, **kwargs):
        super(GitRepoDriver, self).__init__(*args, **kwargs)

    def _get_version_from_pkg(self, packagepath, package):
        self.preprocess(package_name=package)
        pkgdir = os.path.join(packagepath, package)
        for pkgfile in os.listdir(pkgdir):
            if pkgfile.endswith('.spec'):
                spec = specfile.Spec(fn=os.path.join(packagepath, package,
                                                     pkgfile))
                version = spec.get_tag('Version')
                release = spec.get_tag('Release')

        if release.startswith('0'):
            # This is a pre-release version, so we are following trunk
            return None
{'fraction_non_alphanumeric': 0.10317460317460317, 'fraction_numerical': 0.004298941798941799, 'mean_word_length': 2.979605263157895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2229502', 'n_tokens_mistral': 1784, 'n_tokens_neox': 1726, 'n_words': 552}