<commit_before>import os
import requests
import StringIO
import subprocess
import zipfile

from solar import utils


class BaseProvider(object):
    def run(self):
        pass


class DirectoryProvider(BaseProvider):
    def __init__(self, directory):
        self.directory = directory


class GitProvider(BaseProvider):
    def __init__(self, repository, path='.'):
        self.repository = repository
        self.path = path

        repo_name = os.path.split(self.repository)[1]

        resources_directory = os.path.join(
            utils.read_config()['resources-directory'],
            repo_name
        )

        with open('/tmp/git-provider.yaml', 'w') as f:
            f.write("""
---

- hosts: all
  tasks:
    - git: repo={repository} dest={destination} clone={clone} update=yes
            """.format(
                repository=self.repository,
                destination=resources_directory,
                clone='no' if os.path.exists(resources_directory) else 'yes'
            ))

        subprocess.check_call([
            'ansible-playbook',
            '-i', '"localhost,"',
            '-c', 'local',
            '/tmp/git-provider.yaml'
        ])

        if path != '.':
            self.directory = os.path.join(resources_directory, path)
        else:
            self.directory = resources_directory


class RemoteZipProvider(BaseProvider):
    """Download & extract zip from some URL.

    Assumes zip structure of the form:
    <group-name>
      <resource1>
      <resource2>
      ...
    """

    def __init__(self, url, path='.'):
        self.url = url
        self.path = path

        r = requests.get(url)
        s = StringIO.StringIO(r.content)
        z = zipfile.ZipFile(s)

        group_name = os.path.dirname(z.namelist()[0])
        base_resources_directory = utils.read_config()['resources-directory']
        resources_directory = os.path.join(
            base_resources_directory, group_name
        )
        if not os.path.exists(resources_directory):
            z.extractall(base_resources_directory)

        if path != '.':
            self.directory = os.path.join(resources_directory, path)<commit_msg>GitProvider: do not clone when directory is present<commit_after>import os
import requests
import StringIO
import subprocess
import zipfile

from solar import utils


class BaseProvider(object):
    def run(self):
        pass


class DirectoryProvider(BaseProvider):
    def __init__(self, directory):
        self.directory = directory


class GitProvider(BaseProvider):
    def __init__(self, repository, path='.'):
        self.repository = repository
        self.path = path

        resources_directory = self._resources_directory()

        if not os.path.exists(resources_directory):
            self._clone_repo()

        if path != '.':
            self.directory = os.path.join(resources_directory, path)
        else:
            self.directory = resources_directory

    def _resources_directory(self):
        repo_name = os.path.split(self.repository)[1]

        return os.path.join(
            utils.read_config()['resources-directory'],
            repo_name
        )

    def _clone_repo(self):
        resources_directory = self._resources_directory()

        with open('/tmp/git-provider.yaml', 'w') as f:
            f.write("""
---

- hosts: all
  tasks:
    - git: repo={repository} dest={destination} clone={clone} update=yes
            """.format(
                repository=self.repository,
                destination=resources_directory,
                clone='yes'
            ))

        subprocess.check_call([
            'ansible-playbook',
            '-i', '"localhost,"',
            '-c', 'local',
            '/tmp/git-provider.yaml'
        ])


class RemoteZipProvider(BaseProvider):
    """Download & extract zip from some URL.

    Assumes zip structure of the form:
    <group-name>
      <resource1>
      <resource2>
      ...
    """

    def __init__(self, url, path='.'):
        self.url = url
        self.path = path

        r = requests.get(url)
        s = StringIO.StringIO(r.content)
        z = zipfile.ZipFile(s)

        group_name = os.path.dirname(z.namelist()[0])
        base_resources_directory = utils.read_config()['resources-directory']
        resources_directory = os.path.join(
            base_resources_directory, group_name
        )
        if not os.path.exists(resources_directory):
            z.extractall(base_resources_directory)

        if path != '.':
            self.directory = os.path.join(resources_directory, path)
{'fraction_non_alphanumeric': 0.11438550534817725, 'fraction_numerical': 0.0017463435931019428, 'mean_word_length': 2.1405071967100753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3853353', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1244, 'n_words': 268}