<commit_before>from threading import Thread
import glob
import os
import shutil

import pecan
import git
import yaml
import datetime

from joulupukki.common.logger import get_logger, get_logger_path
from joulupukki.common.carrier import Carrier
from joulupukki.common.datamodel.job import Job

from docker import Client

import time

import urlparse

"""
scheduled
clonning
dispatching
finished
"""


class Dispatcher(Thread):
    def __init__(self, build):
        thread_name = "__".join((build.user.username,
                                 build.project.name,
                                 str(build.id_)))
        Thread.__init__(self, name=thread_name)
        self.source_url = build.source_url
        self.source_type = build.source_type
        self.branch = build.branch
        self.commit = build.commit
        self.user = build.user
        self.project = build.project
        self.id_ = str(build.id_)
        self.uuid2 = thread_name
        self.created = time.time()
        self.build = build

        # Create docker client
        self.cli = Client(base_url='unix://var/run/docker.sock', version=pecan.conf.docker_version)
        # Set folders
        self.folder = build.get_folder_path()
        
        self.folder_source = build.get_source_folder_path()
        # Create folders
        if not os.path.exists(self.folder):
            os.makedirs(self.folder)
        if not os.path.isdir(self.folder):
            # TODO handle error
            raise Exception("%s should be a folder" % self.folder)
        # Prepare logger
        self.logger = get_logger(self.build)

    def git_clone(self):
        self.logger.info("Cloning")
        # Reworking url to handle password
        url_splitted = urlparse.urlsplit(self.source_url)
        username = url_splitted.username
        if username is None:
            username = 'anonymous'
        password = url_splitted.password
        if password is None:
            password = '<PASSWORD>'
        url_data = [d for d in list(url_splitted)]
        new_netloc = username + ":" + password + "@" + url_data[1].split('@')[-1]
        url_data[1] = new_netloc
        source_url = urlparse.urlunsplit(url_data)
        # Clone repo
        try:
            repo = git.Repo.clone_from(source_url, self.folder_source)
        except Exception as exp:
            self.logger.error("Clonning error: %s", exp)
            return False
        branch_found = True
        # Get branch/tag if set
        if self.branch is not None:
            branch_found = False
            for ref in repo.refs:
                if ref.name == self.branch:
                    repo.head.reference = repo.commit(ref)
                    branch_found = True
                if ref.name == "origin/" + self.branch:
                    repo.head.reference = repo.commit(ref)
                    branch_found = True
        if branch_found is False:
            self.logger.error("Branch %s not found", self.branch)
            return False
        # Get commit if set<commit_msg>Better git url handling<commit_after>from threading import Thread
import glob
import os
import shutil

import pecan
import git
import yaml
import datetime

from joulupukki.common.logger import get_logger, get_logger_path
from joulupukki.common.carrier import Carrier
from joulupukki.common.datamodel.job import Job

from docker import Client

import time

import urlparse

"""
scheduled
clonning
dispatching
finished
"""


class Dispatcher(Thread):
    def __init__(self, build):
        thread_name = "__".join((build.user.username,
                                 build.project.name,
                                 str(build.id_)))
        Thread.__init__(self, name=thread_name)
        self.source_url = build.source_url
        self.source_type = build.source_type
        self.branch = build.branch
        self.commit = build.commit
        self.user = build.user
        self.project = build.project
        self.id_ = str(build.id_)
        self.uuid2 = thread_name
        self.created = time.time()
        self.build = build

        # Create docker client
        self.cli = Client(base_url='unix://var/run/docker.sock', version=pecan.conf.docker_version)
        # Set folders
        self.folder = build.get_folder_path()
        
        self.folder_source = build.get_source_folder_path()
        # Create folders
        if not os.path.exists(self.folder):
            os.makedirs(self.folder)
        if not os.path.isdir(self.folder):
            # TODO handle error
            raise Exception("%s should be a folder" % self.folder)
        # Prepare logger
        self.logger = get_logger(self.build)

    def git_clone(self):
        self.logger.info("Cloning")
        # Reworking url to handle password
        url_splitted = urlparse.urlsplit(self.source_url)
        if url_splitted.scheme in ['http', 'https']:
            username = url_splitted.username
            if username is None:
                username = 'anonymous'
            password = url_splitted.password
            if password is None:
                password = '<PASSWORD>'
            url_data = [d for d in list(url_splitted)]
            new_netloc = username + ":" + password + "@" + url_data[1].split('@')[-1]
            url_data[1] = new_netloc
            source_url = urlparse.urlunsplit(url_data)
        else:
            source_url = self.source_url
        # Clone repo
        try:
            repo = git.Repo.clone_from(source_url, self.folder_source)
        except Exception as exp:
            self.logger.error("Clonning error: %s", exp)
            return False
        branch_found = True
        # Get branch/tag if set
        if self.branch is not None:
            branch_found = False
            for ref in repo.refs:
                if ref.name == self.branch:
                    repo.head.reference = repo.commit(ref)
                    branch_found = True
                if ref.name == "origin/" + self.branch:
                    repo.head.reference = repo.commit(ref)
                    branch_found = True
        if branch_found is False:
            self.logger.error("Branch %s not found", self.branch)
            return False
        # Get commit if set
{'fraction_non_alphanumeric': 0.0875, 'fraction_numerical': 0.001282051282051282, 'mean_word_length': 1.9231850117096019, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4221156', 'n_tokens_mistral': 1769, 'n_tokens_neox': 1731, 'n_words': 434}