<commit_before>#!/usr/bin/env python
# coding: utf-8

# Copyright (c) 2002-2016 "Neo Technology,"
# Network Engine for Objects in Lund AB [http://neotechnology.com]
#
# This file is part of Neo4j.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import platform
from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
from base64 import b64encode
from hashlib import sha256
from os import getenv, makedirs, listdir
from os.path import join as path_join, normpath, realpath, isdir
from random import randint
from socket import create_connection
from subprocess import call, check_output
from sys import stderr, stdin
from time import sleep

import boltkit.config as config

try:
    from urllib.request import urlopen, Request, HTTPError
    from urllib.parse import urlparse
except ImportError:
    from urllib2 import urlopen, Request, HTTPError
    from urlparse import urlparse


DIST_HOST = "dist.neo4j.org"

CORES_DIR = "cores"
CORE_DIR_FORMAT = "core-%d"
READ_REPLICAS_DIR = "read-replicas"
READ_REPLICA_DIR_FORMAT = "read-replica-%d"

INITIAL_DISCOVERY_PORT = 5000
INITIAL_TRANSACTION_PORT = 6000
INITIAL_RAFT_PORT = 7000
INITIAL_BOLT_PORT = 7687
INITIAL_HTTP_PORT = 7474
INITIAL_HTTPS_PORT = 6474


def bstr(s):
    return s if isinstance(s, (bytes, bytearray)) else s.encode("UTF-8")


class Downloader(object):

    def __init__(self, path, verbose=False):
        self.path = path
        self.verbose = verbose

    def write(self, message):
        if self.verbose:
            stderr.write(message)

    def download_build(self, address, build_id, package):
        """ Download from TeamCity build server.
        """
        url = "https://%s/repository/download/%s/.lastSuccessful/%s" % (address, build_id, package)
        user = getenv("TEAMCITY_USER")
        password = getenv("TEAMCITY_PASSWORD")
        auth_token = b"Basic " + b64encode(("%s:%s" % (user, password)).encode("iso-<PASSWORD>"))
        request = Request(url, headers={"Authorization": auth_token})
        package_path = path_join(self.path, package)

        self.write("Downloading build from %s... " % url)
        fin = urlopen(request)
        try:
            try:
                makedirs(self.path)
            except OSError:
                pass
            with open(package_path, "wb") as fout:
                more = True
                while more:
                    data = fin.read(8192)
                    if data:
                        fout.write(data)
                    else:
                        more = False
        finally:
            fin.close()

        return package_path

    def download_dist(self, address, package):
        """ Download from public distributions.
        """
        url = "http://%s/%s" % (address, package)
        package_path = path_join(self.path, package)

        self.write("Downloading dist from %s... " % url)
        fin = urlopen(url)
        try:
            try:
                makedirs(self.path)
            except OSError:
                pass
            with open(package_path, "wb") as fout:
                more = True
                while more:
                    data = fin.read(8192)
                    if data:
                        fout.write(data)
                    else:
                        more = False
        finally:
            fin.close()

        return package_path

    def download(self, edition, version, package_format):
        version_parts = version.replace("-", ".").replace("_", ".").split(".")
        if len(version_parts) == 3:
            major, minor, patch = version_parts
            milestone = ""
            tag = ""
        elif len(version_parts) == 4:
            major, minor, patch, milestone = version_parts
            tag = ""
        elif len(version_parts) == 5:
            major, minor, patch, milestone, tag = version_parts
        else:
            raise ValueError("Unrecognised version %r" % version)

        # Derive template and package version
        if package_format in ("unix.tar.gz", "windows.zip"):
            template = "neo4j-%s-%s-%s"
            package_version = "%s.%s.%s" % (major, minor, patch)
            if milestone:
                package_version = "%s-%s" % (package_version, milestone)
            if tag:
                package_version = "%s-%s" % (package_version, tag)
        else:
            raise ValueError("Unknown format %r" % package_format)
        package = template % (edition, package_version, package_format)

        dist_address = getenv("DIST_HOST", DIST_HOST)
        build_address = getenv("TEAMCITY_HOST")
        if build_address:
            try:
                build_id = "Neo4j%s%s_Packaging" % (major, minor)
                package = self.download_build(build_address, build_id, package)
            except HTTPError:
                package = self.download_dist(dist_address, package)
        else:
            package = self.download_dist(dist_address, package)
        self.write("\r\n")

        return package


def wait_for_server(host, port, timeout=30):
    port = port or 7474
    running = False
    t = 0
    while not running and t < timeout:
        try:
            s = create_connection((host, port))
        except IOError:
            sleep(1)
            t += 1
        else:
            s.close()
            running = True

    if not running:
        raise RuntimeError("Server %s:%d did not become available in %d seconds" % (host, port, timeout))


def hex_bytes(data):
    return b"".join(b"%02X" % b for b in bytearray(data))


def user_record(user, password):
    salt = bytearray(randint(0x00, 0xFF) for _ in range(16))
    m = sha256()
    m.update(salt)
    m.update(bstr(password))
    return b"%s:SHA-256,%s,%s:" % (bstr(user), hex_bytes(m.digest()), hex_bytes(salt))


class Controller(object):

    package_format = None

    @classmethod
    def extract(cls, archive, path):
        raise NotImplementedError("Not yet supported for this platform")

    @classmethod
    def download(cls, edition, version, path, **kwargs):
        downloader = Downloader(normpath(path), verbose=kwargs.get("verbose"))
        path = downloader.download(edition, version, cls.package_format)
        return realpath(path)

    @classmethod
    def install(cls, edition, version, path, **kwargs):
        package = cls.download(edition, version, path, **kwargs)
        home = cls.extract(package, path)
        return realpath(home)

    @classmethod
    def os_dependent_config(cls, instance_id):
        raise NotImplementedError("Not yet supported for this platform")

    def __init__(self, home, verbosity=0):
        self.home = home
        self.verbosity = verbosity

    def start(self, wait):
        raise NotImplementedError("Not yet supported for this platform")

    def stop(self, kill):
        raise NotImplementedError("Not yet supported for this platform")

    def create_user(self, user, password):
        raise NotImplementedError("Not yet supported for this platform")

    def set_user_role(self, user, role):
        raise NotImplementedError("Not yet supported for this platform")


class UnixController(Controller):

    package_format = "unix.tar.gz"

    @classmethod
    def extract(cls, archive, path):
        from tarfile import TarFile
        with TarFile.open(archive) as files:
            files.extractall(path)
            return path_join(path, files.getnames()[0])

    @classmethod
    def os_dependent_config(cls, instance_id):
        return {}

    def start(self, wait=True):
        http_uri, bolt_uri = config.extract_http_and_bolt_uris(self.home)
        check_output([path_join(self.home, "bin", "neo4j"), "start"])
        if wait:
            wait_for_server(http_uri.hostname, http_uri.port)
        return InstanceInfo(http_uri, bolt_uri, self.home)

    def stop(self, kill):
        if kill:
            output = check_output([path_join(self.home, "bin", "neo4j"), "status"])
            if output.startswith("Neo4j is running"):
                pid = output.split(" ")[-1].strip()
                check_output(["kill", "-9", pid])
            else:
                raise RuntimeError("Neo4j is not running")
        else:
            check_output([path_join(self.home, "bin", "neo4j"), "stop"])

    def create_user(self, user, password):
        data_dbms = path_join(self.home, "data", "dbms")
        try:
            makedirs(data_dbms)
        except OSError:
            pass
        with open(path_join(data_dbms, "auth"), "ab") as f:
            f.write(user_record(user, password))
            f.write(b"\r\n")

    def set_user_role(self, user, role):
        data_dbms = path_join(self.home, "data", "dbms")
        try:
            makedirs(data_dbms)
        except OSError:
            pass
        lines = []
        try:
            with open(path_join(data_dbms, "roles"), "r") as f:
                for line in f:
                    lines.append(line.rstrip())
        except IOError:
            lines += ["admin:", "architect:", "publisher:", "reader:"]
        for i, line in enumerate(lines):
            if line.startswith(role + ":"):
                users = line[6:].split(",")
                users.append(user)
                line = role + ":" + ",".join(users)
            lines[i] = line
        with open(path_join(data_dbms, "roles"), "w") as f:
            for line in lines:
                f.write(line)
                f.write("\n")


class WindowsController(Controller):

    package_format = "windows.zip"

    @classmethod
    def extract(cls, archive, path):
        from zipfile import ZipFile
        with ZipFile(archive, 'r') as files:
            files.extractall(path)
            return path_join(path, files.namelist()[0])

    @classmethod
    def os_dependent_config(cls, instance_id):
        return {config.WINDOWS_SERVICE_NAME_SETTING: instance_id}

    def start(self, wait=True):
        http_uri, bolt_uri = config.extract_http_and_bolt_uris(self.home)
        check_output([path_join(self.home, "bin", "neo4j.bat"), "install-service"])
        check_output([path_join(self.home, "bin", "neo4j.bat"), "start"])
        if wait:
            wait_for_server(http_uri.hostname, http_uri.port)
        return InstanceInfo(http_uri, bolt_uri, self.home)

    def stop(self, kill):<commit_msg>Fix updating of roles file<commit_after>#!/usr/bin/env python
# coding: utf-8

# Copyright (c) 2002-2016 "Neo Technology,"
# Network Engine for Objects in Lund AB [http://neotechnology.com]
#
# This file is part of Neo4j.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import platform
from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
from base64 import b64encode
from hashlib import sha256
from os import getenv, makedirs, listdir
from os.path import join as path_join, normpath, realpath, isdir
from random import randint
from socket import create_connection
from subprocess import call, check_output
from sys import stderr, stdin
from time import sleep

import boltkit.config as config

try:
    from urllib.request import urlopen, Request, HTTPError
    from urllib.parse import urlparse
except ImportError:
    from urllib2 import urlopen, Request, HTTPError
    from urlparse import urlparse


DIST_HOST = "dist.neo4j.org"

CORES_DIR = "cores"
CORE_DIR_FORMAT = "core-%d"
READ_REPLICAS_DIR = "read-replicas"
READ_REPLICA_DIR_FORMAT = "read-replica-%d"

INITIAL_DISCOVERY_PORT = 5000
INITIAL_TRANSACTION_PORT = 6000
INITIAL_RAFT_PORT = 7000
INITIAL_BOLT_PORT = 7687
INITIAL_HTTP_PORT = 7474
INITIAL_HTTPS_PORT = 6474


def bstr(s):
    return s if isinstance(s, (bytes, bytearray)) else s.encode("UTF-8")


class Downloader(object):

    def __init__(self, path, verbose=False):
        self.path = path
        self.verbose = verbose

    def write(self, message):
        if self.verbose:
            stderr.write(message)

    def download_build(self, address, build_id, package):
        """ Download from TeamCity build server.
        """
        url = "https://%s/repository/download/%s/.lastSuccessful/%s" % (address, build_id, package)
        user = getenv("TEAMCITY_USER")
        password = getenv("<PASSWORD>")
        auth_token = b"Basic " + b64encode(("%s:%s" % (user, password)).encode("iso-8859-1"))
        request = Request(url, headers={"Authorization": auth_token})
        package_path = path_join(self.path, package)

        self.write("Downloading build from %s... " % url)
        fin = urlopen(request)
        try:
            try:
                makedirs(self.path)
            except OSError:
                pass
            with open(package_path, "wb") as fout:
                more = True
                while more:
                    data = fin.read(8192)
                    if data:
                        fout.write(data)
                    else:
                        more = False
        finally:
            fin.close()

        return package_path

    def download_dist(self, address, package):
        """ Download from public distributions.
        """
        url = "http://%s/%s" % (address, package)
        package_path = path_join(self.path, package)

        self.write("Downloading dist from %s... " % url)
        fin = urlopen(url)
        try:
            try:
                makedirs(self.path)
            except OSError:
                pass
            with open(package_path, "wb") as fout:
                more = True
                while more:
                    data = fin.read(8192)
                    if data:
                        fout.write(data)
                    else:
                        more = False
        finally:
            fin.close()

        return package_path

    def download(self, edition, version, package_format):
        version_parts = version.replace("-", ".").replace("_", ".").split(".")
        if len(version_parts) == 3:
            major, minor, patch = version_parts
            milestone = ""
            tag = ""
        elif len(version_parts) == 4:
            major, minor, patch, milestone = version_parts
            tag = ""
        elif len(version_parts) == 5:
            major, minor, patch, milestone, tag = version_parts
        else:
            raise ValueError("Unrecognised version %r" % version)

        # Derive template and package version
        if package_format in ("unix.tar.gz", "windows.zip"):
            template = "neo4j-%s-%s-%s"
            package_version = "%s.%s.%s" % (major, minor, patch)
            if milestone:
                package_version = "%s-%s" % (package_version, milestone)
            if tag:
                package_version = "%s-%s" % (package_version, tag)
        else:
            raise ValueError("Unknown format %r" % package_format)
        package = template % (edition, package_version, package_format)

        dist_address = getenv("DIST_HOST", DIST_HOST)
        build_address = getenv("TEAMCITY_HOST")
        if build_address:
            try:
                build_id = "Neo4j%s%s_Packaging" % (major, minor)
                package = self.download_build(build_address, build_id, package)
            except HTTPError:
                package = self.download_dist(dist_address, package)
        else:
            package = self.download_dist(dist_address, package)
        self.write("\r\n")

        return package


def wait_for_server(host, port, timeout=30):
    port = port or 7474
    running = False
    t = 0
    while not running and t < timeout:
        try:
            s = create_connection((host, port))
        except IOError:
            sleep(1)
            t += 1
        else:
            s.close()
            running = True

    if not running:
        raise RuntimeError("Server %s:%d did not become available in %d seconds" % (host, port, timeout))


def hex_bytes(data):
    return b"".join(b"%02X" % b for b in bytearray(data))


def user_record(user, password):
    salt = bytearray(randint(0x00, 0xFF) for _ in range(16))
    m = sha256()
    m.update(salt)
    m.update(bstr(password))
    return b"%s:SHA-256,%s,%s:" % (bstr(user), hex_bytes(m.digest()), hex_bytes(salt))


class Controller(object):

    package_format = None

    @classmethod
    def extract(cls, archive, path):
        raise NotImplementedError("Not yet supported for this platform")

    @classmethod
    def download(cls, edition, version, path, **kwargs):
        downloader = Downloader(normpath(path), verbose=kwargs.get("verbose"))
        path = downloader.download(edition, version, cls.package_format)
        return realpath(path)

    @classmethod
    def install(cls, edition, version, path, **kwargs):
        package = cls.download(edition, version, path, **kwargs)
        home = cls.extract(package, path)
        return realpath(home)

    @classmethod
    def os_dependent_config(cls, instance_id):
        raise NotImplementedError("Not yet supported for this platform")

    def __init__(self, home, verbosity=0):
        self.home = home
        self.verbosity = verbosity

    def start(self, wait):
        raise NotImplementedError("Not yet supported for this platform")

    def stop(self, kill):
        raise NotImplementedError("Not yet supported for this platform")

    def create_user(self, user, password):
        raise NotImplementedError("Not yet supported for this platform")

    def set_user_role(self, user, role):
        raise NotImplementedError("Not yet supported for this platform")


class UnixController(Controller):

    package_format = "unix.tar.gz"

    @classmethod
    def extract(cls, archive, path):
        from tarfile import TarFile
        with TarFile.open(archive) as files:
            files.extractall(path)
            return path_join(path, files.getnames()[0])

    @classmethod
    def os_dependent_config(cls, instance_id):
        return {}

    def start(self, wait=True):
        http_uri, bolt_uri = config.extract_http_and_bolt_uris(self.home)
        check_output([path_join(self.home, "bin", "neo4j"), "start"])
        if wait:
            wait_for_server(http_uri.hostname, http_uri.port)
        return InstanceInfo(http_uri, bolt_uri, self.home)

    def stop(self, kill):
        if kill:
            output = check_output([path_join(self.home, "bin", "neo4j"), "status"])
            if output.startswith("Neo4j is running"):
                pid = output.split(" ")[-1].strip()
                check_output(["kill", "-9", pid])
            else:
                raise RuntimeError("Neo4j is not running")
        else:
            check_output([path_join(self.home, "bin", "neo4j"), "stop"])

    def create_user(self, user, password):
        data_dbms = path_join(self.home, "data", "dbms")
        try:
            makedirs(data_dbms)
        except OSError:
            pass
        with open(path_join(data_dbms, "auth"), "ab") as f:
            f.write(user_record(user, password))
            f.write(b"\r\n")

    def set_user_role(self, user, role):
        data_dbms = path_join(self.home, "data", "dbms")
        try:
            makedirs(data_dbms)
        except OSError:
            pass
        lines = []
        try:
            with open(path_join(data_dbms, "roles"), "r") as f:
                for line in f:
                    lines.append(line.rstrip())
        except IOError:
            lines += ["admin:", "architect:", "publisher:", "reader:"]

        role_set = False
        for i, line in enumerate(lines):
            split = line.split(":")
            role_str = split[0]
            users_str = split[1]

            if role == role_str:
                users = users_str.split(",") if users_str else []
                users.append(user)
                lines[i] = role + ":" + ",".join(users)
                role_set = True

        if not role_set:
            lines.append(role + ":" + user)

        with open(path_join(data_dbms, "roles"), "w") as f:
            for line in lines:
                f.write(line)
                f.write("\n")


class WindowsController(Controller):

    package_format = "windows.zip"

    @classmethod
    def extract(cls, archive, path):
        from zipfile import ZipFile
        with ZipFile(archive, 'r') as files:
            files.extractall(path)
            return path_join(path, files.namelist()[0])

    @classmethod
    def os_dependent_config(cls, instance_id):
        return {config.WINDOWS_SERVICE_NAME_SETTING: instance_id}

    def start(self, wait=True):
        http_uri, bolt_uri = config.extract_http_and_bolt_uris(self.home)
        check_output([path_join(self.home, "bin", "neo4j.bat"), "install-service"])
        check_output([path_join(self.home, "bin", "neo4j.bat"), "start"])
        if wait:
            wait_for_server(http_uri.hostname, http_uri.port)
        return InstanceInfo(http_uri, bolt_uri, self.home)

    def stop(self, kill):
{'fraction_non_alphanumeric': 0.10781990521327015, 'fraction_numerical': 0.00929639081297849, 'mean_word_length': 2.28665568369028, 'pattern_counts': {'":': 7, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6161259', 'n_tokens_mistral': 6630, 'n_tokens_neox': 6341, 'n_words': 1724}