<commit_before>#!/usr/bin/python
# coding: utf8
#######################################################################
#
#   Copyright (c) 2015 Orange
#   <EMAIL>
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
########################################################################

import os
import time
import subprocess
import logging
import argparse
import yaml
import pprint
import sys
import shutil
import json
import datetime
from git import Repo
import keystoneclient.v2_0.client as ksclient
import glanceclient.client as glclient
import novaclient.client as nvclient
from neutronclient.v2_0 import client as ntclient

from orchestrator import *
from clearwater import *

import urllib
pp = pprint.PrettyPrinter(indent=4)


parser = argparse.ArgumentParser()
parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
parser.add_argument("-r", "--report",
                    help="Create json result file",
                    action="store_true")
args = parser.parse_args()

""" logging configuration """
logger = logging.getLogger('vIMS')
logger.setLevel(logging.INFO)

ch = logging.StreamHandler()
if args.debug:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.INFO)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

REPO_PATH = os.environ['repos_dir'] + '/functest/'
if not os.path.exists(REPO_PATH):
    logger.error("Functest repository directory not found '%s'" % REPO_PATH)
    exit(-1)
sys.path.append(REPO_PATH + "testcases/")
import functest_utils

with open("/home/opnfv/functest/conf/config_functest.yaml") as f:
    functest_yaml = yaml.safe_load(f)
f.close()

# Cloudify parameters
VIMS_DIR = REPO_PATH + \
    functest_yaml.get("general").get("directories").get("dir_vIMS")
VIMS_DATA_DIR = functest_yaml.get("general").get(
    "directories").get("dir_vIMS_data") + "/"
VIMS_TEST_DIR = functest_yaml.get("general").get(
    "directories").get("dir_repo_vims_test") + "/"
DB_URL = functest_yaml.get("results").get("test_db_url")

TENANT_NAME = functest_yaml.get("vIMS").get("general").get("tenant_name")
TENANT_DESCRIPTION = functest_yaml.get("vIMS").get(
    "general").get("tenant_description")
IMAGES = functest_yaml.get("vIMS").get("general").get("images")

CFY_MANAGER_BLUEPRINT = functest_yaml.get(
    "vIMS").get("cloudify").get("blueprint")
CFY_MANAGER_REQUIERMENTS = functest_yaml.get(
    "vIMS").get("cloudify").get("requierments")
CFY_INPUTS = functest_yaml.get("vIMS").get("cloudify").get("inputs")

CW_BLUEPRINT = functest_yaml.get("vIMS").get("clearwater").get("blueprint")
CW_DEPLOYMENT_NAME = functest_yaml.get("vIMS").get(
    "clearwater").get("deployment-name")
CW_INPUTS = functest_yaml.get("vIMS").get("clearwater").get("inputs")
CW_REQUIERMENTS = functest_yaml.get("vIMS").get(
    "clearwater").get("requierments")

CFY_DEPLOYMENT_DURATION = 0
CW_DEPLOYMENT_DURATION = 0

RESULTS = {'orchestrator': {'duration': 0, 'result': ''},
           'vIMS': {'duration': 0, 'result': ''},
           'sig_test': {'duration': 0, 'result': ''}}


def download_and_add_image_on_glance(glance, image_name, image_url):
    dest_path = VIMS_DATA_DIR + "tmp/"
    if not os.path.exists(dest_path):
        os.makedirs(dest_path)
    file_name = image_url.rsplit('/')[-1]
    if not functest_utils.download_url(image_url, dest_path):
        logger.error("Failed to download image %s" % file_name)
        return False

    image = functest_utils.create_glance_image(
        glance, image_name, dest_path + file_name)
    if not image:
        logger.error("Failed to upload image on glance")
        return False

    return image


def step_failure(step_name, error_msg):
    logger.error(error_msg)
    set_result(step_name, 0, error_msg)
    push_results()
    exit(-1)


def push_results():
    if args.report:
        logger.debug("Pushing results to DB....")

        scenario = functest_utils.get_scenario(self.logger)
        pod_name = functest_utils.get_pod_name(self.logger)

        functest_utils.push_results_to_db(db_url=DB_URL, case_name="vIMS",
                                          logger=logger, pod_name=pod_name,
                                          version=scenario,
                                          payload=RESULTS)
<commit_msg>bugfix: self.logger<commit_after>#!/usr/bin/python
# coding: utf8
#######################################################################
#
#   Copyright (c) 2015 Orange
#   <EMAIL>
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
########################################################################

import os
import time
import subprocess
import logging
import argparse
import yaml
import pprint
import sys
import shutil
import json
import datetime
from git import Repo
import keystoneclient.v2_0.client as ksclient
import glanceclient.client as glclient
import novaclient.client as nvclient
from neutronclient.v2_0 import client as ntclient

from orchestrator import *
from clearwater import *

import urllib
pp = pprint.PrettyPrinter(indent=4)


parser = argparse.ArgumentParser()
parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
parser.add_argument("-r", "--report",
                    help="Create json result file",
                    action="store_true")
args = parser.parse_args()

""" logging configuration """
logger = logging.getLogger('vIMS')
logger.setLevel(logging.INFO)

ch = logging.StreamHandler()
if args.debug:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.INFO)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

REPO_PATH = os.environ['repos_dir'] + '/functest/'
if not os.path.exists(REPO_PATH):
    logger.error("Functest repository directory not found '%s'" % REPO_PATH)
    exit(-1)
sys.path.append(REPO_PATH + "testcases/")
import functest_utils

with open("/home/opnfv/functest/conf/config_functest.yaml") as f:
    functest_yaml = yaml.safe_load(f)
f.close()

# Cloudify parameters
VIMS_DIR = REPO_PATH + \
    functest_yaml.get("general").get("directories").get("dir_vIMS")
VIMS_DATA_DIR = functest_yaml.get("general").get(
    "directories").get("dir_vIMS_data") + "/"
VIMS_TEST_DIR = functest_yaml.get("general").get(
    "directories").get("dir_repo_vims_test") + "/"
DB_URL = functest_yaml.get("results").get("test_db_url")

TENANT_NAME = functest_yaml.get("vIMS").get("general").get("tenant_name")
TENANT_DESCRIPTION = functest_yaml.get("vIMS").get(
    "general").get("tenant_description")
IMAGES = functest_yaml.get("vIMS").get("general").get("images")

CFY_MANAGER_BLUEPRINT = functest_yaml.get(
    "vIMS").get("cloudify").get("blueprint")
CFY_MANAGER_REQUIERMENTS = functest_yaml.get(
    "vIMS").get("cloudify").get("requierments")
CFY_INPUTS = functest_yaml.get("vIMS").get("cloudify").get("inputs")

CW_BLUEPRINT = functest_yaml.get("vIMS").get("clearwater").get("blueprint")
CW_DEPLOYMENT_NAME = functest_yaml.get("vIMS").get(
    "clearwater").get("deployment-name")
CW_INPUTS = functest_yaml.get("vIMS").get("clearwater").get("inputs")
CW_REQUIERMENTS = functest_yaml.get("vIMS").get(
    "clearwater").get("requierments")

CFY_DEPLOYMENT_DURATION = 0
CW_DEPLOYMENT_DURATION = 0

RESULTS = {'orchestrator': {'duration': 0, 'result': ''},
           'vIMS': {'duration': 0, 'result': ''},
           'sig_test': {'duration': 0, 'result': ''}}


def download_and_add_image_on_glance(glance, image_name, image_url):
    dest_path = VIMS_DATA_DIR + "tmp/"
    if not os.path.exists(dest_path):
        os.makedirs(dest_path)
    file_name = image_url.rsplit('/')[-1]
    if not functest_utils.download_url(image_url, dest_path):
        logger.error("Failed to download image %s" % file_name)
        return False

    image = functest_utils.create_glance_image(
        glance, image_name, dest_path + file_name)
    if not image:
        logger.error("Failed to upload image on glance")
        return False

    return image


def step_failure(step_name, error_msg):
    logger.error(error_msg)
    set_result(step_name, 0, error_msg)
    push_results()
    exit(-1)


def push_results():
    if args.report:
        logger.debug("Pushing results to DB....")

        scenario = functest_utils.get_scenario(logger)
        pod_name = functest_utils.get_pod_name(logger)

        functest_utils.push_results_to_db(db_url=DB_URL, case_name="vIMS",
                                          logger=logger, pod_name=pod_name,
                                          version=scenario,
                                          payload=RESULTS)

{'fraction_non_alphanumeric': 0.16290893015030947, 'fraction_numerical': 0.005083996463306808, 'mean_word_length': 4.162007986309185, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7455458', 'n_tokens_mistral': 3046, 'n_tokens_neox': 2975, 'n_words': 559}