<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2014-2015 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import


import os
import time
import re

from bigmler.tests.world import world, res_filename
from subprocess import check_call, CalledProcessError
from bigmler.checkpoint import file_number_of_lines
from bigmler.utils import SYSTEM_ENCODING, PYTHON3, open_mode
from bigml.api import check_resource
from bigmler.tests.common_steps import check_debug


def python3_contents(filename, prior_contents):
    """Check for a file that has alternative contents for Python3 and return
       its contents

    """
    directory = os.path.dirname(filename)
    basename = os.path.basename(filename)
    basename_name, basename_ext = basename.split(".")
    filename = os.path.join(directory, "%s_py3.%s" % ( \
        basename_name, basename_ext))
    try:
        with open(filename, open_mode("r")) as file_handler:
            return file_handler.read().strip("\n")
    except IOError:
        return prior_contents


#@step(r'I create a reify output for the resource in "(.*)" for "(.*)')
def i_create_output(step, output=None, language=None, resource_type='source',
                    add_fields=False):
    if output is None and language is None:
        assert False
    world.directory = os.path.dirname(output)
    world.folders.append(world.directory)
    resource_id = getattr(world, resource_type)['resource']
    try:
        command = (u"bigmler reify --id " + resource_id +
                   u" --store --output " + output)
        if add_fields:
            command += u' --add-fields'
        command = check_debug(command)
        if not PYTHON3:
            command.encode(SYSTEM_ENCODING)
        retcode = check_call(command, shell=True)
        if retcode < 0:
            assert False
        else:
            world.output = world.directory
            assert True
    except (OSError, CalledProcessError, IOError) as exc:
        assert False, str(exc)


#@step(r'the "(.*)" file is like "(.*)"')
def i_check_output_file(step, output=None, check_file=None):
    if check_file is None or output is None:
        assert False
    check_file = res_filename(check_file)
    try:
        output_file = os.path.join(world.directory, "reify.py")
        with open(check_file, open_mode("r")) as check_file_handler:
            check_contents = check_file_handler.read().strip("\n")
        # remove unicode mark for strings if Python3
        if PYTHON3:
            check_contents = check_contents.replace( \
                " u'", " '").replace("{u'", "{'")
        with open(output_file, open_mode("r")) as output_file:
            output_file_contents = output_file.read()
        #strip comments at the beginning of the file
        output_file_contents = re.sub(r'""".*"""', '', output_file_contents,
                                      flags=re.S).strip("\n")
        if check_contents == output_file_contents:
            assert True
        else:
            if PYTHON3:
                # look for an alternative in PYTHON3
                check_contents = python3_contents(check_file, check_contents)
            if check_contents == output_file_contents:
                assert True
            else:
                assert False, ("File contents:\n%s\nExpected contents:\n%s" %
                               (output_file_contents, check_contents))
    except Exception, exc:
        assert False, str(exc)


#@step(r'I create a BigML source with data "(.*)" and params "(.*)"')
def create_source(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename), args)
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
<commit_msg>Change in bigmler reify tests to cope with python 2 to 3 strings' changes<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2014-2015 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import


import os
import time
import re

from bigmler.tests.world import world, res_filename
from subprocess import check_call, CalledProcessError
from bigmler.checkpoint import file_number_of_lines
from bigmler.utils import SYSTEM_ENCODING, PYTHON3, open_mode
from bigml.api import check_resource
from bigmler.tests.common_steps import check_debug


def python3_contents(filename, prior_contents):
    """Check for a file that has alternative contents for Python3 and return
       its contents

    """
    directory = os.path.dirname(filename)
    basename = os.path.basename(filename)
    basename_name, basename_ext = basename.split(".")
    filename = os.path.join(directory, "%s_py3.%s" % ( \
        basename_name, basename_ext))
    try:
        with open(filename, open_mode("r")) as file_handler:
            return file_handler.read().strip("\n")
    except IOError:
        return prior_contents


#@step(r'I create a reify output for the resource in "(.*)" for "(.*)')
def i_create_output(step, output=None, language=None, resource_type='source',
                    add_fields=False):
    if output is None and language is None:
        assert False
    world.directory = os.path.dirname(output)
    world.folders.append(world.directory)
    resource_id = getattr(world, resource_type)['resource']
    try:
        command = (u"bigmler reify --id " + resource_id +
                   u" --store --output " + output)
        if add_fields:
            command += u' --add-fields'
        command = check_debug(command)
        if not PYTHON3:
            command.encode(SYSTEM_ENCODING)
        retcode = check_call(command, shell=True)
        if retcode < 0:
            assert False
        else:
            world.output = world.directory
            assert True
    except (OSError, CalledProcessError, IOError) as exc:
        assert False, str(exc)


#@step(r'the "(.*)" file is like "(.*)"')
def i_check_output_file(step, output=None, check_file=None):
    if check_file is None or output is None:
        assert False
    check_file = res_filename(check_file)
    try:
        output_file = os.path.join(world.directory, "reify.py")
        with open(check_file, open_mode("r")) as check_file_handler:
            check_contents = check_file_handler.read().strip("\n")
        # remove unicode mark for strings if Python3
        if PYTHON3:
            check_contents = check_contents.replace( \
                " u'", " '").replace("{u'", "{'").replace(' u"', ' "')
        with open(output_file, open_mode("r")) as output_file:
            output_file_contents = output_file.read()
        #strip comments at the beginning of the file
        output_file_contents = re.sub(r'""".*"""', '', output_file_contents,
                                      flags=re.S).strip("\n")
        if check_contents == output_file_contents:
            assert True
        else:
            if PYTHON3:
                # look for an alternative in PYTHON3
                check_contents = python3_contents(check_file, check_contents)
            if check_contents == output_file_contents:
                assert True
            else:
                assert False, ("File contents:\n%s\nExpected contents:\n%s" %
                               (output_file_contents, check_contents))
    except Exception, exc:
        assert False, str(exc)


#@step(r'I create a BigML source with data "(.*)" and params "(.*)"')
def create_source(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename), args)
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])

{'fraction_non_alphanumeric': 0.1015349630471859, 'fraction_numerical': 0.005685048322910745, 'mean_word_length': 2.746166950596252, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2239622', 'n_tokens_mistral': 2552, 'n_tokens_neox': 2485, 'n_words': 737}