<commit_before>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

from test_support.smvbasetest import SmvBaseTest
from smv import *
from smv.dqm import *

import pyspark
from pyspark.context import SparkContext
from pyspark.sql import SQLContext, HiveContext
from pyspark.sql.functions import col, lit
from py4j.protocol import Py4JJavaError


class SmvFrameworkTest(SmvBaseTest):
    @classmethod
    def smvAppInitArgs(cls):
        return ['--smv-props', 'smv.stages=smvframework.stage']

    def _escapeRegex(self, s):
        import re
        return re.sub(r"([\[\]\(\)])", r"\\\1", s)

    def test_SmvCsvStringData(self):
        fqn = "smvframework.stage.modules.D1"
        df = self.df(fqn)
        expect = self.createDF("a:String;b:Integer", "x,10;y,1")
        self.should_be_same(expect, df)

    def test_SmvCsvStringData_with_error(self):
        fqn = D1WithError.fqn()
        with self.assertRaisesRegexp(Py4JJavaError, "SmvDqmValidationError"):
            df = self.df(fqn)

    def test_SmvMultiCsvFiles(self):
        self.createTempInputFile("multiCsvTest/f1", "col1\na\n")
        self.createTempInputFile("multiCsvTest/f2", "col1\nb\n")
        self.createTempInputFile("multiCsvTest.schema", "col1: String\n")

        fqn = "smvframework.stage.modules.MultiCsv"
        df = self.df(fqn)
        exp = self.createDF("col1: String", "a;b")
        self.should_be_same(df, exp)

    def test_SmvCsvFileWithUserSchema(self):
        self.createTempInputFile("test3.csv", "col1\na\nb\n")
        self.createTempInputFile("test3.schema", "col1: String\n")

        fqn = "smvframework.stage.modules.CsvFile"
        df = self.df(fqn)
        exp = self.createDF(CsvFile.UserSchema, "a;b")
        self.should_be_same(df, exp)

    def test_SmvMultiCsvFilesWithUserSchema(self):
        self.createTempInputFile("test3/f1", "col1\na\n")
        self.createTempInputFile("test3/f2", "col1\nb\n")
        self.createTempInputFile("test3.schema", "col1: String\n")

        fqn = "smvframework.stage.modules.MultiCsvWithUserSchema"
        df = self.df(fqn)
        exp = self.createDF(CsvFile.UserSchema, "a;b")
        self.should_be_same(df, exp)

    def test_SmvDQM(self):
        fqn = "smvframework.stage.modules.D3"

        msg =""": org.tresamigos.smv.SmvDqmValidationError: {
  "passed":false,
  "errorMessages": [
    {"FailTotalRuleCountPolicy(2)":"true"},
    {"FailTotalFixCountPolicy(1)":"false"},
    {"FailParserCountPolicy(1)":"true"}
  ],
  "checkLog": [
    "Rule: b_lt_03, total count: 1",
    "org.tresamigos.smv.dqm.DQMRuleError: b_lt_03 @FIELDS: b=0.5",
    "Fix: a_lt_1_fix, total count: 1"
  ]
}"""

        with self.assertRaisesRegexp(Py4JJavaError, self._escapeRegex(msg)):
            df = self.df(fqn)
            df.smvDumpDF()

    #TODO: add other SmvDataSet unittests

class SmvRunConfigTest1(SmvBaseTest):

    @classmethod
    def smvAppInitArgs(cls):
        return ['--smv-props', 'smv.config.s=s1', 'smv.config.i=1', 'smv.config.b=True', 'smv.stages=smvframework.stage',
                '-m', "None"]

    def test_SmvCsvStringData_with_SmvRunConfig(self):
        fqn = "smvframework.stage.modules.D4"
        df = self.df(fqn)
        expect = self.createDF("a: String;b: Integer",
            """test1_s1,1;
                test2_not_i2,4;
                test3_b,5""")
        self.should_be_same(expect, df)

class SmvRunConfigTest2(SmvBaseTest):

    @classmethod
    def smvAppInitArgs(cls):
        return ['--smv-props', 'smv.config.s=s2', 'smv.config.i=2', 'smv.config.b=false', 'smv.stages=smvframework.stage',
                '-m', "None"]

    def test_SmvCsvStringData_with_SmvRunConfig(self):
        fqn = "smvframework.stage.modules.D4"
        df = self.df(fqn)
        expect = self.createDF("a:String;b:Integer",
            """test1_not_s1,2;
                test2_i2,3;
                test3_not_b,6""")
        self.should_be_same(expect, df)
<commit_msg>Python tests shouldn't import their resources<commit_after>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

from test_support.smvbasetest import SmvBaseTest
from smv import *
from smv.dqm import *

import pyspark
from pyspark.context import SparkContext
from pyspark.sql import SQLContext, HiveContext
from pyspark.sql.functions import col, lit
from py4j.protocol import Py4JJavaError


class SmvFrameworkTest(SmvBaseTest):
    @classmethod
    def smvAppInitArgs(cls):
        return ['--smv-props', 'smv.stages=smvframework.stage']

    def _escapeRegex(self, s):
        import re
        return re.sub(r"([\[\]\(\)])", r"\\\1", s)

    def test_SmvCsvStringData(self):
        fqn = "smvframework.stage.modules.D1"
        df = self.df(fqn)
        expect = self.createDF("a:String;b:Integer", "x,10;y,1")
        self.should_be_same(expect, df)

    def test_SmvCsvStringData_with_error(self):
        fqn = "smvframework.stage.modules.D1WithError"
        with self.assertRaisesRegexp(Py4JJavaError, "SmvDqmValidationError"):
            df = self.df(fqn)

    def test_SmvMultiCsvFiles(self):
        self.createTempInputFile("multiCsvTest/f1", "col1\na\n")
        self.createTempInputFile("multiCsvTest/f2", "col1\nb\n")
        self.createTempInputFile("multiCsvTest.schema", "col1: String\n")

        fqn = "smvframework.stage.modules.MultiCsv"
        df = self.df(fqn)
        exp = self.createDF("col1: String", "a;b")
        self.should_be_same(df, exp)

    def test_SmvCsvFileWithUserSchema(self):
        self.createTempInputFile("test3.csv", "col1\na\nb\n")
        self.createTempInputFile("test3.schema", "col1: String\n")

        fqn = "smvframework.stage.modules.CsvFile"
        df = self.df(fqn)
        exp = self.createDF("1loc: String", "a;b")
        self.should_be_same(df, exp)

    def test_SmvMultiCsvFilesWithUserSchema(self):
        self.createTempInputFile("test3/f1", "col1\na\n")
        self.createTempInputFile("test3/f2", "col1\nb\n")
        self.createTempInputFile("test3.schema", "col1: String\n")

        fqn = "smvframework.stage.modules.MultiCsvWithUserSchema"
        df = self.df(fqn)
        exp = self.createDF("1loc: String", "a;b")
        self.should_be_same(df, exp)

    def test_SmvDQM(self):
        fqn = "smvframework.stage.modules.D3"

        msg =""": org.tresamigos.smv.SmvDqmValidationError: {
  "passed":false,
  "errorMessages": [
    {"FailTotalRuleCountPolicy(2)":"true"},
    {"FailTotalFixCountPolicy(1)":"false"},
    {"FailParserCountPolicy(1)":"true"}
  ],
  "checkLog": [
    "Rule: b_lt_03, total count: 1",
    "org.tresamigos.smv.dqm.DQMRuleError: b_lt_03 @FIELDS: b=0.5",
    "Fix: a_lt_1_fix, total count: 1"
  ]
}"""

        with self.assertRaisesRegexp(Py4JJavaError, self._escapeRegex(msg)):
            df = self.df(fqn)
            df.smvDumpDF()

    #TODO: add other SmvDataSet unittests

class SmvRunConfigTest1(SmvBaseTest):

    @classmethod
    def smvAppInitArgs(cls):
        return ['--smv-props', 'smv.config.s=s1', 'smv.config.i=1', 'smv.config.b=True', 'smv.stages=smvframework.stage',
                '-m', "None"]

    def test_SmvCsvStringData_with_SmvRunConfig(self):
        fqn = "smvframework.stage.modules.D4"
        df = self.df(fqn)
        expect = self.createDF("a: String;b: Integer",
            """test1_s1,1;
                test2_not_i2,4;
                test3_b,5""")
        self.should_be_same(expect, df)

class SmvRunConfigTest2(SmvBaseTest):

    @classmethod
    def smvAppInitArgs(cls):
        return ['--smv-props', 'smv.config.s=s2', 'smv.config.i=2', 'smv.config.b=false', 'smv.stages=smvframework.stage',
                '-m', "None"]

    def test_SmvCsvStringData_with_SmvRunConfig(self):
        fqn = "smvframework.stage.modules.D4"
        df = self.df(fqn)
        expect = self.createDF("a:String;b:Integer",
            """test1_not_s1,2;
                test2_i2,3;
                test3_not_b,6""")
        self.should_be_same(expect, df)

{'fraction_non_alphanumeric': 0.13219734937075397, 'fraction_numerical': 0.015591936741285221, 'mean_word_length': 3.07811080835604, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1063387', 'n_tokens_mistral': 3260, 'n_tokens_neox': 3160, 'n_words': 597}