<commit_before># author: <NAME> 26/4/2017

from wherehows.common.schemas import LhotseFlowRecord
from wherehows.common.schemas import LhotseJobRecord
from wherehows.common.schemas import LhotseFlowDagRecord
from wherehows.common.schemas import LhotseFlowOwnerRecord

from wherehows.common.writers import FileWriter
from wherehows.common import Constant
from wherehows.common.utils import DateFormater
from wherehows.common.enums import SchedulerType
from com.ziclix.python.sql import zxJDBC
from org.slf4j import LoggerFactory
import os, sys, json, gzip
import StringIO
import datetime, time
import DbUtil


class LhotseExtract:

    def __init__(self, args):
        self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__)
        self.app_id = int(args[Constant.APP_ID_KEY])
        self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY])
        self.lz_con = zxJDBC.connect(args[Constant.LZ_DB_URL_KEY],
                                     args[Constant.LZ_DB_USERNAME_KEY],
                                     args[Constant.LZ_DB_PASSWORD_KEY],
                                     args[Constant.LZ_DB_DRIVER_KEY])
        self.lz_cursor = self.lz_con.cursor()
        self.lookback_period = args[Constant.LZ_EXEC_ETL_LOOKBACK_MINS_KEY]
        self.app_folder = args[Constant.WH_APP_FOLDER_KEY]
        self.metadata_folder = self.app_folder + "/" + str(SchedulerType.LHOTSE) + "/" + str(self.app_id)

        if not os.path.exists(self.metadata_folder):
            try:<commit_msg>debug lhotse metadata etl: version 1.0<commit_after># author: <NAME> 26/4/2017

from wherehows.common.schemas import LhotseFlowRecord
from wherehows.common.schemas import LhotseJobRecord
from wherehows.common.schemas import LhotseFlowDagRecord
from wherehows.common.schemas import LhotseFlowOwnerRecord

from wherehows.common.writers import FileWriter
from wherehows.common import Constant
from wherehows.common.utils import DateFormater
from wherehows.common.enums import SchedulerType
from com.ziclix.python.sql import zxJDBC
from org.slf4j import LoggerFactory
import os, sys, json, gzip
import StringIO
import datetime, time
import DbUtil


class LhotseExtract:

    def __init__(self, args):
        ## set the charset
        reload(sys)
        sys.setdefaultencoding('utf-8')

        self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__)
        self.app_id = int(args[Constant.APP_ID_KEY])
        self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY])
        self.lz_con = zxJDBC.connect(args[Constant.LZ_DB_URL_KEY],
                                     args[Constant.LZ_DB_USERNAME_KEY],
                                     args[Constant.LZ_DB_PASSWORD_KEY],
                                     args[Constant.LZ_DB_DRIVER_KEY])
        self.lz_cursor = self.lz_con.cursor()
        self.lookback_period = args[Constant.LZ_EXEC_ETL_LOOKBACK_MINS_KEY]
        self.app_folder = args[Constant.WH_APP_FOLDER_KEY]
        self.metadata_folder = self.app_folder + "/" + str(SchedulerType.LHOTSE) + "/" + str(self.app_id)

        if not os.path.exists(self.metadata_folder):
            try:
{'fraction_non_alphanumeric': 0.08261836669844295, 'fraction_numerical': 0.0060374960279631395, 'mean_word_length': 3.142105263157895, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '527349', 'n_tokens_mistral': 1038, 'n_tokens_neox': 997, 'n_words': 185}