<commit_before>'''
Created on Apr 16, 2014

@author: jtaghiyar
'''

import os
import subprocess as sub
from plumber import Plumber
from job_manager import LocalJobManager
from workflow_manager import WorkFlow
from helpers import trim, make_dir, export_to_environ 


class ComponentAbstract(object):
    
    """
    component template.
    """
    
    def __init__(self, component_name, component_parent_dir=None, seed_dir_name=None):
        ''' 
        initialize general attributes that each component must have. 
        '''

        ## export component parent directory to the PYTHONPATH env var        
        if component_parent_dir is not None:
            export_to_environ(component_parent_dir, 'PYTHONPATH')
        
        ## import modules of the component, i.e. component_reqs and component_params.
        ## if component_parent_dir==None, then components directory must have been exported to 
        ## the PYTHONPATH env var beforehand.
        list_of_modules = ['component_' + x for x in['reqs', 'params']]
        m = __import__(component_name, globals(), locals(), list_of_modules, -1)

        if component_parent_dir is None:
            component_parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(m.__file__)))
       
        if seed_dir_name is None:
            seed_dir_name = 'component_seed'
        
        ## The component_ui is NOT imported, since all the input arguments should be passed to 
        ## the component_main from config file via updating self.args attribute that happens in
        ## the corresponding task of the component. Therefore, an empty namespace is initialized
        ## here.
        import argparse
        parser = argparse.ArgumentParser()
        args, _ = parser.parse_known_args()
#         args.__dict__['return_value'] = None
        
        ## general attribute
        self.component_name = component_name
        self.component_dir  = component_parent_dir
        self.seed_dir       = os.path.join(self.component_dir, component_name, seed_dir_name)
        
        ## modules and args
        self.args             = args
        self._modules         = m
        self.component_reqs   = self._modules.component_reqs
        self.component_params = self._modules.component_params
                
        ## from the component_reqs
        self.env_vars        = self.component_reqs.env_vars
        self.memory          = self.component_reqs.memory
        self.parallel        = self.component_reqs.parallel
        self.requirements    = self.component_reqs.requirements
        self.seed_version    = self.component_reqs.seed_version
        self.version         = self.component_reqs.version

    def run(self):
        """run component via system command line locally."""
        cmd, cmd_args = self.make_cmd()
        ljm = LocalJobManager()
        ljm.run_job(cmd, cmd_args, self.component_name)
    
    def focus(self, cmd, cmd_args, chunk):
        "update the cmd and cmd_args for each chunk."
        
        raise NotImplementedError("focus method called before implementation")
        return cmd, cmd_args
    
    def make_cmd(self, chunk=None):
        """make a command."""
        cmd = None
        cmd_args = None
        
        raise NotImplementedError("make_cmd method called before implementation")
        return cmd, cmd_args
    
    def test(self):
        """run unittest of the component."""
        raise NotImplementedError("test method called before implementation")  
<commit_msg>minor bug fix.<commit_after>'''
Created on Apr 16, 2014

@author: jtaghiyar
'''

import os
import subprocess as sub
from plumber import Plumber
from job_manager import LocalJobManager
from workflow_manager import WorkFlow
from helpers import trim, make_dir, export_to_environ 


class ComponentAbstract(object):
    
    """
    component template.
    """
    
    def __init__(self, component_name, component_parent_dir=None, seed_dir_name=None):
        ''' 
        initialize general attributes that each component must have. 
        '''

        ## export component parent directory to the PYTHONPATH env var        
        if component_parent_dir is not None:
            export_to_environ(component_parent_dir, 'PYTHONPATH')
        
        ## import modules of the component, i.e. component_reqs and component_params.
        ## if component_parent_dir==None, then components directory must have been exported to 
        ## the PYTHONPATH env var beforehand.
        list_of_modules = ['component_' + x for x in['reqs', 'params']]
        m = __import__(component_name, globals(), locals(), list_of_modules, -1)

        if component_parent_dir is None:
            component_parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(m.__file__)))
       
        if seed_dir_name is None:
            seed_dir_name = 'component_seed'
        
        ## The component_ui is NOT imported, since all the input arguments should be passed to 
        ## the component_main from config file via updating self.args attribute that happens in
        ## the corresponding task of the component. Therefore, an empty namespace is initialized
        ## here.
        import argparse
        parser = argparse.ArgumentParser()
        args, _ = parser.parse_known_args()
#         args.__dict__['return_value'] = None
        
        ## general attribute
        self.component_name = component_name
        self.component_dir  = component_parent_dir
        self.seed_dir       = os.path.join(self.component_dir, component_name, seed_dir_name)
        
        ## modules and args
        self.args             = args
        self._modules         = m
        self.component_reqs   = self._modules.component_reqs
        self.component_params = self._modules.component_params
                
        ## from the component_reqs
        self.env_vars        = self.component_reqs.env_vars
        self.memory          = self.component_reqs.memory
        self.parallel        = self.component_reqs.parallel
        self.requirements    = self.component_reqs.requirements.copy()
        self.seed_version    = self.component_reqs.seed_version
        self.version         = self.component_reqs.version

    def run(self):
        """run component via system command line locally."""
        cmd, cmd_args = self.make_cmd()
        ljm = LocalJobManager()
        ljm.run_job(cmd, cmd_args, self.component_name)
    
    def focus(self, cmd, cmd_args, chunk):
        "update the cmd and cmd_args for each chunk."
        
        raise NotImplementedError("focus method called before implementation")
        return cmd, cmd_args
    
    def make_cmd(self, chunk=None):
        """make a command."""
        cmd = None
        cmd_args = None
        
        raise NotImplementedError("make_cmd method called before implementation")
        return cmd, cmd_args
    
    def test(self):
        """run unittest of the component."""
        raise NotImplementedError("test method called before implementation")  

{'fraction_non_alphanumeric': 0.0745164960182025, 'fraction_numerical': 0.0019908987485779293, 'mean_word_length': 2.4058111380145277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5346670', 'n_tokens_mistral': 1887, 'n_tokens_neox': 1851, 'n_words': 538}