<commit_before># ===============================================================================
# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
import md5
import os
import pickle

from envisage.ui.tasks.task_extension import TaskExtension
from envisage.ui.tasks.task_factory import TaskFactory
from pyface.tasks.action.schema import SMenu, SGroup
from pyface.tasks.action.schema_addition import SchemaAddition


# ============= standard library imports ========================
# ============= local library imports  ==========================
from pychron.envisage.browser.interpreted_age_browser_model import InterpretedAgeBrowserModel
from pychron.envisage.tasks.base_task_plugin import BaseTaskPlugin
from pychron.paths import paths, get_file_text
from pychron.pipeline.tasks.actions import ConfigureRecallAction, IdeogramAction, IsochronAction, SpectrumAction, \
    SeriesAction, BlanksAction, ICFactorAction, ResetFactoryDefaultsAction, VerticalFluxAction
from pychron.pipeline.tasks.browser_task import BrowserTask
from pychron.pipeline.tasks.preferences import PipelinePreferencesPane
from pychron.pipeline.tasks.task import PipelineTask
from pychron.envisage.browser.sample_browser_model import SampleBrowserModel


class PipelinePlugin(BaseTaskPlugin):
    def _file_defaults_default(self):
        ov = True
        files = [['pipeline_template_file', 'PIPELINE_TEMPLATES', ov],
                 ['icfactor_template', 'ICFACTOR', ov],
                 ['blanks_template', 'BLANKS', ov],
                 ['iso_evo_template', 'ISOEVO', ov],
                 ['ideogram_template', 'IDEO', ov],
                 ['spectrum_template', 'SPEC', ov],
                 ['isochron_template', 'ISOCHRON', ov],
                 ['csv_ideogram_template', 'CSV_IDEO', ov],
                 ['vertical_flux_template', 'VERTICAL_FLUX', ov],
                 ['analysis_table_template', 'ANALYSIS_TABLE', ov],
                 ['interpreted_age_table_template', 'INTERPRETED_AGE_TABLE', ov]]

        # open the manifest file to set the overwrite flag
        if os.path.isfile(paths.template_manifest_file):
            with open(paths.template_manifest) as rfile:
                manifest = pickle.load(rfile)
        else:
            manifest = {}

        for item in files:
            fn, t, o = item
            txt = get_file_text(t)
            h = md5.hash(txt)
            if fn in manifest and h == manifest[fn]:
                item[2] = False

            manifest[fn] = h

        with open(paths.template_manifest_file) as wfile:
            pickle.dump(manifest, wfile)

    def _pipeline_factory(self):
        model = self.application.get_service(SampleBrowserModel)
        iamodel = self.application.get_service(InterpretedAgeBrowserModel)
        t = PipelineTask(browser_model=model,
                         interpreted_age_browser_model=iamodel)
        return t

    def _browser_factory(self):
        model = self.application.get_service(SampleBrowserModel)
        t = BrowserTask(browser_model=model)
        return t

    def _browser_model_factory(self):
        return SampleBrowserModel(application=self.application)
<commit_msg>fixed pipeline template manifest #701<commit_after># ===============================================================================
# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
from envisage.ui.tasks.task_extension import TaskExtension
from envisage.ui.tasks.task_factory import TaskFactory
from pyface.tasks.action.schema import SMenu, SGroup
from pyface.tasks.action.schema_addition import SchemaAddition
# ============= standard library imports ========================
from hashlib import md5
import os
import pickle
# ============= local library imports  ==========================
from pychron.envisage.browser.interpreted_age_browser_model import InterpretedAgeBrowserModel
from pychron.envisage.tasks.base_task_plugin import BaseTaskPlugin
from pychron.paths import paths, get_file_text
from pychron.pipeline.tasks.actions import ConfigureRecallAction, IdeogramAction, IsochronAction, SpectrumAction, \
    SeriesAction, BlanksAction, ICFactorAction, ResetFactoryDefaultsAction, VerticalFluxAction
from pychron.pipeline.tasks.browser_task import BrowserTask
from pychron.pipeline.tasks.preferences import PipelinePreferencesPane
from pychron.pipeline.tasks.task import PipelineTask
from pychron.envisage.browser.sample_browser_model import SampleBrowserModel


class PipelinePlugin(BaseTaskPlugin):
    def _file_defaults_default(self):
        ov = True
        files = [['pipeline_template_file', 'PIPELINE_TEMPLATES', ov],
                 ['icfactor_template', 'ICFACTOR', ov],
                 ['blanks_template', 'BLANKS', ov],
                 ['iso_evo_template', 'ISOEVO', ov],
                 ['ideogram_template', 'IDEO', ov],
                 ['spectrum_template', 'SPEC', ov],
                 ['isochron_template', 'ISOCHRON', ov],
                 ['csv_ideogram_template', 'CSV_IDEO', ov],
                 ['vertical_flux_template', 'VERTICAL_FLUX', ov],
                 ['analysis_table_template', 'ANALYSIS_TABLE', ov],
                 ['interpreted_age_table_template', 'INTERPRETED_AGE_TABLE', ov]]

        # open the manifest file to set the overwrite flag
        if os.path.isfile(paths.template_manifest_file):
            with open(paths.template_manifest_file) as rfile:
                manifest = pickle.load(rfile)
        else:
            manifest = {}

        for item in files:
            fn, t, o = item
            txt = get_file_text(t)
            h = md5(txt).hexdigest()
            if fn in manifest and h == manifest[fn]:
                item[2] = False

            manifest[fn] = h

        with open(paths.template_manifest_file, 'w') as wfile:
            pickle.dump(manifest, wfile)

        return files

    def _pipeline_factory(self):
        model = self.application.get_service(SampleBrowserModel)
        iamodel = self.application.get_service(InterpretedAgeBrowserModel)
        t = PipelineTask(browser_model=model,
                         interpreted_age_browser_model=iamodel)
        return t

    def _browser_factory(self):
        model = self.application.get_service(SampleBrowserModel)
        t = BrowserTask(browser_model=model)
        return t

    def _browser_model_factory(self):
        return SampleBrowserModel(application=self.application)

{'fraction_non_alphanumeric': 0.1449087850950964, 'fraction_numerical': 0.0032345710958726873, 'mean_word_length': 3.5151869158878504, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3200793', 'n_tokens_mistral': 2080, 'n_tokens_neox': 1962, 'n_words': 541}