<commit_before>#!/usr/bin/env python
#---------------------------------------------------------------------------
# Copyright 2012 The Open Source Electronic Health Record Agent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#---------------------------------------------------------------------------

from __future__ import with_statement
import sys
import os
import subprocess
import shutil
import glob
import argparse
# add the current to sys.path
SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(SCRIPTS_DIR)
from VistATestClient import VistATestClientFactory, createTestClientArgParser
from VistARoutineExport import VistARoutineExport
from VistARoutineImport import VistARoutineImport
from VistAGlobalExport import VistAGlobalExport
from LoggerManager import logger, initConsoleLogging, initFileLogging
from VistATaskmanUtil import VistATaskmanUtil
from MCompReposReadMeGenerator import MCompReposReadMeGenerator
from GitUtils import switchBranch, getStatus

""" List of routine names that are excluded from export process """
ROUTINE_EXTRACT_EXCLUDE_LIST = (
   "ZGO", "ZGI", "xobw.*", "%*",
   "CHK2LEV", "CHKOP", "GENDASH", "GENOUT",
   "GETPASS", "GTMHELP", "GTMHLPLD", "LOADOP",
   "LOADVX", "MSG", "PINENTRY", "TTTGEN",
   "TTTSCAN"
)

""" List of routine names that are to be included again after the above has excluded a set of them """
ROUTINE_EXTRACT_EXCEPTION_LIST = (
  "%ut*",
)

""" Extract routines/globals from a VistA instance and import
    to the git repository
"""
class VistADataExtractor:
  def __init__(self, vistARepoDir, outputResultDir,
               outputLogDir, routineOutDir=None,
               gitBranch=None, generateReadMe=False,
               serialExport=False):
    assert os.path.exists(vistARepoDir)
    assert os.path.exists(outputResultDir)
    assert os.path.exists(outputLogDir)
    self._vistARepoDir = vistARepoDir
    self._outputLogDir = outputLogDir
    self._outputResultDir = outputResultDir
    self._packagesDir = os.path.join(self._vistARepoDir, "Packages")
    assert os.path.exists(self._packagesDir)
    self._packagesCSV = os.path.normpath(os.path.join(_vistARepoDir,
                                                 "Packages.csv"))
    assert os.path.exists(self._packagesCSV)
    self._routineOutputFile = os.path.join(self._outputResultDir, "Routines.ro")
    self._globalOutputDir = os.path.join(self._outputResultDir, "Globals")
    if not os.path.exists(self._globalOutputDir):
      os.mkdir(self._globalOutputDir)
    if routineOutDir:
      assert os.path.exists(routineOutDir)
    self._routineOutDir = routineOutDir
    self._generateReadMe = generateReadMe
    self._gitBranch = gitBranch
    self._serialExport = serialExport
  def extractData(self, vistATestClient):
    self.__setupLogging__(vistATestClient)
    self.__switchBranch__()
    self.__stopTaskman__(vistATestClient)
    self.__extractRoutines__(vistATestClient)
    self.__importZGORoutine__(vistATestClient)
    self.__exportAllGlobals__(vistATestClient)
    self.__chmodGlobalOutput__()
    self.__removePackagesTree__()
    self.__unpackRoutines__()
    self.__copyAllGlobals__()
    self.__splitGlobalFiles__()
    self.__populatePackageFiles__()
    self.__generatePackageReadMes__()
    self.__reportGitStatus__()
    self.__cleanup__()

  def __setupLogging__(self, vistATestClient):
    DEFAULT_LOGGING_FILENAME = "VistADataExtractor.log"<commit_msg>Fix vistaRepoDir usage<commit_after>#!/usr/bin/env python
#---------------------------------------------------------------------------
# Copyright 2012 The Open Source Electronic Health Record Agent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#---------------------------------------------------------------------------

from __future__ import with_statement
import sys
import os
import subprocess
import shutil
import glob
import argparse
# add the current to sys.path
SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(SCRIPTS_DIR)
from VistATestClient import VistATestClientFactory, createTestClientArgParser
from VistARoutineExport import VistARoutineExport
from VistARoutineImport import VistARoutineImport
from VistAGlobalExport import VistAGlobalExport
from LoggerManager import logger, initConsoleLogging, initFileLogging
from VistATaskmanUtil import VistATaskmanUtil
from MCompReposReadMeGenerator import MCompReposReadMeGenerator
from GitUtils import switchBranch, getStatus

""" List of routine names that are excluded from export process """
ROUTINE_EXTRACT_EXCLUDE_LIST = (
   "ZGO", "ZGI", "xobw.*", "%*",
   "CHK2LEV", "CHKOP", "GENDASH", "GENOUT",
   "GETPASS", "GTMHELP", "GTMHLPLD", "LOADOP",
   "LOADVX", "MSG", "PINENTRY", "TTTGEN",
   "TTTSCAN"
)

""" List of routine names that are to be included again after the above has excluded a set of them """
ROUTINE_EXTRACT_EXCEPTION_LIST = (
  "%ut*",
)

""" Extract routines/globals from a VistA instance and import
    to the git repository
"""
class VistADataExtractor:
  def __init__(self, vistARepoDir, outputResultDir,
               outputLogDir, routineOutDir=None,
               gitBranch=None, generateReadMe=False,
               serialExport=False):
    assert os.path.exists(vistARepoDir)
    assert os.path.exists(outputResultDir)
    assert os.path.exists(outputLogDir)
    self._vistARepoDir = vistARepoDir
    self._outputLogDir = outputLogDir
    self._outputResultDir = outputResultDir
    self._packagesDir = os.path.join(self._vistARepoDir, "Packages")
    assert os.path.exists(self._packagesDir)
    self._packagesCSV = os.path.normpath(os.path.join(self._vistARepoDir,
                                                 "Packages.csv"))
    assert os.path.exists(self._packagesCSV)
    self._routineOutputFile = os.path.join(self._outputResultDir, "Routines.ro")
    self._globalOutputDir = os.path.join(self._outputResultDir, "Globals")
    if not os.path.exists(self._globalOutputDir):
      os.mkdir(self._globalOutputDir)
    if routineOutDir:
      assert os.path.exists(routineOutDir)
    self._routineOutDir = routineOutDir
    self._generateReadMe = generateReadMe
    self._gitBranch = gitBranch
    self._serialExport = serialExport
  def extractData(self, vistATestClient):
    self.__setupLogging__(vistATestClient)
    self.__switchBranch__()
    self.__stopTaskman__(vistATestClient)
    self.__extractRoutines__(vistATestClient)
    self.__importZGORoutine__(vistATestClient)
    self.__exportAllGlobals__(vistATestClient)
    self.__chmodGlobalOutput__()
    self.__removePackagesTree__()
    self.__unpackRoutines__()
    self.__copyAllGlobals__()
    self.__splitGlobalFiles__()
    self.__populatePackageFiles__()
    self.__generatePackageReadMes__()
    self.__reportGitStatus__()
    self.__cleanup__()

  def __setupLogging__(self, vistATestClient):
    DEFAULT_LOGGING_FILENAME = "VistADataExtractor.log"
{'fraction_non_alphanumeric': 0.11912026442918891, 'fraction_numerical': 0.002288329519450801, 'mean_word_length': 3.666073546856465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5306097', 'n_tokens_mistral': 2333, 'n_tokens_neox': 2305, 'n_words': 575}