<commit_before>import glob
import os
from subprocess import call, STDOUT
import concurrent.futures

# This class contains execution logic for gempak scripts.

class Gempak:

  def __init__(self, dataGetter):
    self.constants = dataGetter.constants
    self.redisConn = dataGetter.redisConn
    self.DEBUG = dataGetter.DEBUG
    return

  def runGempakScripts(self):
    prevWd = os.getcwd()
    os.chdir("scripts")
    for key,http in self.constants.modelGems.items():
      if key in self.constants.runTimes:
        if 'files' in http and key != "gfs" and key != "nam":
          for file in glob.glob("gempak/hres/*.sh"):
            print "Executing HiRes gempak scripts."
            # Customized Gempak scripts for High resolution data in scripts/gempak/hres
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            self.runCmd(cmd)
            print cmd
        else:
          print "Executing Non-HiRes gempak scripts."
          for file in glob.glob("gempak/*.sh"):
            # Non High-Res scripts
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            print cmd
            self.runCmd(cmd)
      # Flag all forecast hours as processed.
      if self.DEBUG == False:
        # Set to not processing status.
        self.redisConn.set(key, "0")
        for fHour in self.constants.modelTimes[key]:
          # Set all forecast hours processed.
          self.redisConn.set(key + '-' + fHour, "1")

    os.chdir(prevWd)
    return

  def doThreadedGempakScripts(self):
    cmdList = []
    prevWd = os.getcwd()
    os.chdir("scripts")
    for key,http in self.constants.modelGems.items():
      if key in self.constants.runTimes:
        if 'files' in http and key != "gfs" and key != "nam":
          for file in glob.glob("gempak/hres/*.sh"):
            print "Executing HiRes gempak scripts."
            # Customized Gempak scripts for High resolution data in scripts/gempak/hres
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            #cmdList.append(cmd)
            self.runCmd(cmd)
            print cmd
        else:
          print "Executing Non-HiRes gempak scripts."
          for file in glob.glob("gempak/*.sh"):
            # Non High-Res scripts
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            print cmd
            cmdList.append(cmd)
            #self.runCmd(cmd)

      if len(cmdList) > 0:
        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
          result = ''.join(executor.map(self.runCmd, cmdList))

      print "Done with Threads."

      # Flag all forecast hours as processed.
      if self.DEBUG == False:
        # Set to not processing status.
        self.redisConn.set(key, "0")
        for fHour in self.constants.modelTimes[key]:
          # Set all forecast hours processed.
          self.redisConn.set(key + '-' + fHour, "1")

    return

  def runCmd(self, cmd):
    FNULL = open(os.devnull, 'w')
    print "Currently executing..."
    print cmd
    call(cmd, shell=True, stdout=FNULL, stderr=STDOUT)
    return ''<commit_msg>More bugs.<commit_after>import glob
import os
from subprocess import call, STDOUT
import concurrent.futures

# This class contains execution logic for gempak scripts.

class Gempak:

  def __init__(self, dataGetter):
    self.constants = dataGetter.constants
    self.redisConn = dataGetter.redisConn
    self.DEBUG = dataGetter.DEBUG
    return

  def runGempakScripts(self):
    prevWd = os.getcwd()
    os.chdir("scripts")
    for key,http in self.constants.modelGems.items():
      if key in self.constants.runTimes:
        if 'files' in http and key != "gfs" and key != "nam":
          for file in glob.glob("gempak/hres/*.sh"):
            print "Executing HiRes gempak scripts."
            # Customized Gempak scripts for High resolution data in scripts/gempak/hres
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            self.runCmd(cmd)
            print cmd
        else:
          print "Executing Non-HiRes gempak scripts."
          for file in glob.glob("gempak/*.sh"):
            # Non High-Res scripts
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            print cmd
            self.runCmd(cmd)
      # Flag all forecast hours as processed.
      if self.DEBUG == False:
        # Set to not processing status.
        self.redisConn.set(key, "0")
        for fHour in self.constants.modelTimes[key]:
          # Set all forecast hours processed.
          self.redisConn.set(key + '-' + fHour, "1")

    os.chdir(prevWd)
    return

  def doThreadedGempakScripts(self):
    cmdList = []
    prevWd = os.getcwd()
    os.chdir("scripts")
    for key,http in self.constants.modelGems.items():
      if key in self.constants.runTimes:
        if 'files' in http and key != "gfs" and key != "nam":
          for file in glob.glob("gempak/hres/*.sh"):
            print "Executing HiRes gempak scripts."
            # Customized Gempak scripts for High resolution data in scripts/gempak/hres
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            #cmdList.append(cmd)
            self.runCmd(cmd)
            print cmd
        else:
          print "Executing Non-HiRes gempak scripts."
          for file in glob.glob("gempak/*.sh"):
            # Non High-Res scripts
            cmd = "tcsh "+ file + " " + key + " " + ",".join(self.constants.modelTimes[key]) + " " + self.constants.runTimes[key] + " " + self.constants.dataDirEnv
            print cmd
            cmdList.append(cmd)
            #self.runCmd(cmd)

      if len(cmdList) > 0:
        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
          result = ''.join(executor.map(self.runCmd, cmdList))

      print "Done with Threads."

      # Flag all forecast hours as processed.
      if self.DEBUG == False:
        # Set to not processing status.
        self.redisConn.set(key, "0")
        for fHour in self.constants.modelTimes[key]:
          # Set all forecast hours processed.
          self.redisConn.set(key + '-' + fHour, "1")

    os.chdir(prevWd)
    return

  def runCmd(self, cmd):
    FNULL = open(os.devnull, 'w')
    print "Currently executing..."
    print cmd
    call(cmd, shell=True, stdout=FNULL, stderr=STDOUT)
    return ''
{'fraction_non_alphanumeric': 0.12574850299401197, 'fraction_numerical': 0.001710863986313088, 'mean_word_length': 2.2582443102647467, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '199172', 'n_tokens_mistral': 2207, 'n_tokens_neox': 2232, 'n_words': 544}