<commit_before>from resource_management import *
import os

def create_hdfs_dir(path, owner, perms):
  Execute('hadoop fs -mkdir -p '+path, user='hdfs')
  Execute('hadoop fs -chown ' + owner + ' ' + path, user='hdfs')
  Execute('hadoop fs -chmod ' + str(perms) + ' ' + path, user='hdfs')

def package(name):
  import params
  Execute(params.package_mgr + ' install -y ' + name, user='root')

def add_repo(source, dest):
  import params
  if not os.path.isfile(dest + params.repo_file):
    Execute('cp ' + source + ' ' + dest)
    Execute(params.key_cmd)
    Execute(params.cache_cmd)

def cdap_config(name=None):
  import params
  print 'Setting up CDAP configuration for ' + name
  # We're only setup for *NIX, for now
  Directory(params.etc_prefix_dir,
            mode=0755
  )

  Directory(params.cdap_conf_dir,
            owner = params.cdap_user,
            group = params.user_group,
            recursive = True
  )

  XmlConfig("cdap-site.xml",
            conf_dir = params.cdap_conf_dir,
            configurations = params.config['configurations']['cdap-site'],
            owner = params.cdap_user,
            group = params.user_group
  )

  File(format("{params.cdap_conf_dir}/cdap-env.sh"),
       owner = params.cdap_user,
       content=InlineTemplate(params.cdap_env_sh_template)
  )

  cleanup_opts()

  # Copy logback.xml and logback-container.xml
  for i in 'logback.xml', 'logback-container.xml':
    no_op_test = 'ls ' + params.cdap_conf_dir + '/' + i + ' 2>/dev/null'
    Execute('cp -f /etc/cdap/conf.dist/' + i + ' ' + params.cdap_conf_dir, not_if=no_op_test)

  Execute('update-alternatives --install /etc/cdap/conf cdap-conf ' + params.cdap_conf_dir + ' 50')

def has_hive():
  import params
  if len(params.hive_metastore_host) > 0:
    return true
  else:
    return false

def get_hdp_version():
  command = 'hadoop version | head -n 1 | cut -d. -f4-'
  return_code, hdp_output = shell.call(command, timeout=20)

  if return_code != 0:
    raise Fail(
      'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))

  hdp_version = hdp_output.rstrip()
  match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)

  if match is None:
    raise Fail('Failed to get extracted version')

  return hdp_version

def cleanup_opts():
  Execute('sed -i \'s/"$OPTS"/$OPTS/g\' /opt/cdap/*/bin/service')<commit_msg>Pass name through to cleanup_opts and use call over Execute<commit_after>from resource_management import *
import os

def create_hdfs_dir(path, owner, perms):
  Execute('hadoop fs -mkdir -p '+path, user='hdfs')
  Execute('hadoop fs -chown ' + owner + ' ' + path, user='hdfs')
  Execute('hadoop fs -chmod ' + str(perms) + ' ' + path, user='hdfs')

def package(name):
  import params
  Execute(params.package_mgr + ' install -y ' + name, user='root')

def add_repo(source, dest):
  import params
  if not os.path.isfile(dest + params.repo_file):
    Execute('cp ' + source + ' ' + dest)
    Execute(params.key_cmd)
    Execute(params.cache_cmd)

def cdap_config(name=None):
  import params
  print 'Setting up CDAP configuration for ' + name
  # We're only setup for *NIX, for now
  Directory(params.etc_prefix_dir,
            mode=0755
  )

  Directory(params.cdap_conf_dir,
            owner = params.cdap_user,
            group = params.user_group,
            recursive = True
  )

  XmlConfig("cdap-site.xml",
            conf_dir = params.cdap_conf_dir,
            configurations = params.config['configurations']['cdap-site'],
            owner = params.cdap_user,
            group = params.user_group
  )

  File(format("{params.cdap_conf_dir}/cdap-env.sh"),
       owner = params.cdap_user,
       content=InlineTemplate(params.cdap_env_sh_template)
  )

  cleanup_opts(name)

  # Copy logback.xml and logback-container.xml
  for i in 'logback.xml', 'logback-container.xml':
    no_op_test = 'ls ' + params.cdap_conf_dir + '/' + i + ' 2>/dev/null'
    Execute('cp -f /etc/cdap/conf.dist/' + i + ' ' + params.cdap_conf_dir, not_if=no_op_test)

  Execute('update-alternatives --install /etc/cdap/conf cdap-conf ' + params.cdap_conf_dir + ' 50')

def has_hive():
  import params
  if len(params.hive_metastore_host) > 0:
    return true
  else:
    return false

def get_hdp_version():
  command = 'hadoop version | head -n 1 | cut -d. -f4-'
  return_code, hdp_output = shell.call(command, timeout=20)

  if return_code != 0:
    raise Fail(
      'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))

  hdp_version = hdp_output.rstrip()
  match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)

  if match is None:
    raise Fail('Failed to get extracted version')

  return hdp_version

def cleanup_opts(name):
  command = 'sed -i \'s/"$OPTS"/$OPTS/g\' /opt/cdap/' + name + '/bin/service'
  return_code, output = shell.call(command, timeout=20)
  # Execute('sed -i \'s/"$OPTS"/$OPTS/g\' /opt/cdap/' + name + '/bin/service', not_if=no_op_test)
{'fraction_non_alphanumeric': 0.14237355635205098, 'fraction_numerical': 0.009956192751891677, 'mean_word_length': 3.2388185654008437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '952202', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1768, 'n_words': 394}