<commit_before>
def run
  cleanCanonicalConfig
  cleanDatabrowserConfig
  cleanAdminConfig
  cleanDashboardConfig
  cleanPortalConfig
  cleanPortalExtConfig

  removeFiles
end

def cleanCanonicalConfig()
  contents = File.read('config/config.in/canonical_config.yml')

  contents = removeProfile(contents,"devapp1")
  contents = removeProfile(contents,"devjuggernauts")
  contents = removeProfile(contents,"local-pit")
  contents = removeProfile(contents,"local-maestro")
  contents = removeProfile(contents,"nxteam-pit")
  contents = removeProfile(contents,"nxteam-maestro")
  contents = removeProfile(contents,"nxteam")
  # not really profiles, but oh well
  contents = removeProfile(contents,"# Properties for systems being")
  contents = removeProfile(contents,"# These are Amazon west instances")

  contents = sanitizeConfigProperties(contents)

  File.open('config/config.in/canonical_config2.yml', 'w') {|f| f.write(contents) }
end

def cleanDatabrowserConfig
  contents = File.read('databrowser/config/config.yml')
  contents = sanitizeConfigProperties(contents)
  File.open('databrowser/config/config2.yml', 'w') {|f| f.write(contents) }
end

def cleanAdminConfig
  contents = File.read('admin-tools/admin-rails/config/config.yml')
  contents = removeProfile(contents,"test") # refers to devapp1
  contents = sanitizeConfigProperties(contents)
  File.open('admin-tools/admin-rails/config/config2.yml', 'w') {|f| f.write(contents) }
end


def cleanDashboardConfig
  contents = File.read('config/config.in/dashboard_config.yml')
  contents = sanitizeConfigProperties(contents)
  File.open('config/config.in/dashboard_config2.yml', 'w') {|f| f.write(contents) }
end

def cleanPortalConfig
  contents = File.read('config/config.in/portal_config.yml')
  contents = removeProfile(contents,"devlycans")
  contents = removeProfile(contents,"devlr1")
  contents = removeProfile(contents,"devlr2")
  contents = removeProfile(contents,"testlr1")
  contents = removeProfile(contents,"demo")
  contents = sanitizeConfigProperties(contents)
  File.open('config/config.in/portal_config2.yml', 'w') {|f| f.write(contents) }
end

def cleanPortalExtConfig
  contents = File.read('config/config.in/portal_ext_config.yml')
  contents = removeProfile(contents,"devlycans")
  contents = removeProfile(contents,"devlr1")
  contents = removeProfile(contents,"devlr2")
  contents = removeProfile(contents,"testlr1")
  contents = removeProfile(contents,"demo")
  contents = sanitizeConfigProperties(contents)
  File.open('config/config.in/portal_ext_config2.yml', 'w') {|f| f.write(contents) }
end


def removeFiles

  def deleteIfExists(file)
    if File.exists?(file)
      File.delete(file)
    end
  end

  deleteIfExists('data-access/dal/keyStore/ciKeyStore.jks')
  deleteIfExists('data-access/dal/keyStore/localKeyStore.jks')
  deleteIfExists('data-access/dal/keyStore/localRailsKey')
  deleteIfExists('databrowser/config/devmegatron_databrowser_config.yml')
  deleteIfExists('databrowser/config/megtomcat01_databrowser_config.yml')

end

def removeProfile(contents,profile)

  inProfile = false
  newContents = Array.new

  contents.split("\n").each do |line|
    if line.start_with?(profile)
      inProfile = true
    elsif line.match(/^[^\s].*/)
      inProfile = false
    end

    if !inProfile
      newContents << line
    end
  end

  newContents.join("\n")
end

def sanitizeConfigProperties(contents)<commit_msg>[us5225] Updated clean-config<commit_after>
SLI_ROOT = "."


def run
  cleanCanonicalConfig
  cleanDatabrowserConfig
  cleanAdminConfig
  cleanDashboardConfig
  cleanPortalConfig
  cleanPortalExtConfig

  removeFiles
end

def cleanCanonicalConfig()

  filename = "#{SLI_ROOT}/config/config.in/canonical_config.yml"
  contents = File.read(filename)

  contents = removeProfile(contents,"devapp1")
  contents = removeProfile(contents,"devjuggernauts")
  contents = removeProfile(contents,"local-pit")
  contents = removeProfile(contents,"local-maestro")
  contents = removeProfile(contents,"nxteam-pit")
  contents = removeProfile(contents,"nxteam-maestro")
  contents = removeProfile(contents,"nxteam")
  # not really profiles, but oh well
  contents = removeProfile(contents,"# Properties for systems being")
  contents = removeProfile(contents,"# These are Amazon west instances")

  contents = sanitizeConfigProperties(contents)

  File.open(filename, 'w') {|f| f.write(contents) }
end

def cleanDatabrowserConfig
  filename = "#{SLI_ROOT}/databrowser/config/config.yml"
  contents = File.read(filename)
  contents = sanitizeConfigProperties(contents)
  File.open(filename, 'w') {|f| f.write(contents) }
end

def cleanAdminConfig
  filename = "#{SLI_ROOT}/admin-tools/admin-rails/config/config.yml"
  contents = File.read(filename)
  contents = removeProfile(contents,"test") # refers to devapp1
  contents = sanitizeConfigProperties(contents)
  File.open(filename, 'w') {|f| f.write(contents) }
end


def cleanDashboardConfig
  filename = "#{SLI_ROOT}/config/config.in/dashboard_config.yml"
  contents = File.read(filename)
  contents = sanitizeConfigProperties(contents)
  File.open(filename, 'w') {|f| f.write(contents) }
end

def cleanPortalConfig
  filename = "#{SLI_ROOT}/config/config.in/portal_config.yml"
  contents = File.read(filename)
  contents = removeProfile(contents,"devlycans")
  contents = removeProfile(contents,"devlr1")
  contents = removeProfile(contents,"devlr2")
  contents = removeProfile(contents,"testlr1")
  contents = removeProfile(contents,"demo")
  contents = sanitizeConfigProperties(contents)
  File.open(filename, 'w') {|f| f.write(contents) }
end

def cleanPortalExtConfig
  filename = "#{SLI_ROOT}/config/config.in/portal_ext_config.yml"
  contents = File.read(filename)
  contents = removeProfile(contents,"devlycans")
  contents = removeProfile(contents,"devlr1")
  contents = removeProfile(contents,"devlr2")
  contents = removeProfile(contents,"testlr1")
  contents = removeProfile(contents,"demo")
  contents = sanitizeConfigProperties(contents)
  File.open(filename, 'w') {|f| f.write(contents) }
end


def removeFiles

  def deleteIfExists(file)
    if File.exists?(file)
      File.delete(file)
    end
  end

  deleteIfExists("#{SLI_ROOT}/data-access/dal/keyStore/ciKeyStore.jks")
  deleteIfExists("#{SLI_ROOT}/data-access/dal/keyStore/localKeyStore.jks")
  deleteIfExists("#{SLI_ROOT}/data-access/dal/keyStore/localRailsKey")
  deleteIfExists("#{SLI_ROOT}/databrowser/config/devmegatron_databrowser_config.yml")
  deleteIfExists("#{SLI_ROOT}/databrowser/config/megtomcat01_databrowser_config.yml")

end

def removeProfile(contents,profile)

  inProfile = false
  newContents = Array.new

  contents.split("\n").each do |line|
    if line.start_with?(profile)
      inProfile = true
    elsif line.match(/^[^\s].*/)
      inProfile = false
    end

    if !inProfile
      newContents << line
    end
  end

  newContents.join("\n")
end

def sanitizeConfigProperties(contents)
{'fraction_non_alphanumeric': 0.1242791234140715, 'fraction_numerical': 0.004325259515570935, 'mean_word_length': 5.525870178739416, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '766546', 'n_tokens_mistral': 2226, 'n_tokens_neox': 2162, 'n_words': 370}