<commit_before># create_dirs.rb
# -------
# Creates HDFS directories and manages quota (if applicable)
# Run this in an HBase jruby shell
# -------
# General usage:
# hbase shell create_dirs.rb [user|groups] dir_file quota_yml home_dir perms
#
# User example:
# hbase shell create_dirs.rb user /tmp/user_dirs /tmp/user_quotas.yml /user 0700
#
# Group example:
# hbase shell create_dirs.rb groups /tmp/group_dirs /tmp/group_quotas.yml /groups 0770

include Java

require 'yaml'

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.hdfs.protocol.HdfsConstants

usage = 'hbase shell '\
  'create_dirs.rb [user|groups] dir_file quota_yml home_dir perms'
banner = "usage: #{usage}\n  -- need "

config = {
  'mode' => (ARGV[0] or raise "#{banner} [user|groups]"),
  'dir_file' => (ARGV[1] or raise "#{banner} dir_file"),
  'quota_file' => (ARGV[2] or raise "#{banner} quota_yml"),
  'home' => (ARGV[3] or raise "#{banner} home_dir"),
  'perms' => (ARGV[4] or raise "#{banner} perms")
}

# hdfs file system handle
fs = FileSystem.newInstance(Configuration.new)

# create home directory
path = Path.new(config['home'])
fs.mkdirs(path)
fs.setOwner(path, 'hdfs', 'hdfs')
fs.setPermission(path, FsPermission.new(0775))

# load quotas from file
quotas = YAML.load_file(config['quota_file'])

File.open(config['dir_file']) do |f|
  f.each_line do |dir|
    dir = dir.strip
    path = Path.new("#{config['home']}/#{dir}")
    puts "updating path: #{path}"

    # enforce ownership and permissions
    owner = config['mode'] == 'user' ? dir : 'hdfs'
    group = config['mode'] == 'groups' ? dir : 'hdfs'
    perms = FsPermission.new(config['perms'].to_i(8))

    # create the directory
    fs.mkdirs(path)
    fs.setOwner(path, owner, group)
    fs.setPermission(path, perms)

    # Takes a string with a unit-prefix
    # returns the result as a "long"
    # Examples are:
    #   string2long(600) => 600
    #   string2long('1') => 1
    #   string2long('1k') => 1024
    #   string2long('30T') => 32985348833280
    def string2long(s)
      s = s.to_s.downcase
      prefixes = {
        ''  => 1,
        'k' => 1024,
        'm' => 1024**2,
        'g' => 1024**3,
        't' => 1024**4,
        'p' => 1024**5
      }

      base = s[/[0-9.]*/].to_i
      prefix = prefixes[s[/[kmgtp]/]]

      base * prefix
    end

    # set space quota
    if quotas[dir]
      max_quota = HdfsConstants::QUOTA_DONT_SET
      quota = if quotas[dir] == 'NO_QUOTA'
                max_quota - 1
              else
                string2long(quotas[dir])
              end
      fs.setQuota(path, max_quota, quota)
    end
  end
end

exit<commit_msg>utilize the quota_reset hdfs constant<commit_after># create_dirs.rb
# -------
# Creates HDFS directories and manages quota (if applicable)
# Run this in an HBase jruby shell
# -------
# General usage:
# hbase shell create_dirs.rb [user|groups] dir_file quota_yml home_dir perms
#
# User example:
# hbase shell create_dirs.rb user /tmp/user_dirs /tmp/user_quotas.yml /user 0700
#
# Group example:
# hbase shell create_dirs.rb groups /tmp/group_dirs /tmp/group_quotas.yml /groups 0770

include Java

require 'yaml'

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.hdfs.protocol.HdfsConstants

usage = 'hbase shell '\
  'create_dirs.rb [user|groups] dir_file quota_yml home_dir perms'
banner = "usage: #{usage}\n  -- need "

config = {
  'mode' => (ARGV[0] or raise "#{banner} [user|groups]"),
  'dir_file' => (ARGV[1] or raise "#{banner} dir_file"),
  'quota_file' => (ARGV[2] or raise "#{banner} quota_yml"),
  'home' => (ARGV[3] or raise "#{banner} home_dir"),
  'perms' => (ARGV[4] or raise "#{banner} perms")
}

# hdfs file system handle
fs = FileSystem.newInstance(Configuration.new)

# create home directory
path = Path.new(config['home'])
fs.mkdirs(path)
fs.setOwner(path, 'hdfs', 'hdfs')
fs.setPermission(path, FsPermission.new(0775))

# load quotas from file
quotas = YAML.load_file(config['quota_file'])

File.open(config['dir_file']) do |f|
  f.each_line do |dir|
    dir = dir.strip
    path = Path.new("#{config['home']}/#{dir}")
    puts "updating path: #{path}"

    # enforce ownership and permissions
    owner = config['mode'] == 'user' ? dir : 'hdfs'
    group = config['mode'] == 'groups' ? dir : 'hdfs'
    perms = FsPermission.new(config['perms'].to_i(8))

    # create the directory
    fs.mkdirs(path)
    fs.setOwner(path, owner, group)
    fs.setPermission(path, perms)

    # Takes a string with a unit-prefix
    # returns the result as a "long"
    # Examples are:
    #   string2long(600) => 600
    #   string2long('1') => 1
    #   string2long('1k') => 1024
    #   string2long('30T') => 32985348833280
    def string2long(s)
      s = s.to_s.downcase
      prefixes = {
        ''  => 1,
        'k' => 1024,
        'm' => 1024**2,
        'g' => 1024**3,
        't' => 1024**4,
        'p' => 1024**5
      }

      base = s[/[0-9.]*/].to_i
      prefix = prefixes[s[/[kmgtp]/]]

      base * prefix
    end

    # set space quota
    if quotas[dir]
      max_quota = HdfsConstants::QUOTA_RESET
      quota =
        if quotas[dir] == 'NO_QUOTA'
          max_quota
        else
          string2long(quotas[dir])
        end
      fs.setQuota(path, max_quota, quota)
    end
  end
end

exit
{'fraction_non_alphanumeric': 0.15363784076548112, 'fraction_numerical': 0.029066618523199132, 'mean_word_length': 3.0379008746355685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4663470', 'n_tokens_mistral': 2163, 'n_tokens_neox': 2000, 'n_words': 472}