<commit_before># encoding: UTF-8

# Author::    <NAME>
# Copyright:: Copyright (c) 2009 <NAME>

require 'rubygems'
require 'yaml'
require 'ya2yaml'

class Sofa::Storage::File < Sofa::Storage

	def self.available?
		Sofa['STORAGE']['File'] && Sofa['STORAGE']['File']['data_dir']
	end

	def initialize(sd)
		super
		@dir = Sofa['STORAGE']['File']['data_dir'] + @sd[:folder][:dir]
	end

	def val(id = nil)
		if id
			load_yaml id
		else
			# this could be HUGE.
			_select_all({}).inject({}) {|v,id|
				v[id] = load_yaml id
				v
			}
		end
	end

	def build(v)
		clear
		v.each {|id,v| store(id,v) }
		self
	end

	def clear
		glob.each {|file| ::File.unlink ::File.join(@dir,file) }
		self
	end

	def store(id,v)
		save_yaml(id,v)
	end

	def delete(id)
		remove_file(id) && id
	end

	private

	def _select_by_id(conds)
		glob(conds[:id].to_a).collect {|f| f[/(?=\d).*/][Sofa::REX::ID] }.compact
	end

	def _select_by_d(conds)
		glob(conds[:d].to_s).collect {|f| f[/(?=\d).*/][Sofa::REX::ID] }.compact
	end

	def _select_all(conds)
		glob.collect {|f| f[/(?=\d).*/][Sofa::REX::ID] }.compact
	end

	def glob(id = :all)
		glob_pattern = "#{file_prefix}#{pattern_for id}.[a-z]*"
		::Dir.chdir(@dir) { ::Dir.glob glob_pattern }
	end

	def file_prefix
		(@sd[:name] == 'main') ? '' : @sd[:name].sub(/^main-/,'') + '_'
	end

	def pattern_for(id)
		if id.is_a? Array
			"{#{id.join ','}}"
		elsif id == :all
			'[0-9]*_*'
		elsif id =~ /\A\d{4,8}\z/
			"#{id}*"
		end
	end

	def load_yaml(id)
		v = nil
		file = glob(id.to_a).sort.first
		::File.open(::File.join(@dir,file),'r') {|f|
			f.flock ::File::LOCK_SH
			v = f.read
			f.flock ::File::LOCK_UN
		} if file
		(file[/\.yaml$/] ? YAML.load(v) : v) if v
	end

	def save_yaml(id,v)
		new_id = false
		if id == :new_id
			id = new_id(v)
			new_id = true
		end

		file = "#{file_prefix}#{id}.yaml"
		::File.open(::File.join(@dir,file),'a') {|f|
			break if new_id && f.pos != 0 # duplicate id

			f.flock ::File::LOCK_EX
			f.seek 0
			f.truncate 0
			f << v.ya2yaml(:syck_compatible => true)
			f.flock ::File::LOCK_UN
			f.chmod 0664
		} && id
	end

	def remove_file(id)
		glob(id.to_a) .each {|file|
			::File.unlink ::File.join(@dir,file) # may include child files
		}<commit_msg>Storage::File#save_yaml treat raw files with their original exts.<commit_after># encoding: UTF-8

# Author::    <NAME>
# Copyright:: Copyright (c) 2009 <NAME>

require 'rubygems'
require 'yaml'
require 'ya2yaml'

class Sofa::Storage::File < Sofa::Storage

	def self.available?
		Sofa['STORAGE']['File'] && Sofa['STORAGE']['File']['data_dir']
	end

	def initialize(sd)
		super
		@dir = Sofa['STORAGE']['File']['data_dir'] + @sd[:folder][:dir]
	end

	def val(id = nil)
		if id
			load_yaml id
		else
			# this could be HUGE.
			_select_all({}).inject({}) {|v,id|
				v[id] = load_yaml id
				v
			}
		end
	end

	def build(v)
		clear
		v.each {|id,v| store(id,v) }
		self
	end

	def clear
		glob.each {|file| ::File.unlink ::File.join(@dir,file) }
		self
	end

	def store(id,v)
		save_yaml(id,v)
	end

	def delete(id)
		remove_file(id) && id
	end

	private

	def _select_by_id(conds)
		glob(conds[:id].to_a).collect {|f| f[/(?=\d).*/][Sofa::REX::ID] }.compact
	end

	def _select_by_d(conds)
		glob(conds[:d].to_s).collect {|f| f[/(?=\d).*/][Sofa::REX::ID] }.compact
	end

	def _select_all(conds)
		glob.collect {|f| f[/(?=\d).*/][Sofa::REX::ID] }.compact
	end

	def glob(id = :all)
		glob_pattern = "#{file_prefix}#{pattern_for id}.[a-z]*"
		::Dir.chdir(@dir) { ::Dir.glob glob_pattern }
	end

	def file_prefix
		(@sd[:name] == 'main') ? '' : @sd[:name].sub(/^main-/,'') + '_'
	end

	def pattern_for(id)
		if id.is_a? Array
			"{#{id.join ','}}"
		elsif id == :all
			'[0-9]*_*'
		elsif id =~ /\A\d{4,8}\z/
			"#{id}*"
		end
	end

	def load_yaml(id)
		v = nil
		file = glob(id.to_a).sort.first
		::File.open(::File.join(@dir,file),'r') {|f|
			f.flock ::File::LOCK_SH
			v = f.read
			f.flock ::File::LOCK_UN
		} if file
		(file[/\.yaml$/] ? YAML.load(v) : v) if v
	end

	def save_yaml(id,v,ext)
		if id == :new_id
			id = new_id(v)
			new_id = true
		end

		if ext
			val = v
			ext = 'y' if ext == 'yaml'
			remove_file id
		else
			val = v.ya2yaml(:syck_compatible => true)
			ext = 'yaml'
		end

		file = "#{file_prefix}#{id}.#{ext}"
		::File.open(::File.join(@dir,file),'a') {|f|
			break if new_id && f.pos != 0 # duplicate id

			f.flock ::File::LOCK_EX
			f.seek 0
			f.truncate 0
			f << val
			f.flock ::File::LOCK_UN
			f.chmod 0664
		} && id
	end

	def remove_file(id)
		glob(id.to_a) .each {|file|
			::File.unlink ::File.join(@dir,file) # may include child files
		}
{'fraction_non_alphanumeric': 0.20882225119513256, 'fraction_numerical': 0.007822685788787484, 'mean_word_length': 3.2778810408921935, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '621193', 'n_tokens_mistral': 2294, 'n_tokens_neox': 2048, 'n_words': 424}