<commit_before>require "filemagic"
require "pathname"
require "yaml"

require_relative "directory"

module Tint
	class File
		attr_reader :path

		def initialize(path, name=nil)
			@path = Pathname.new(path).realpath.cleanpath
			unless @path.to_s.start_with?(PROJECT_PATH.to_s)
				raise "File is outside of project scope!"
			end

			@name = name
		end

		def self.get(params)
			Tint::File.new(PROJECT_PATH.join(params['splat'].join('/')))
		end

		def directory?
			::File.directory?(path)
		end

		def parent
			@parent ||= Tint::Directory.new(path.dirname)
		end

		def text?
			FileMagic.open(:mime) do |magic|
				magic.file(path.to_s).split('/').first == 'text'
			end
		end

		def markdown?
			[".md", ".markdown"].include? extension
		end

		def yml?
			[".yaml", ".yml"].include? extension
		end

		def root?
			path == PROJECT_PATH
		end

		def route
			"/files/#{relative_path}"
		end

		def relative_path
			path.relative_path_from(PROJECT_PATH)
		end

		def name
			@name ||= path.basename.to_s
		end

		def stream
			::File.foreach(path).with_index do |line, idx|
				yield line.chomp, idx
			end
		end

		def stream_content
			has_frontmatter = false
			doc_start = 0
			stream do |line, idx|
				if doc_start < 2
					has_frontmatter = true if line == '---' && idx == 0
					doc_start += 1 if line == '---'
					next if has_frontmatter
				end

				yield line
			end
		end

		def content?
			detect_content_or_frontmatter[0]
		end

		def frontmatter?
			detect_content_or_frontmatter[1]
		end

		def frontmatter
			YAML.safe_load(open(path))
		end

		def to_directory
			Tint::Directory.new(path)
		end

	protected

		def extension
			@extension ||= path.extname
		end

		def detect_content_or_frontmatter
			return @content_or_frontmatter if @content_or_frontmatter

			has_frontmatter = false
			::File.foreach(path).with_index do |line, idx|
				line.chomp!
				if line == '---' && idx == 0
					has_frontmatter = true
					next
				end

				if has_frontmatter && line == '---'
					return [true, has_frontmatter]
				end
			end

			@content_or_frontmatter = [!has_frontmatter, has_frontmatter]
		end
	end
end
<commit_msg>Allow deserialization of Time objects<commit_after>require "filemagic"
require "pathname"
require "yaml"

require_relative "directory"

module Tint
	class File
		attr_reader :path

		def initialize(path, name=nil)
			@path = Pathname.new(path).realpath.cleanpath
			unless @path.to_s.start_with?(PROJECT_PATH.to_s)
				raise "File is outside of project scope!"
			end

			@name = name
		end

		def self.get(params)
			Tint::File.new(PROJECT_PATH.join(params['splat'].join('/')))
		end

		def directory?
			::File.directory?(path)
		end

		def parent
			@parent ||= Tint::Directory.new(path.dirname)
		end

		def text?
			FileMagic.open(:mime) do |magic|
				magic.file(path.to_s).split('/').first == 'text'
			end
		end

		def markdown?
			[".md", ".markdown"].include? extension
		end

		def yml?
			[".yaml", ".yml"].include? extension
		end

		def root?
			path == PROJECT_PATH
		end

		def route
			"/files/#{relative_path}"
		end

		def relative_path
			path.relative_path_from(PROJECT_PATH)
		end

		def name
			@name ||= path.basename.to_s
		end

		def stream
			::File.foreach(path).with_index do |line, idx|
				yield line.chomp, idx
			end
		end

		def stream_content
			has_frontmatter = false
			doc_start = 0
			stream do |line, idx|
				if doc_start < 2
					has_frontmatter = true if line == '---' && idx == 0
					doc_start += 1 if line == '---'
					next if has_frontmatter
				end

				yield line
			end
		end

		def content?
			detect_content_or_frontmatter[0]
		end

		def frontmatter?
			detect_content_or_frontmatter[1]
		end

		def frontmatter
			YAML.safe_load(open(path, [Time]))
		end

		def to_directory
			Tint::Directory.new(path)
		end

	protected

		def extension
			@extension ||= path.extname
		end

		def detect_content_or_frontmatter
			return @content_or_frontmatter if @content_or_frontmatter

			has_frontmatter = false
			::File.foreach(path).with_index do |line, idx|
				line.chomp!
				if line == '---' && idx == 0
					has_frontmatter = true
					next
				end

				if has_frontmatter && line == '---'
					return [true, has_frontmatter]
				end
			end

			@content_or_frontmatter = [!has_frontmatter, has_frontmatter]
		end
	end
end

{'fraction_non_alphanumeric': 0.11226851851851852, 'fraction_numerical': 0.0032407407407407406, 'mean_word_length': 2.9935304990757854, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7604227', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1501, 'n_words': 371}