<commit_before># Copyright (c) 2014, 2015, 2016, 2017 <NAME> under LGPL version 2.1. See LICENSE for more information.
#
# fields.bytes - Ensures field is stored as "bytes"
#


# vim: set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :

from . import IRField, IR_NULL_STRINGS, irNull

from ..compat_str import tobytes

class IRBytesField(IRField):
	'''
		IRBytesField - Ensure the data is always "bytes" type.
		
		Similar to IRRawField, except IRRawField does not touch encoding, this forces to bytes.

		This is replacement for BINARY_FIELDS
	'''

	CAN_INDEX = False

	def __init__(self, name='', defaultValue=irNull):
		self.valueType = None
		self.defaultValue = defaultValue

	convert = IRField._convertBytes

	convertFromInput = IRField._convertBytes

	toStorage = IRField._convertBytes

	def __new__(self, name='', defaultValue=irNull):
		return IRField.__new__(self, name)


# vim: set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :<commit_msg>Allow specifying an explicit encoding for an IRBytesField, to convert to bytes using an alternative than defaultIREncoding<commit_after># Copyright (c) 2014, 2015, 2016, 2017 <NAME> under LGPL version 2.1. See LICENSE for more information.
#
# fields.bytes - Ensures field is stored as "bytes"
#


# vim: set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :

from . import IRField, IR_NULL_STRINGS, irNull

from ..compat_str import tobytes

class IRBytesField(IRField):
	'''
		IRBytesField - Ensure the data is always "bytes" type.
		
		Similar to IRRawField, except IRRawField does not touch encoding, this forces to bytes.

		This is replacement for BINARY_FIELDS
	'''

	CAN_INDEX = False

	def __init__(self, name='', defaultValue=irNull, encoding=None):
		'''
			__init__ - Create an IRBytesField object

			@param name <str> - Field name
			@param defaultValue <any> default irNull - Default value for this field
			@param encoding <None/str> - If None, defaultIREncoding will be used when converting to bytes,
			  otherwise you can provide an explicit encoding
		'''
		self.valueType = None
		self.defaultValue = defaultValue
		self.encoding = encoding

	def _convertBytes(self, value):
		return tobytes(value, self.encoding)

	convert = _convertBytes

	convertFromInput = _convertBytes

	toStorage = _convertBytes


	def __new__(self, name='', defaultValue=irNull, encoding=None):
		return IRField.__new__(self, name)


# vim: set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :
{'fraction_non_alphanumeric': 0.08559670781893004, 'fraction_numerical': 0.019753086419753086, 'mean_word_length': 4.512471655328798, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2333168', 'n_tokens_mistral': 829, 'n_tokens_neox': 730, 'n_words': 258}