<commit_before>#!/usr/bin/env python
#
# A class for auto identifying BioInformatics file formats.
# By Lee & Matt

import sys
import re
import json
import mmap
import codecs
from binaryornot.check import is_binary


# BioFormat: Defines the properties of a bioinformatic file format object:
class BioFormat(object):
	def __init__(self, name, file_type, compression, markers):
		self.name = str(name)
		self.file_type = str(file_type)
		self.compression = str(compression)
		self.markers = list(markers)


# BioFormat: Defines the properties of a bioinformatic file identification object:
class BioID(object):
	def __init__(self, path):
		self.binary_definitions = []
		self.text_definitions = []

		with open(path, "rU") as definition_file:
			contents = definition_file.read()

		# Load JSON format definitions and parse into binary and text lists
		for definition in json.loads(contents)["formats"]:
			if definition["type"] == "BIN":
				self.binary_definitions.append(
					BioFormat(definition["name"], definition["type"], definition["compression"], definition["markers"]))
			elif definition["type"] == "TEXT":
				self.text_definitions.append(
					BioFormat(definition["name"], definition["type"], definition["compression"], definition["markers"]))

	# Method used to search binary files for raw byte sequences
	def identify_binary(self, file_path):
		binary_input_file = open(file_path, "rb")  # Read in binary file as binary ("rb")

		if sys.platform == "win32":  # Check if os is windows
			mapped_file = mmap.mmap(binary_input_file.fileno(), 0, None, mmap.ACCESS_READ)
		else:
			mapped_file = mmap.mmap(binary_input_file.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ)

		for binary_file_type in self.binary_definitions:
			pattern_match = False
			for field in binary_file_type.markers:
				# Check for byte field in binary file
				if mapped_file.find(codecs.decode(field, "unicode_escape").encode("latin1")) != -1:
					pattern_match = True
				else:
					pattern_match = False
					break  # All byte fields must be present in order for the file type check to pass
			if pattern_match:
				mapped_file.close()
				binary_input_file.close()
				return binary_file_type.name

		mapped_file.close()<commit_msg>Switched format definition format from JSON to YAML<commit_after>#!/usr/bin/env python
#
# A class for auto identifying BioInformatics file formats.
# By Lee & Matt

import sys
import re
import yaml
import mmap
import codecs
from binaryornot.check import is_binary


# BioFormat: Defines the properties of a bioinformatic file format object:
class BioFormat(object):
	def __init__(self, name, file_type, compression, markers):
		self.name = str(name)
		self.file_type = str(file_type)
		self.compression = str(compression)
		self.markers = list(markers)


# BioFormat: Defines the properties of a bioinformatic file identification object:
class BioID(object):
	def __init__(self, path):
		self.binary_definitions = []
		self.text_definitions = []

		with open(path, "rU") as definition_file:
			contents = definition_file.read()

		# Load JSON format definitions and parse into binary and text lists
		for definition in yaml.safe_load(contents):
			if definition["type"] == "BIN":
				self.binary_definitions.append(
					BioFormat(definition["name"], definition["type"], definition["compression"], definition["markers"]))
			elif definition["type"] == "TEXT":
				self.text_definitions.append(
					BioFormat(definition["name"], definition["type"], definition["compression"], definition["markers"]))

	# Method used to search binary files for raw byte sequences
	def identify_binary(self, file_path):
		binary_input_file = open(file_path, "rb")  # Read in binary file as binary ("rb")

		if sys.platform == "win32":  # Check if os is windows
			mapped_file = mmap.mmap(binary_input_file.fileno(), 0, None, mmap.ACCESS_READ)
		else:
			mapped_file = mmap.mmap(binary_input_file.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ)

		for binary_file_type in self.binary_definitions:
			pattern_match = False
			for field in binary_file_type.markers:
				# Check for byte field in binary file
				if mapped_file.find(codecs.decode(field, "unicode_escape").encode("latin1")) != -1:
					pattern_match = True
				else:
					pattern_match = False
					break  # All byte fields must be present in order for the file type check to pass
			if pattern_match:
				mapped_file.close()
				binary_input_file.close()
				return binary_file_type.name

		mapped_file.close()
{'fraction_non_alphanumeric': 0.10364083091355819, 'fraction_numerical': 0.002680366316729953, 'mean_word_length': 4.668354430379747, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1124819', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1349, 'n_words': 404}