<commit_before>// Copyright 2015 <NAME> <<EMAIL>>.
//
// Licensed under the MIT license<LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed except according to those terms.

extern crate libc;
extern crate rustc_serialize;

mod hash;
mod cache;

use std::path::Path;
use std::ffi::CStr;
use cache::Cache;

static LIB_CACHE: Cache<'static> = Cache { cache_dir: cache::CACHE_DIR, use_cache: true };

/**
 * Prepare the library for work.
 *
 * Not performing this step may cause parts to fail.
 */
 #[no_mangle]
 pub extern "C" fn init() {
 	match LIB_CACHE.init() {
 		Ok(_) => {}
 		Err(e) => println!("Error: {}", e),
 	}
 }

/**
 * Teardown for the library
 */
 #[no_mangle]
 pub extern "C" fn teardown() {
 	match LIB_CACHE.clean() {
 		Ok(_) => {}
 		Err(e) => println!("Error: {}", e),
 	}
 }

 pub fn get_phashes(path: &Path) -> hash::PerceptualHashes {
 	hash::get_perceptual_hashes(path, &hash::Precision::Medium, &LIB_CACHE)
 }


 pub fn get_ahash(path: &Path) -> u64 {
 	hash::get_perceptual_hash(&path, &hash::Precision::Medium, &hash::HashType::AHash, &LIB_CACHE)
 }

 pub fn get_dhash(path: &Path) -> u64 {
 	hash::get_perceptual_hash(&path, &hash::Precision::Medium, &hash::HashType::DHash, &LIB_CACHE)
 }

 pub fn get_phash(path: &Path) -> u64 {
 	hash::get_perceptual_hash(&path, &hash::Precision::Medium, &hash::HashType::DHash, &LIB_CACHE)
 }

 pub fn get_hamming_distance(hash1: u64, hash2: u64) -> u64 {
 	hash::calculate_hamming_distance(hash1, hash2)
 }

 // External proxies for the get_*hash methods

 #[no_mangle]
 pub extern "C" fn ext_get_ahash(path_char: *const libc::c_char) -> libc::uint64_t {
 	unsafe {
 		let path_str = CStr::from_ptr(path_char);
 		let image_path = match path_str.to_str() {
 			Ok(result) => result,
 			Err(e) => {
 				println!("Error: {}. Unable to parse '{}'",
 					e,
 					to_hex_string(path_str.to_bytes()));
 				panic!("Unable to parse path")
 			}
 		};
 		let path = Path::new(&image_path);
 		get_ahash(&path)
 	}
 }

 #[no_mangle]
 pub extern "C" fn ext_get_dhash(path_char: *const libc::c_char) -> libc::uint64_t {
 	unsafe {
 		let path_str = CStr::from_ptr(path_char);
 		let image_path = match path_str.to_str() {
 			Ok(result) => result,
 			Err(e) => {
 				println!("Error: {}. Unable to parse '{}'",
 					e,
 					to_hex_string(path_str.to_bytes()));
 				panic!("Unable to parse path")
 			}
 		};
 		let path = Path::new(&image_path);
 		get_dhash(&path)
 	}
 }

 #[no_mangle]
 pub extern "C" fn ext_get_phash(path_char: *const libc::c_char) -> libc::uint64_t {
 	unsafe {
 		let path_str = CStr::from_ptr(path_char);
 		let image_path = match path_str.to_str() {
 			Ok(result) => result,
 			Err(e) => {
 				println!("Error: {}. Unable to parse '{}'",
 					e,
 					to_hex_string(path_str.to_bytes()));
 				panic!("Unable to parse path")
 			}
 		};
 		let path = Path::new(&image_path);
 		get_phash(&path)
 	}
 }

 fn to_hex_string(bytes: &[u8]) -> String {
 	println!("length: {}", bytes.len());
 	let mut strs: Vec<String> = Vec::new();
 	for byte in bytes {
 		// println!("{:02x}", byte);
 		strs.push(format!("{:02x}", byte));
 	}
 	strs.join("\\x")
 }

 // Module for the tests
 //
 #[cfg(test)]
 mod tests {

 	use super::*;
 	use std::fs;
 	use std::path::Path;
 	use hash;

 	#[test]
 	fn test_can_get_test_images() {
 		let paths = fs::read_dir(&Path::new("./test_images")).unwrap();
 		let mut num_paths = 0;
 		for path in paths {
 			let orig_path = path.unwrap().path();
 			let ext = Path::new(&orig_path).extension();
 			match ext {
 				Some(_) => {
 					if ext.unwrap() == "jpg" {<commit_msg>Cleanup<commit_after>// Copyright 2015 <NAME> <<EMAIL>>.
//
// Licensed under the MIT license<LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed except according to those terms.

extern crate libc;
extern crate rustc_serialize;

mod hash;
mod cache;

use std::path::Path;
use std::ffi::CStr;
use cache::Cache;

static LIB_CACHE: Cache<'static> = Cache { cache_dir: cache::CACHE_DIR, use_cache: true };

/**
 * Prepare the library for work.
 *
 * Not performing this step may cause parts to fail.
 */
 #[no_mangle]
 pub extern "C" fn init() {
 	match LIB_CACHE.init() {
 		Ok(_) => {}
 		Err(e) => println!("Error: {}", e),
 	}
 }

/**
 * Teardown for the library
 */
 #[no_mangle]
 pub extern "C" fn teardown() {
 	match LIB_CACHE.clean() {
 		Ok(_) => {}
 		Err(e) => println!("Error: {}", e),
 	}
 }

 pub fn get_phashes(path: &Path) -> hash::PerceptualHashes {
 	hash::get_perceptual_hashes(path, &hash::Precision::Medium, &LIB_CACHE)
 }


 pub fn get_ahash(path: &Path) -> u64 {
 	hash::get_perceptual_hash(&path, &hash::Precision::Medium, &hash::HashType::AHash, &LIB_CACHE)
 }

 pub fn get_dhash(path: &Path) -> u64 {
 	hash::get_perceptual_hash(&path, &hash::Precision::Medium, &hash::HashType::DHash, &LIB_CACHE)
 }

 pub fn get_phash(path: &Path) -> u64 {
 	hash::get_perceptual_hash(&path, &hash::Precision::Medium, &hash::HashType::DHash, &LIB_CACHE)
 }

 pub fn get_hamming_distance(hash1: u64, hash2: u64) -> u64 {
 	hash::calculate_hamming_distance(hash1, hash2)
 }

 // External proxies for the get_*hash methods

 #[no_mangle]
 pub extern "C" fn ext_get_ahash(path_char: *const libc::c_char) -> libc::uint64_t {
 	unsafe {
 		let path_str = CStr::from_ptr(path_char);
 		let image_path = match path_str.to_str() {
 			Ok(result) => result,
 			Err(e) => {
 				println!("Error: {}. Unable to parse '{}'",
 					e,
 					to_hex_string(path_str.to_bytes()));
 				panic!("Unable to parse path")
 			}
 		};
 		let path = Path::new(&image_path);
 		get_ahash(&path)
 	}
 }

 #[no_mangle]
 pub extern "C" fn ext_get_dhash(path_char: *const libc::c_char) -> libc::uint64_t {
 	unsafe {
 		let path_str = CStr::from_ptr(path_char);
 		let image_path = match path_str.to_str() {
 			Ok(result) => result,
 			Err(e) => {
 				println!("Error: {}. Unable to parse '{}'",
 					e,
 					to_hex_string(path_str.to_bytes()));
 				panic!("Unable to parse path")
 			}
 		};
 		let path = Path::new(&image_path);
 		get_dhash(&path)
 	}
 }

 #[no_mangle]
 pub extern "C" fn ext_get_phash(path_char: *const libc::c_char) -> libc::uint64_t {
 	unsafe {
 		let path_str = CStr::from_ptr(path_char);
 		let image_path = match path_str.to_str() {
 			Ok(result) => result,
 			Err(e) => {
 				println!("Error: {}. Unable to parse '{}'",
 					e,
 					to_hex_string(path_str.to_bytes()));
 				panic!("Unable to parse path")
 			}
 		};
 		let path = Path::new(&image_path);
 		get_phash(&path)
 	}
 }

 fn to_hex_string(bytes: &[u8]) -> String {
 	println!("length: {}", bytes.len());
 	let mut strs: Vec<String> = Vec::new();
 	for byte in bytes {
 		// println!("{:02x}", byte);
 		strs.push(format!("{:02x}", byte));
 	}
 	strs.join("\\x")
 }

 // Module for the tests
 //
 #[cfg(test)]
 mod tests {

 	use std::fs;
 	use std::path::Path;
 	use hash;

 	#[test]
 	fn test_can_get_test_images() {
 		let paths = fs::read_dir(&Path::new("./test_images")).unwrap();
 		let mut num_paths = 0;
 		for path in paths {
 			let orig_path = path.unwrap().path();
 			let ext = Path::new(&orig_path).extension();
 			match ext {
 				Some(_) => {
 					if ext.unwrap() == "jpg" {
{'fraction_non_alphanumeric': 0.17792421746293247, 'fraction_numerical': 0.008786381109280615, 'mean_word_length': 3.6195307545973368, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3767134', 'n_tokens_mistral': 3131, 'n_tokens_neox': 2815, 'n_words': 608}