<commit_before>'use strict';

const _ = require('lodash')
const chrono = require('chrono-node')
const humanname = require('humanname')
const phoneUtil = require('google-libphonenumber').PhoneNumberUtil.getInstance()

var getTheRightData = function (node, {
	attr = null,
	extractor = null,
	filter = null,
	parser = null
} = {}) {

	var result = null

	if (attr) {
		result = node.attr(attr)
	} else if (extractor === "html") {
		result = node.html()
	} else {
		result = node.text()
	}

	if (extractor && extractor !== "html") {
		result = extractByExtractor(result, extractor)
	}
	if (filter) {
		result = filterData(result, filter)
	}
	if (parser) {
		result = parseData(result, parser)
	}

	return result
}

var parseData = function (data, regex) {
	var result = data
	if (regex) {
		try {
			var rgx = regex
			if (_.isString(regex)) {
				rgx = new RegExp(regex, 'gim')
			}
			result = data.match(rgx)[0]
		} catch (error) {
			// console.log("Regex error: ", error);
		}
	}

	return result
}

var filterData = function (data, filter) {
	var result = data
	if (["raw"].includes(filter)) {
		// let the raw data
	} else if (["trim"].includes(filter)) {
		result = result.trim()
	} else if (["lowercase", "lcase"].includes(filter)) {
		result = result.toLowerCase()
	} else if (["uppercase", "ucase"].includes(filter)) {
		result = result.toUpperCase()
	} else if (["capitalize", "cap"].includes(filter)) {
		result = _.startCase(result)
	} else if (["numbers"].includes(filter)) {
		result = result.replace(/\D/g, "")
	} else if (result) {
		// Default trim and set one spaces
		result = result.replace(/\s+/gm, " ").trim()
	}
	return result
}

var extractByExtractor = function (data, extractor, plural = false) {
	var result = data
	var emailRegex = /([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)/gi

	if (extractor.includes("phone")) {
		// let countryCode = result.match(/([A-Z])+/)[0]
		// console.log("countryCode", countryCode);
		// if (countryCode) {
		// 	try {
		// 		result = phoneUtil.parse(result, "US") // with option, the country code number
		// 	} catch (e) {
		// 		//
		// 	}
		// }
	} else if (["email", "mail", "@"].includes(extractor)) {
		if (plural) {
			result = data.match(emailRegex) || data
		} else {
			result = data.match(emailRegex)[0] || data
		}
	} else if (["date", "d"].includes(extractor)) {
		result = chrono.casual.parseDate(data).toString()
	} else if (["fullName", "firstName", "lastName", "initials", "suffix", "salutation"].includes(extractor)) {
		result = humanname.parse(data)
		if ("fullName".includes(extractor)) {
			// return the object
		} else if ("firstName".includes(extractor)) {
			result = result.firstName
		} else if ("lastName".includes(extractor)) {
			result = result.lastName
		} else if ("initials".includes(extractor)) {
			result = result.initials<commit_msg>Adding a PhoneRegex<commit_after>'use strict';

const _ = require('lodash')
const chrono = require('chrono-node')
const humanname = require('humanname')
const phoneUtil = require('google-libphonenumber').PhoneNumberUtil.getInstance()

var getTheRightData = function (node, {
	attr = null,
	extractor = null,
	filter = null,
	parser = null
} = {}) {

	var result = null

	if (attr) {
		result = node.attr(attr)
	} else if (extractor === "html") {
		result = node.html()
	} else {
		result = node.text()
	}

	if (extractor && extractor !== "html") {
		result = extractByExtractor(result, extractor)
	}

	if(_.isObject(result)){
		_.forOwn(result, function(value, key){
			result[key] = filterData(result[key], filter)
		})
	} else {
		result = filterData(result, filter)
	}

	if (parser) {
		result = parseData(result, parser)
	}

	return result
}

var parseData = function (data, regex) {
	var result = data
	if (regex) {
		try {
			var rgx = regex
			if (_.isString(regex)) {
				rgx = new RegExp(regex, 'gim')
			}
			result = data.match(rgx)[0]
		} catch (error) {
			// console.log("Regex error: ", error);
		}
	}

	return result
}

var filterData = function (data, filter) {
	var result = data
	if (["raw"].includes(filter)) {
		// let the raw data
	} else if (["trim"].includes(filter)) {
		result = result.trim()
	} else if (["lowercase", "lcase"].includes(filter)) {
		result = result.toLowerCase()
	} else if (["uppercase", "ucase"].includes(filter)) {
		result = result.toUpperCase()
	} else if (["capitalize", "cap"].includes(filter)) {
		result = _.startCase(result)
	} else if (["numbers"].includes(filter)) {
		result = result.replace(/\D/g, "")
	} else{
		// Default trim and set one spaces
		result = result.replace(/\s+/gm, " ").trim()
	}
	return result
}

var extractByExtractor = function (data, extractor, plural = false) {
	var result = data
	var emailRegex = /([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)/gmi
	var phoneRegex = /\+?\(?\d*\)? ?\(?\d+\)?\d*([\s./-]\d{2,})+/gmi

	if (["phone", "telephone"].includes(extractor)) {
		if (plural) {
			result = data.match(phoneRegex) || data
		} else {
			result = data.match(phoneRegex) !== null ? data.match(phoneRegex)[0] : data
		}
		// let countryCode = result.match(/([A-Z])+/)[0]
		// console.log("countryCode", countryCode);
		// if (countryCode) {
		// 	try {
		// 		result = phoneUtil.parse(result, "US") // with option, the country code number
		// 	} catch (e) {
		// 		//
		// 	}
		// }
	} else if (["email", "mail", "@"].includes(extractor)) {
		if (plural) {
			result = data.match(emailRegex) || data
		} else {
			result = data.match(emailRegex) !== null ? data.match(emailRegex)[0] : data
		}
	} else if (["date", "d"].includes(extractor)) {
		result = chrono.casual.parseDate(data).toString()
	} else if (["fullName", "firstName", "lastName", "initials", "suffix", "salutation"].includes(extractor)) {
		result = humanname.parse(data)
		if ("fullName".includes(extractor)) {
			// return the object
		} else if ("firstName".includes(extractor)) {
			result = result.firstName
		} else if ("lastName".includes(extractor)) {
			result = result.lastName
		} else if ("initials".includes(extractor)) {
			result = result.initials
{'fraction_non_alphanumeric': 0.1901850627891606, 'fraction_numerical': 0.003304692663582287, 'mean_word_length': 3.5374812593703147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1280888', 'n_tokens_mistral': 2317, 'n_tokens_neox': 2148, 'n_words': 481}