<commit_before>#!/usr/bin/env python
import argparse
import glob
import re
import os
import textract
import pyexcel as pe


EMAIL_REGEX = re.compile(r"(?i)([a-z0-9._-]{1,}@[a-z0-9-]{1,}\.[a-z]{2,})")


def main():
	parser = argparse.ArgumentParser(description='Search a directory containing documents for emails addresses')
	parser.add_argument('directory', help='Directory containing documents')
	parser.add_argument('-o', '--outfile', help='File to write found emails address to', default='emails_out.txt')
	args = parser.parse_args()

	found_emails = []
	unprocessed_files = []

	directories = get_files(args.directory)
	for doc in directories:
		try:
			extension = (doc.split('.')[-1]).lower()

			# Process xlsm documents
			if extension == 'xlsm':
				emails = search_xlsm(doc)

			# Process all other documents
			else:
				emails = search_docs(doc)

			# Unique emails
			if len(emails) > 0:
				print("{0} -> {1}".format(emails, doc))
				for email in emails:
					email = email.lower()
					if email in found_emails:
						continue
					else:
						found_emails.append(email)

		except Exception as error:
			print("[-] Unable to process: {0}".format(doc))
			unprocessed_files.append(doc)
			continue

	# Write emails to file
	if len(found_emails) > 0:
		display_emails(found_emails, args.outfile, unprocessed_files)
	else:
		print("[-] No emails found in '{0}'".format(args.directory))


def get_files(directory):
	directories = []

	for root, dirs, filenames in os.walk(directory):
		for filename in filenames:
			directories.append(os.path.join(root, filename))

	return directories


def search_xlsm(doc):
	emails = []

	doc_name = doc.split('/')[-1]
	new_doc = "{0}.xls".format(doc_name.split('.')[0])
	sheet = pe.get_book(file_name=doc)
	sheet.save_as("/tmp/{0}".format(new_doc))

	emails = search_docs("/tmp/{0}".format(new_doc))

	return emails


def search_docs(doc):<commit_msg>Added the ability to search text based files<commit_after>#!/usr/bin/env python
import argparse
import glob
import re
import os
import textract
import pyexcel as pe


EMAIL_REGEX = re.compile(r"(?i)([a-z0-9._-]{1,}@[a-z0-9-]{1,}\.[a-z]{2,})")
FLAT_FORMATS = ['txt', 'out', 'log', 'csv', 'ini']


def main():
	parser = argparse.ArgumentParser(description='Search a directory containing documents for emails addresses')
	parser.add_argument('directory', help='Directory containing documents')
	parser.add_argument('-o', '--outfile', help='File to write found emails address to', default='emails_out.txt')
	args = parser.parse_args()

	found_emails = []
	unprocessed_files = []

	directories = get_files(args.directory)
	for doc in directories:
		try:
			extension = (doc.split('.')[-1]).lower()

			# Process xlsm documents
			if extension == 'xlsm':
				emails = search_xlsm(doc)

			# Process text documents
			elif extension in FLAT_FORMATS:
				emails = search_text(doc)

			# Process all other documents
			else:
				emails = search_docs(doc)

			# Unique emails
			if len(emails) > 0:
				print("{0} -> {1}".format(emails, doc))
				for email in emails:
					email = email.lower()
					if email in found_emails:
						continue
					else:
						found_emails.append(email)

		except Exception as error:
			print("[-] Unable to process: {0}".format(doc))
			unprocessed_files.append(doc)
			continue

	# Write emails to file
	if len(found_emails) > 0:
		display_emails(found_emails, args.outfile, unprocessed_files)
	else:
		print("[-] No emails found in '{0}'".format(args.directory))


def get_files(directory):
	directories = []

	for root, dirs, filenames in os.walk(directory):
		for filename in filenames:
			directories.append(os.path.join(root, filename))

	return directories


def search_text(doc):
	emails = []

	text = open(doc, 'rb')
	for line in text:
		email = EMAIL_REGEX.search(line)
		if email:
			emails.append(email.group(0))

	return emails


def search_xlsm(doc):
	emails = []

	doc_name = doc.split('/')[-1]
	new_doc = "{0}.xls".format(doc_name.split('.')[0])
	sheet = pe.get_book(file_name=doc)
	sheet.save_as("/tmp/{0}".format(new_doc))

	emails = search_docs("/tmp/{0}".format(new_doc))

	return emails


def search_docs(doc):
{'fraction_non_alphanumeric': 0.149289670117987, 'fraction_numerical': 0.009390801830002408, 'mean_word_length': 4.141089108910891, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2792936', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1433, 'n_words': 344}