<commit_before>"""
creditparser.py

Parses Santander UK credit card reports and produces a JSON file with the transactions.

Copyright (C) 2016 by <NAME>
Please the the file /LICENSE for the license details.
"""

import argparse
import re
from decimal import Decimal
import json


def parse_file(input_file):
	"""
	Parses a TXT credit report and returns a list of transactions with the following attributes:

	date - the transaction's date.
	card - the credit card associated with the transaction.
	type - the textual description of the transaction type.
	location - the place where the transaction occurred.
	description - the textual description of the transaction.
	amount - the transaction's amount (negative value for purchases, positive for money in).
	"""

	# Removes unnecessary clutter from a report's lines:
	lines = [re.sub(r"\t{2,}", r"\t", line.rstrip("\n")) for line in  open(input_file)]

	return [parse_transaction(line) for line in lines if is_transaction(line)]


# [date]	[card]	[description]	([amount_in]|[money_out])
TRANSACTION_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2})\t(\*\*\ \d{4})\t([^\t]+)\t((\d+\.\d{2})?)\t?((\d+\.\d{2})?)$")

def is_transaction(line):
	"""Returns a match if the supplied line is a transaction."""
	return TRANSACTION_PATTERN.match(line)


def parse_transaction(line):
	"""Parses a transaction line."""

	transaction = {}
	match = TRANSACTION_PATTERN.match(line)
	transaction["date"] = match.group(1)
	transaction["card"] = match.group(2)
	transaction.update(parse_description(match.group(3)))
	amount = Decimal(match.group(4))
	# All "money-out" transaction lines end with a non-tab. This is a bit of a hack :(.
	if not line.endswith("\t"):
		amount = amount * Decimal("-1")
	transaction["amount"] = amount

	return transaction


# PURCHASE - [type]	[location]	[description]
PURCHASE_PATTERN = re.compile(r"^PURCHASE\ -\ ([^\t]+)\t([^\t]+)\t(.*)$")

# PURCHASE [type] REFUND	[location]	[description]
REFUND_PATTERN = re.compile(r"^PURCHASE ([^\ ]+) REFUND\t([^\t]+)\t(.*)$")

def parse_description(description):
	"""Parses a transaction description."""

	# Replaces the description multiple spaces separators with tabs:
	description = re.sub(r"\s{2,}", r"\t", description).strip()
	match = PURCHASE_PATTERN.match(description)
	# Regular purchase:
	if match:
		return build_transaction_description(match, "PURCHASE")
	else:
		match = REFUND_PATTERN.match(description)
		# Refund:
		if match:
			return build_transaction_description(match, "REFUND")
		# Some other transaction (initial balance, payment, etc.):
		else:
			return {"type": description}


def build_transaction_description(match, type):
	"""Builds a transaction's description from a regex match."""
	return {
		"type": type, 
		"sub_type": match.group(1),
		"location": match.group(2),
		"description": match.group(3)
	}
<commit_msg>Added support for recurrent credit transactions parsing.<commit_after>"""
creditparser.py

Parses Santander UK credit card reports and produces a JSON file with the transactions.

Copyright (C) 2016 by <NAME>
Please the the file /LICENSE for the license details.
"""

import argparse
import re
from decimal import Decimal
import json


def parse_file(input_file):
	"""
	Parses a TXT credit report and returns a list of transactions with the following attributes:

	date - the transaction's date.
	card - the credit card associated with the transaction.
	type - the textual description of the transaction type.
	location - the place where the transaction occurred.
	description - the textual description of the transaction.
	amount - the transaction's amount (negative value for purchases, positive for money in).
	"""

	# Removes unnecessary clutter from a report's lines:
	lines = [re.sub(r"\t{2,}", r"\t", line.rstrip("\n")) for line in  open(input_file)]

	return [parse_transaction(line) for line in lines if is_transaction(line)]


# [date]	[card]	[description]	([amount_in]|[money_out])
TRANSACTION_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2})\t(\*\*\ \d{4})\t([^\t]+)\t((\d+\.\d{2})?)\t?((\d+\.\d{2})?)$")

def is_transaction(line):
	"""Returns a match if the supplied line is a transaction."""
	return TRANSACTION_PATTERN.match(line)


def parse_transaction(line):
	"""Parses a transaction line."""

	transaction = {}
	match = TRANSACTION_PATTERN.match(line)
	transaction["date"] = match.group(1)
	transaction["card"] = match.group(2)
	transaction.update(parse_description(match.group(3)))
	amount = Decimal(match.group(4))
	# All "money-out" transaction lines end with a non-tab. This is a bit of a hack :(.
	if not line.endswith("\t"):
		amount = amount * Decimal("-1")
	transaction["amount"] = amount

	return transaction


# PURCHASE - [type]	[location]	[description]
PURCHASE_PATTERN = re.compile(r"^PURCHASE\ -\ ([^\t]+)\t([^\t]+)\t(.*)$")

# PURCHASE [type] REFUND	[location]	[description]
REFUND_PATTERN = re.compile(r"^PURCHASE ([^\ ]+) REFUND\t([^\t]+)\t(.*)$")

# RECURRENT TRANSACTION	[location]	[description]
RECURRENT_PATTERN = re.compile(r"^([^\ ]+) TRANSACTION\t([^\t]+)\t(.*)$")

def parse_description(description):
	"""Parses a transaction description."""

	# Replaces the description multiple spaces separators with tabs:
	description = re.sub(r"\s{2,}", r"\t", description).strip()

	# Regular purchase:
	match = PURCHASE_PATTERN.match(description)
	if match:
		return build_transaction_description(match, "PURCHASE")

	# Refund:
	match = REFUND_PATTERN.match(description)
	if match:
		return build_transaction_description(match, "REFUND")

	# Recurrent:
	match = RECURRENT_PATTERN.match(description)
	if match:
		return build_transaction_description(match, "PURCHASE")

	# Some other transaction (initial balance, payment, etc.):
	return {"type": description}


def build_transaction_description(match, type):
	"""Builds a transaction's description from a regex match."""
	return {
		"type": type, 
		"sub_type": match.group(1),
		"location": match.group(2),
		"description": match.group(3)
	}

{'fraction_non_alphanumeric': 0.15051961112973516, 'fraction_numerical': 0.00670465973851827, 'mean_word_length': 4.937313432835821, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3914048', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1907, 'n_words': 559}