<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  trapeza-import.py
#  
#  Copyright 2013-2014 <NAME> <<EMAIL>>
#  This file is available under the terms of the MIT License.
#

import trapeza, trapeza.match, tempfile, pickle, cStringIO, os, hashlib
from flask import Flask, request, session, make_response, render_template, abort
from operator import itemgetter, attrgetter
from itertools import groupby

# Accept three Source objects. Run match_incoming_records() on them.
# Render a template showing the user matches and allowing them to select a 
# matching record.
# On selection, run diff on the selected records (comparing the columns that are 
# mapped by the profile) and offer the user the chance to select one of the two versions 
# of each field or to enter their own data.
# On output, generate a copy of the input sheet with a unique ID column added specifying
# the matched record in the database. For each diffed column, if the user did not choose 
# to retain existing data, include the user-input value or the value from incoming.
# Maintain all other columns in the original input sheet unaltered for import.

app = Flask(__name__)
app.secret_key = "THIS_IS_A_TESTING_SECRET_KEY"

# Filters and functions for use within the match template
def flatten_record(record, headers):
    return "; ".join(["{0}: {1}".format(key, record.values[key]) for key in headers])

def get_identifier(element = "", incoming_line = None, record_id = "", key = ""):
    if not isinstance(incoming_line, str):
        line = str(incoming_line)
    else:
        line = incoming_line or ""
        
    return "{0}{1}".format(element, hashlib.md5(".".join([line, record_id or "", key or ""])).hexdigest())

def generate_header_mapping(profile):
    header_map = {}
    
    for mapping in profile.mappings:
        if not mapping.key in header_map:
            header_map[mapping.key] = mapping.master_key
    
    return header_map

def group_results(results): 
    list_results = []
    
    gb = groupby(results, lambda res: res.incoming.input_line)
    
    for (k, g) in gb:
        list_results.append((k, sorted(list(g), key=attrgetter("score"), reverse=True)))
    
    return sorted(list_results, key=itemgetter(0))

@app.route("/")
def start():
    return render_template("index.html")

@app.route("/run", methods=["POST"])
def step_one():

    # Load files. We assume, at present, that the files are CSV format.
    master = trapeza.load_source(request.files["master"], "csv")
    incoming = trapeza.load_source(request.files["incoming"], "csv")
    profile = trapeza.match.Profile(source = trapeza.load_source(request.files["profile"], "csv"))
    
    # Determine primary key and set in Master. We do not set the primary key on incoming or the output.
    # If the user feels like matching multiple input lines to the same master records, that's fine;
    # we don't want to throw an exception.
    primary_key = request.form["primary_key"]
    master.set_primary_key(primary_key)
    
    # Bring in the parameters. FIXME: will this crash if a non-integer parameter is submitted?
    cutoff = int(request.form["cutoff"]) or 0
    display_diff = "display_diff" in request.form 
    include_unmatched_records = "include_unmatched_records" in request.form 
    output_only_modified_entries = "output_only_modified_entries" in request.form 
    include_re_new_address_flag = "include_re_new_address_flag" in request.form
    
    # Perform the comparison.
    results = group_results(profile.compare_sources(master, incoming, cutoff))
    
    # Save the details of the operation
    operation = {   "master": master, "incoming": incoming, "profile": profile, 
                    "primary_key": primary_key, "results": results,
                    "cutoff": cutoff, 
                    "display_diff": display_diff, 
                    "include_unmatched_records": include_unmatched_records, 
                    "output_only_modified_entries": output_only_modified_entries,
                    "include_re_new_address_flag": include_re_new_address_flag }

    outfile = tempfile.NamedTemporaryFile(delete=False)
    session["file"] = outfile.name
    pickle.dump(operation, outfile, pickle.HIGHEST_PROTOCOL)
    outfile.close()
<commit_msg>Autodetect format of uploaded files.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  trapeza-import.py
#  
#  Copyright 2013-2014 <NAME> <<EMAIL>>
#  This file is available under the terms of the MIT License.
#

import trapeza, trapeza.match, tempfile, pickle, cStringIO, os, hashlib
from flask import Flask, request, session, make_response, render_template, abort
from operator import itemgetter, attrgetter
from itertools import groupby

# Accept three Source objects. Run match_incoming_records() on them.
# Render a template showing the user matches and allowing them to select a 
# matching record.
# On selection, run diff on the selected records (comparing the columns that are 
# mapped by the profile) and offer the user the chance to select one of the two versions 
# of each field or to enter their own data.
# On output, generate a copy of the input sheet with a unique ID column added specifying
# the matched record in the database. For each diffed column, if the user did not choose 
# to retain existing data, include the user-input value or the value from incoming.
# Maintain all other columns in the original input sheet unaltered for import.

app = Flask(__name__)
app.secret_key = "THIS_IS_A_TESTING_SECRET_KEY"

# Filters and functions for use within the match template
def flatten_record(record, headers):
    return "; ".join(["{0}: {1}".format(key, record.values[key]) for key in headers])

def get_identifier(element = "", incoming_line = None, record_id = "", key = ""):
    if not isinstance(incoming_line, str):
        line = str(incoming_line)
    else:
        line = incoming_line or ""
        
    return "{0}{1}".format(element, hashlib.md5(".".join([line, record_id or "", key or ""])).hexdigest())

def generate_header_mapping(profile):
    header_map = {}
    
    for mapping in profile.mappings:
        if not mapping.key in header_map:
            header_map[mapping.key] = mapping.master_key
    
    return header_map

def group_results(results): 
    list_results = []
    
    gb = groupby(results, lambda res: res.incoming.input_line)
    
    for (k, g) in gb:
        list_results.append((k, sorted(list(g), key=attrgetter("score"), reverse=True)))
    
    return sorted(list_results, key=itemgetter(0))

@app.route("/")
def start():
    return render_template("index.html")

@app.route("/run", methods=["POST"])
def step_one():

    # Load files. 
    master = trapeza.load_source(request.files["master"], trapeza.get_format(request.files["master"].filename, "csv"))
    incoming = trapeza.load_source(request.files["incoming"], trapeza.get_format(request.files["incoming"].filename, "csv"))
    profile = trapeza.match.Profile(source = trapeza.load_source(request.files["profile"], trapeza.get_format(request.files["profile"].filename, "csv")))
    
    # Determine primary key and set in Master. We do not set the primary key on incoming or the output.
    # If the user feels like matching multiple input lines to the same master records, that's fine;
    # we don't want to throw an exception.
    primary_key = request.form["primary_key"]
    master.set_primary_key(primary_key)
    
    # Bring in the parameters. FIXME: will this crash if a non-integer parameter is submitted?
    cutoff = int(request.form["cutoff"]) or 0
    display_diff = "display_diff" in request.form 
    include_unmatched_records = "include_unmatched_records" in request.form 
    output_only_modified_entries = "output_only_modified_entries" in request.form 
    include_re_new_address_flag = "include_re_new_address_flag" in request.form
    
    # Perform the comparison.
    results = group_results(profile.compare_sources(master, incoming, cutoff))
    
    # Save the details of the operation
    operation = {   "master": master, "incoming": incoming, "profile": profile, 
                    "primary_key": primary_key, "results": results,
                    "cutoff": cutoff, 
                    "display_diff": display_diff, 
                    "include_unmatched_records": include_unmatched_records, 
                    "output_only_modified_entries": output_only_modified_entries,
                    "include_re_new_address_flag": include_re_new_address_flag }

    outfile = tempfile.NamedTemporaryFile(delete=False)
    session["file"] = outfile.name
    pickle.dump(operation, outfile, pickle.HIGHEST_PROTOCOL)
    outfile.close()

{'fraction_non_alphanumeric': 0.09948066935949221, 'fraction_numerical': 0.003693017888055395, 'mean_word_length': 3.6742179072276158, 'pattern_counts': {'":': 20, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2310943', 'n_tokens_mistral': 2572, 'n_tokens_neox': 2507, 'n_words': 799}