<commit_before>/**
 * 
 */
package ca.eandb.sortable.json;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

import ca.eandb.sortable.Product;
import ca.eandb.sortable.StringUtil;
import ca.eandb.sortable.TrieNode;

/**
 * An object that processes a set of product listings, matches them with a
 * product, and prints the results to a specified <code>PrintStream</code>.
 * The listings are given as JSON objects, with one listing per line.  See
 * comments in {@link #match(TrieNode, String, Set)} for details on how the
 * matching is performed.
 * @author <NAME>
 */
public final class JSONListingReader {
	
	/**
	 * The root <code>TrieNode</code> containing the set of strings that match
	 * the "manufacturer" field of a <code>Product</code>.
	 * 
	 * @see ca.eandb.sortable.ProductTrieBuilder
	 */
	private final TrieNode manufacturerTrie;
	
	/**
	 * The root <code>TrieNode</code> containing the set of strings that match
	 * the model name of a <code>Product</code>.  This could be the substrings
	 * of the "product_name" field, the "model" field, or the concatenation of
	 * the "family" and "model" fields (if the "family" field is present).  See
	 * {@link ca.eandb.sortable.ProductTrieBuilder} for details.
	 * 
	 * @see ca.eandb.sortable.ProductTrieBuilder
	 */
	private final TrieNode modelTrie;
	
	/**
	 * Creates a new <code>JSONListingReader</code>. 
	 * @param manufacturerTrie The root <code>TrieNode</code> containing the
	 * 		set of strings that match the "manufacturer" field of a
	 * 		<code>Product</code>.
	 * @param modelTrie The root <code>TrieNode</code> containing the set of
	 * 		strings that match the model name of a <code>Product</code>.
	 */
	public JSONListingReader(TrieNode manufacturerTrie, TrieNode modelTrie) {
		this.manufacturerTrie = manufacturerTrie;
		this.modelTrie = modelTrie;
	}

	/**
	 * Reads the JSON-formatted listings from the file (one listing per line),
	 * matches the listings to at most one product, and prints the results to
	 * the specified <code>PrintStream</code>. 
	 * @param in The <code>Reader</code> to read the listings from.
	 * @param out The <code>PrintStream</code> to write the results to.
	 * @throws IOException If an exception is thrown while reading from
	 * 		<code>in</code>.
	 * @throws ParseException If a line in the file does not represent a valid
	 * 		JSON object.
	 */
	public void read(Reader in, Writer out_) throws IOException, ParseException {
		
		// Map to store the matching listings corresponding to each product.
		Map<String, JSONArray> matches = new HashMap<String, JSONArray>();
		
		BufferedReader buf = in instanceof BufferedReader ? (BufferedReader) in : new BufferedReader(in);
		
		JSONParser parser = new JSONParser();
		
		while (true) {
			String line = buf.readLine();
			if (line == null) {
				break;
			}
			
			JSONObject json = (JSONObject) parser.parse(line);
		
			/* Find all the products with a matching manufacturer. */
			Set<Product> mp = match(manufacturerTrie, (String) json.get("manufacturer"), null);
			
			if (mp != null) { // don't continue if we didn't find any
				
				/* Eliminate everything after the word "for" (or its french
				 * translation "pour"), as everything that follows is most
				 * likely not the product itself.  For example:
				 * 
				 *   "Battery pack *for* Canon EOS 7D"
				 *   "Leather case *for* Nikon S6100"
				 * 
				 * We could make this more generic, for example by using an
				 * online translation service to translate detect the language
				 * and translate it to English, but this will do for
				 * demonstration purposes.
				 */
				String title = (String) json.get("title");
				title = title.replaceFirst(" for .*", "");
				title = title.replaceFirst(" pour .*", "");
				
				/* Match the listing title against the model name, only
				 * considering those products with the correct manufacturer.
				 */
				Product product = matchOne(modelTrie, title, mp);
			
				/* If we found a match, add some fields identifying the matched
				 * product to the listing JSON and reprint it.
				 */
				if (product != null) {
					JSONArray array = matches.get(product.getName());
					if (array == null) {
						array = new JSONArray();
						matches.put(product.getName(), array);
					}
					
					array.add(json);
				}
				
			}
			
		}
		
		
		
		// Wrap the output writer in a PrintWriter if it is not already a
		// PrintWriter.
		PrintWriter out = out_ instanceof PrintWriter ? (PrintWriter) out_ : new PrintWriter(out_);
		
		// print the list of matching listings.
		for (Map.Entry<String, JSONArray> e : matches.entrySet()) {
			JSONObject obj = new JSONObject();
			obj.put("product_name", e.getKey());
			obj.put("listings", e.getValue());
			obj.writeJSONString(out);
			out.println();
		}
		
	}

	/**
	 * Matches the specified string against the <code>Product</code>s stored in
	 * the specified trie.<commit_msg>Added summary printout at the end.<commit_after>/**
 * 
 */
package ca.eandb.sortable.json;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

import ca.eandb.sortable.Product;
import ca.eandb.sortable.StringUtil;
import ca.eandb.sortable.TrieNode;

/**
 * An object that processes a set of product listings, matches them with a
 * product, and prints the results to a specified <code>PrintStream</code>.
 * The listings are given as JSON objects, with one listing per line.  See
 * comments in {@link #match(TrieNode, String, Set)} for details on how the
 * matching is performed.
 * @author <NAME>
 */
public final class JSONListingReader {
	
	/**
	 * The root <code>TrieNode</code> containing the set of strings that match
	 * the "manufacturer" field of a <code>Product</code>.
	 * 
	 * @see ca.eandb.sortable.ProductTrieBuilder
	 */
	private final TrieNode manufacturerTrie;
	
	/**
	 * The root <code>TrieNode</code> containing the set of strings that match
	 * the model name of a <code>Product</code>.  This could be the substrings
	 * of the "product_name" field, the "model" field, or the concatenation of
	 * the "family" and "model" fields (if the "family" field is present).  See
	 * {@link ca.eandb.sortable.ProductTrieBuilder} for details.
	 * 
	 * @see ca.eandb.sortable.ProductTrieBuilder
	 */
	private final TrieNode modelTrie;
	
	/**
	 * Creates a new <code>JSONListingReader</code>. 
	 * @param manufacturerTrie The root <code>TrieNode</code> containing the
	 * 		set of strings that match the "manufacturer" field of a
	 * 		<code>Product</code>.
	 * @param modelTrie The root <code>TrieNode</code> containing the set of
	 * 		strings that match the model name of a <code>Product</code>.
	 */
	public JSONListingReader(TrieNode manufacturerTrie, TrieNode modelTrie) {
		this.manufacturerTrie = manufacturerTrie;
		this.modelTrie = modelTrie;
	}

	/**
	 * Reads the JSON-formatted listings from the file (one listing per line),
	 * matches the listings to at most one product, and prints the results to
	 * the specified <code>PrintStream</code>. 
	 * @param in The <code>Reader</code> to read the listings from.
	 * @param out The <code>PrintStream</code> to write the results to.
	 * @throws IOException If an exception is thrown while reading from
	 * 		<code>in</code>.
	 * @throws ParseException If a line in the file does not represent a valid
	 * 		JSON object.
	 */
	public void read(Reader in, Writer out_) throws IOException, ParseException {
		
		// Map to store the matching listings corresponding to each product.
		Map<String, JSONArray> matches = new HashMap<String, JSONArray>();
		int numListings = 0;	// total number of listings
		int numMatches = 0;		// number of listings with a unique product match
		
		BufferedReader buf = in instanceof BufferedReader ? (BufferedReader) in : new BufferedReader(in);
		
		JSONParser parser = new JSONParser();
		
		while (true) {
			String line = buf.readLine();
			if (line == null) {
				break;
			}
			
			JSONObject json = (JSONObject) parser.parse(line);
		
			/* Find all the products with a matching manufacturer. */
			Set<Product> mp = match(manufacturerTrie, (String) json.get("manufacturer"), null);
			
			if (mp != null) { // don't continue if we didn't find any
				
				/* Eliminate everything after the word "for" (or its french
				 * translation "pour"), as everything that follows is most
				 * likely not the product itself.  For example:
				 * 
				 *   "Battery pack *for* Canon EOS 7D"
				 *   "Leather case *for* Nikon S6100"
				 * 
				 * We could make this more generic, for example by using an
				 * online translation service to translate detect the language
				 * and translate it to English, but this will do for
				 * demonstration purposes.
				 */
				String title = (String) json.get("title");
				title = title.replaceFirst(" for .*", "");
				title = title.replaceFirst(" pour .*", "");
				
				/* Match the listing title against the model name, only
				 * considering those products with the correct manufacturer.
				 */
				Product product = matchOne(modelTrie, title, mp);
			
				/* If we found a match, add some fields identifying the matched
				 * product to the listing JSON and reprint it.
				 */
				if (product != null) {
					JSONArray array = matches.get(product.getName());
					if (array == null) {
						array = new JSONArray();
						matches.put(product.getName(), array);
					}
			
					numMatches++;
					array.add(json);
				}
				
			}
			
			numListings++;
		}
		
		
		
		// Wrap the output writer in a PrintWriter if it is not already a
		// PrintWriter.
		PrintWriter out = out_ instanceof PrintWriter ? (PrintWriter) out_ : new PrintWriter(out_);
		
		// print the list of matching listings.
		for (Map.Entry<String, JSONArray> e : matches.entrySet()) {
			JSONObject obj = new JSONObject();
			obj.put("product_name", e.getKey());
			obj.put("listings", e.getValue());
			obj.writeJSONString(out);
			out.println();
		}
		
		double pctMatch = 100.0 * (double) numMatches / (double) numListings;
		System.err.printf("Matched %d of %d listings (%4.1f%%).", numMatches, numListings, pctMatch);
		
	}

	/**
	 * Matches the specified string against the <code>Product</code>s stored in
	 * the specified trie.
{'fraction_non_alphanumeric': 0.10773153628995538, 'fraction_numerical': 0.0016391949731354156, 'mean_word_length': 3.3407114624505927, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3869451', 'n_tokens_mistral': 3735, 'n_tokens_neox': 3159, 'n_words': 1148}