<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import codecs

products = []
listings = {}
results = []


def read_source():
    '''Read source files.

    Returns:
        products (list): list of product dictionaries.
        listings (dict): listings dictionary indexed by manufacturer.
    '''
    with codecs.open('products.txt', encoding='ascii') as products_file:
        for product in products_file:
            products.append(json.loads(product))

    with codecs.open('listings.txt', encoding='utf-8') as listings_file:
        for _listing in listings_file:
            listing = json.loads(_listing)
            manufacturer = listing['manufacturer'].lower() or u'brandless'

            if manufacturer in listings:
                listings[manufacturer].append(listing)
            else:
                listings[manufacturer] = [listing]

    return products, listings


def listing_is_product(listing, product):
    '''Simple Matching Algorithm.

    Params:
        listing (dict): listing from listings.txt
        product (dict): product from products.txt

    Returns:
        bool: True if listing and product match, False otherwise.
    '''
    title = listing['title'].lower().replace('-', '')
    family = product.get('family', '').lower().replace('-', '')
    model = product['model'].lower().replace('-', '')

    if model in title:
        if family and family in title:
            return True
    return False


def find_results():
    '''
    TODO Write docstring
    '''
    _results = {}

    for product in products:
        _listings = listings.get(product['manufacturer'].lower(), [])
        for _listing in _listings:
            if listing_is_product(_listing, product):
                if _results.get(product['product_name']):
                    _results[product['product_name']].append(_listing)
                else:
                    _results[product['product_name']] = [_listing]

    for product_name in _results:
        results.append(
            {'product_name': product_name, 'listings': _results[product_name]}
        )


if __name__ == '__main__':
    read_source()
    find_results()
    with open('results.json', 'w') as results_file:
        json.dump(results, results_file, indent=4)<commit_msg>- Add small clean function. - Refactor read_source. - TODO: better model number matching.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import codecs

products = []
listings = {}
results = []


def listing_is_product(listing, product):
    '''Simple Matching Algorithm.

    Params:
        listing (dict): listing from listings.txt
        product (dict): product from products.txt

    Returns:
        bool: True if listing and product match, False otherwise.
    '''
    def clean(input):
        # TODO handle spaces in models: e.g: SP-600UZ vs SP-600 UZ
        return input.lower().replace('-', '')

    title = clean(listing['title'])
    family = clean(product.get('family', ''))
    model = clean(product['model'])

    if model in title:
        if family and family in title:
            return True
    return False


def find_results():
    '''
    TODO Write docstring
    '''
    _results = {}

    for product in products:
        _listings = listings.get(product['manufacturer'].lower(), [])
        for _listing in _listings:
            if listing_is_product(_listing, product):
                if _results.get(product['product_name']):
                    _results[product['product_name']].append(_listing)
                else:
                    _results[product['product_name']] = [_listing]

    for product_name in _results:
        results.append(
            {'product_name': product_name, 'listings': _results[product_name]}
        )


def read_source():
    '''Read source files.

    Returns:
        products (list): list of product dictionaries.
        listings (dict): listings dictionary indexed by manufacturer.
    '''
    with codecs.open('products.txt', encoding='ascii') as products_file:
        for product in products_file:
            products.append(json.loads(product))

    with codecs.open('listings.txt', encoding='utf-8') as listings_file:
        for _listing in listings_file:
            listing = json.loads(_listing)
            manufacturer = listing['manufacturer'].lower() or u'brandless'

            if manufacturer in listings:
                listings[manufacturer].append(listing)
            else:
                listings[manufacturer] = [listing]

    return products, listings

if __name__ == '__main__':
    read_source()
    find_results()
    with open('results.json', 'w') as results_file:
        json.dump(results, results_file, indent=4)
{'fraction_non_alphanumeric': 0.11924464247825164, 'fraction_numerical': 0.002546148949713558, 'mean_word_length': 2.50222882615156, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5340775', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1251, 'n_words': 343}