<commit_before>import os
import csv
from oscar.services import import_module


catalogue_exception = import_module('catalogue_import.exceptions', ['CatalogueImportException'])
product_models = import_module('product.models', ['ItemClass', 'Item', 'AttributeType', 'ItemAttributeValue', 'Option'])
stock_models = import_module('stock.models', ['Partner', 'StockRecord'])

class CatalogueImport(object):
    u"""A catalogue importer object"""
    
    csv_types = {
                 'simple': {'fields': [
                               [product_models.Item, 'upc'],
                               [product_models.Item, 'title'],
                               [product_models.Item, 'description'],
                               [product_models.ItemClass, 'name'],
                               [stock_models.StockRecord, 'partner'],
                               [stock_models.StockRecord, 'partner_reference'],
                               [stock_models.StockRecord, 'price_excl_tax'],
                               [stock_models.StockRecord, 'num_in_stock'],
                           ],
                           'copy_fields': [
                               [[stock_models.StockRecord, 'partner'], [stock_models.Partner, 'name']],
                           ]}
                 }
    flush = False
    item = []
    
    def handle(self):
        u"""Handles the actual import process"""
        if self.file is None:
            raise catalogue_exception.CatalogueImportException("You need to pass a file argument")
        self.catalogue_import_file = CatalogueImportFile()
        self.catalogue_import_file.file = self.file
        self.test_file()
        if self.flush is True:
            self.flush()
        self.csv_content = self.load_csv()
        self.iterate()
        for line in self.item:
            print line
    
    def test_file(self):
        u"""Run various tests on the file before import can happen"""
        self.catalogue_import_file.file_exists()
        self.catalogue_import_file.is_file()
        self.catalogue_import_file.file_is_readable()
        
    def flush(self):
        u"""Flush out product and stock models"""
        product_models.ItemClass.objects.all().delete()
        product_models.Item.objects.all().delete()
        product_models.AttributeType.objects.all().delete()
        product_models.ItemAttributeValue.objects.all().delete()
        product_models.Option.objects.all().delete()
        product_models.Partner.objects.all().delete()
        product_models.StockRecord.objects.all().delete()
    def load_csv(self):
        u"""Load the CSV content"""
        csv_reader = CatalogueCsvReader()
        return csv_reader.get_csv_contents(self.file)
<commit_msg>- Mapping of a CSV field to a model object and string column name - Created copyfields to copy the content of a CSV column to a second location (matches based on origin model object and string field name list) - Merge functionality to merge fields and copyfields in to a list of lists, mapping object, field and value - Created a proper row iterator function instead of grabbing the CSV content in one go<commit_after>import os
import csv
from oscar.services import import_module


catalogue_exception = import_module('catalogue_import.exceptions', ['CatalogueImportException'])
product_models = import_module('product.models', ['ItemClass', 'Item', 'AttributeType', 'ItemAttributeValue', 'Option'])
stock_models = import_module('stock.models', ['Partner', 'StockRecord'])

class CatalogueImport(object):
    u"""A catalogue importer object"""
    
    csv_types = {
                 'simple': {'fields': [
                               [product_models.Item, 'upc'],
                               [product_models.Item, 'title'],
                               [product_models.Item, 'description'],
                               [product_models.ItemClass, 'name'],
                               [stock_models.StockRecord, 'partner'],
                               [stock_models.StockRecord, 'partner_reference'],
                               [stock_models.StockRecord, 'price_excl_tax'],
                               [stock_models.StockRecord, 'num_in_stock'],
                           ],
                           'copy_fields': [
                               [[stock_models.StockRecord, 'partner'], [stock_models.Partner, 'name']],
                           ]}
                 }
    flush = False
    item = []
    
    def handle(self):
        u"""Handles the actual import process"""
        if self.file is None:
            raise catalogue_exception.CatalogueImportException("You need to pass a file argument")
        self.catalogue_import_file = CatalogueImportFile()
        self.catalogue_import_file.file = self.file
        self.test_file()
        if self.flush is True:
            self.flush()
        self.csv_content = self.load_csv()
        self.iterate()
        for line in self.item:
            print line
    
    def test_file(self):
        u"""Run various tests on the file before import can happen"""
        self.catalogue_import_file.file_exists()
        self.catalogue_import_file.is_file()
        self.catalogue_import_file.file_is_readable()
        
    def flush(self):
        u"""Flush out product and stock models"""
        product_models.ItemClass.objects.all().delete()
        product_models.Item.objects.all().delete()
        product_models.AttributeType.objects.all().delete()
        product_models.ItemAttributeValue.objects.all().delete()
        product_models.Option.objects.all().delete()
        product_models.Partner.objects.all().delete()
        product_models.StockRecord.objects.all().delete()
        
    def load_csv(self):
        u"""Load the CSV content"""
        csv_reader = CatalogueCsvReader()
        return csv_reader.get_csv_contents(self.file)

{'fraction_non_alphanumeric': 0.10357751277683135, 'fraction_numerical': 0.0, 'mean_word_length': 2.1295309168443497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6063119', 'n_tokens_mistral': 1521, 'n_tokens_neox': 1485, 'n_words': 337}