<commit_before># -*- coding: utf-8 -*-
# This file is part of Shuup.
#
# Copyright (c) 2012-2020, Shoop Commerce Ltd. All rights reserved.
#
# This source code is licensed under the OSL-3.0 license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import unicode_literals, with_statement

import codecs
import os
import sys

import openpyxl
import six
import xlrd
from django.conf import settings
from django.utils.translation import ugettext_lazy as _

from shuup.utils.excs import Problem

if sys.version_info >= (3, 0):
    import csv
else:
    import unicodecsv as csv


class RowYielder(object):
    def __init__(self, sheet_or_data):
        self.sheet_or_data = sheet_or_data

    def transform_values(self, row):
        return row


class XLSRowYielder(RowYielder):
    def __iter__(self):
        for y in range(self.sheet_or_data.nrows):
            yield self.transform_values(self.sheet_or_data.row_values(y))


class XLSXRowYielder(RowYielder):
    def __iter__(self):
        for row in self.sheet_or_data.rows:
            yield self.transform_values([cell.value for cell in row])


class TransformedData(object):
    def __init__(self, mode, headers, rows, **meta):<commit_msg>Importer: force text for Excel cell values to consistent import<commit_after># -*- coding: utf-8 -*-
# This file is part of Shuup.
#
# Copyright (c) 2012-2020, Shoop Commerce Ltd. All rights reserved.
#
# This source code is licensed under the OSL-3.0 license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import unicode_literals, with_statement

import codecs
import os
import sys

import openpyxl
import six
import xlrd
from django.conf import settings
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _

from shuup.utils.excs import Problem

if sys.version_info >= (3, 0):
    import csv
else:
    import unicodecsv as csv


class RowYielder(object):
    def __init__(self, sheet_or_data):
        self.sheet_or_data = sheet_or_data

    def transform_values(self, row):
        return row


class XLSRowYielder(RowYielder):
    def __iter__(self):
        for y in range(self.sheet_or_data.nrows):
            yield self.transform_values(self.sheet_or_data.row_values(y))


class XLSXRowYielder(RowYielder):
    def __iter__(self):
        for row in self.sheet_or_data.rows:
            yield self.transform_values([(force_text(cell.value) if cell.value else None) for cell in row])


class TransformedData(object):
    def __init__(self, mode, headers, rows, **meta):
{'fraction_non_alphanumeric': 0.08015414258188824, 'fraction_numerical': 0.01001926782273603, 'mean_word_length': 3.798521256931608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4012264', 'n_tokens_mistral': 841, 'n_tokens_neox': 795, 'n_words': 266}