<commit_before># -*- coding: utf8 -*-

# Copyright 2015 <NAME>
#
# This file is part of AwesomeShop.
#
# AwesomeShop is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# AwesomeShop is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with AwesomeShop. If not, see <http://www.gnu.org/licenses/>.

import datetime

from .helpers import Setting
from .shipping.models import Carrier
from .shop.models.order import Order
from .shop.models.product import Product
from .shop.models.url import Url

###############################################################################
# Functions used to upgrade the database content
#
# These functions must not modify content which is already correct (so that it
# doesn't break if someone runs the upgrade functions on an already-upgraded
# database)


def add_product_cls():
    pass
    # Deprecated because it has been reverted
    # products = BaseProduct._get_collection()
    # products.update_many({'_cls': None},
    #                      {'$set': {'_cls': 'BaseProduct.Product'}})
    # urls = Url._get_collection()
    # urls.update_many(
    #         {'doc._cls': 'Product'},
    #         {'$set':{'doc._cls': 'BaseProduct.Product'}}
    #         )


def add_ondemand():
    products = Product._get_collection()
    products.update_many({'dem': None}, {'$set': {'dem': False}})


def add_creationdate():
    old_date = datetime.datetime(1970, 1, 1)
    products = Product._get_collection()
    products.update_many({'create': None}, {'$set': {'create': old_date}})


def merge_weights_and_costs():
    # Old format:
    #
    # weights: list of weights (integers)
    # costs : list of dicts associating countries IDs (strings) or
    #         countriesgroups IDs (strings)
    #         to dicts associating weights (strings) with costs (floats)
    #
    # New format:
    #
    # weights: list of lists, in which the first field is the weight (integer)
    #          and the second field is a dict associating countries IDs
    #          (strings) or countriesgroups IDs (strings) to costs (floats)
    carriers = Carrier._get_collection()
    for carrier in carriers.find(projection=('weights', 'costs')):
        if 'weights' in carrier:
            # If "weights" does not exist in the object, then it already used
            # the new format
            old_weights = carrier['weights']
            old_weights.sort()
            new_weights = {}
            for weight in old_weights:
                new_weights[str(weight)] = {}
            for country, costs in carrier['costs'].iteritems():
                for weight, cost in costs.iteritems():
                    if weight in new_weights:
                        # Lose data if the weight is not defined
                        new_weights[weight][country] = cost
            new_costs_as_list = []
            for weight in old_weights:
                new_costs_as_list.append({
                            'weight': weight,
                            'costs': new_weights[str(weight)]
                            })
            carriers.update_one(
                    {'_id': carrier['_id']},
                    {
                        '$set': {'costs': new_costs_as_list},
                        '$unset': {'weights': ''}
                        }
                    )


def reunite_products():
    products = Product._get_collection()
    products.update_many({},
                         {'$unset': {'_cls': ''}})
    urls = Url._get_collection()
    urls.update_many(
            {'doc._cls': 'BaseProduct.Product'},
            {'$set': {'doc._cls': 'Product'}}
            )


def remove_insuff_stock_from_orders():
    orders = Order._get_collection()
    for o in orders.find():
        newproducts = o['products'][:]
        for p in newproducts:
            if 'insuff_stock' in p:
                p.pop('insuff_stock')
        orders.find_one_and_update(
                    {'_id': o['_id']},
                    {'$set': {'products': newproducts}}
                    )

###############################################################################
# Ordered list of all upgrade functions
upgrades = [
    (add_product_cls, '2015: allow subproducts (deprecated)'),
    (add_ondemand, '2015: allow "on demand" products'),
    (add_creationdate, '2015: add creation date to products'),
    (
        merge_weights_and_costs,
        '18/01/2016: change how weights and costs are stored'
        ),
    (reunite_products, '09/02/2016: cancel the subproducts feature'),
    (
        remove_insuff_stock_from_orders,
        '16/04/2016: remove the "insufficient stock" info from orders'
        )
    ]


def upgrade():
    latest = len(upgrades)
    try:
        version = Setting.objects.get(name='db_upgrade_version')
    except:
        version = Setting(name='db_upgrade_version', value=0)
    if version.value < latest:
        print 'Upgrading database from version {} to version {}...'.format(
                                                                version.value,
                                                                latest
                                                                )
        for fct, desc in upgrades[version.value:]:
            print '>', desc
            fct()
        version.value = latest
        version.save()
        print 'Done upgrading!'<commit_msg>Split payment icon from payment description<commit_after># -*- coding: utf8 -*-

# Copyright 2015 <NAME>
#
# This file is part of AwesomeShop.
#
# AwesomeShop is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# AwesomeShop is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with AwesomeShop. If not, see <http://www.gnu.org/licenses/>.

import datetime
import re

from .helpers import Setting
from .shipping.models import Carrier
from .shop.models.order import Order
from .shop.models.product import Product
from .shop.models.url import Url

###############################################################################
# Functions used to upgrade the database content
#
# These functions must not modify content which is already correct (so that it
# doesn't break if someone runs the upgrade functions on an already-upgraded
# database)


def add_product_cls():
    pass
    # Deprecated because it has been reverted
    # products = BaseProduct._get_collection()
    # products.update_many({'_cls': None},
    #                      {'$set': {'_cls': 'BaseProduct.Product'}})
    # urls = Url._get_collection()
    # urls.update_many(
    #         {'doc._cls': 'Product'},
    #         {'$set':{'doc._cls': 'BaseProduct.Product'}}
    #         )


def add_ondemand():
    products = Product._get_collection()
    products.update_many({'dem': None}, {'$set': {'dem': False}})


def add_creationdate():
    old_date = datetime.datetime(1970, 1, 1)
    products = Product._get_collection()
    products.update_many({'create': None}, {'$set': {'create': old_date}})


def merge_weights_and_costs():
    # Old format:
    #
    # weights: list of weights (integers)
    # costs : list of dicts associating countries IDs (strings) or
    #         countriesgroups IDs (strings)
    #         to dicts associating weights (strings) with costs (floats)
    #
    # New format:
    #
    # weights: list of lists, in which the first field is the weight (integer)
    #          and the second field is a dict associating countries IDs
    #          (strings) or countriesgroups IDs (strings) to costs (floats)
    carriers = Carrier._get_collection()
    for carrier in carriers.find(projection=('weights', 'costs')):
        if 'weights' in carrier:
            # If "weights" does not exist in the object, then it already used
            # the new format
            old_weights = carrier['weights']
            old_weights.sort()
            new_weights = {}
            for weight in old_weights:
                new_weights[str(weight)] = {}
            for country, costs in carrier['costs'].iteritems():
                for weight, cost in costs.iteritems():
                    if weight in new_weights:
                        # Lose data if the weight is not defined
                        new_weights[weight][country] = cost
            new_costs_as_list = []
            for weight in old_weights:
                new_costs_as_list.append({
                            'weight': weight,
                            'costs': new_weights[str(weight)]
                            })
            carriers.update_one(
                    {'_id': carrier['_id']},
                    {
                        '$set': {'costs': new_costs_as_list},
                        '$unset': {'weights': ''}
                        }
                    )


def reunite_products():
    products = Product._get_collection()
    products.update_many({},
                         {'$unset': {'_cls': ''}})
    urls = Url._get_collection()
    urls.update_many(
            {'doc._cls': 'BaseProduct.Product'},
            {'$set': {'doc._cls': 'Product'}}
            )


def remove_insuff_stock_from_orders():
    orders = Order._get_collection()
    for o in orders.find():
        newproducts = o['products'][:]
        for p in newproducts:
            if 'insuff_stock' in p:
                p.pop('insuff_stock')
        orders.find_one_and_update(
                    {'_id': o['_id']},
                    {'$set': {'products': newproducts}}
                    )


def change_payment_description():
    orders = Order._get_collection()
    for o in orders.find({'p_desc': {'$regex': '^<i class'}}):
        print o
        m = re.match('<i class="fa fa-(.*)"></i> (.*)', o['p_desc'])
        orders.find_one_and_update(
                {'_id': o['_id']},
                {'$set': {'p_ico': m.group(1), 'p_desc': m.group(2)}}
                )

###############################################################################
# Ordered list of all upgrade functions
upgrades = [
    (add_product_cls, '2015: allow subproducts (deprecated)'),
    (add_ondemand, '2015: allow "on demand" products'),
    (add_creationdate, '2015: add creation date to products'),
    (
        merge_weights_and_costs,
        '18/01/2016: change how weights and costs are stored'
        ),
    (reunite_products, '09/02/2016: cancel the subproducts feature'),
    (
        remove_insuff_stock_from_orders,
        '16/04/2016: remove the "insufficient stock" info from orders'
        ),
    (
        change_payment_description,
        '16/04/2016: split the payment description and its icon'
        )
    ]


def upgrade():
    latest = len(upgrades)
    try:
        version = Setting.objects.get(name='db_upgrade_version')
    except:
        version = Setting(name='db_upgrade_version', value=0)
    if version.value < latest:
        print 'Upgrading database from version {} to version {}...'.format(
                                                                version.value,
                                                                latest
                                                                )
        for fct, desc in upgrades[version.value:]:
            print '>', desc
            fct()
        version.value = latest
        version.save()
        print 'Done upgrading!'
{'fraction_non_alphanumeric': 0.1309171597633136, 'fraction_numerical': 0.008875739644970414, 'mean_word_length': 2.0940757691329774, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4119997', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3143, 'n_words': 993}