<commit_before># -*- coding: utf-8 -*-

#   Copyright 2014 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Utilities for category information."""
import logging
from collections import defaultdict

from .vendor.titlecase import titlecase

from .db import output_row
from .db import select_all_categories
from .db import select_brand_categories
from .db import select_company_categories
from .norm import fix_bad_chars
from .norm import group_by_keys
from .norm import merge_dicts
from .norm import simplify_whitespace


BAD_CATEGORIES = ['Other']

CATEGORY_CORRECTIONS = {
    ('rankabrand', 'Luxury brands'): 'Luxury Apparel',
    ('rankabrand', 'Premium brands'): 'Premium Apparel',
}

log = logging.getLogger(__name__)


def fix_category(category, campaign_id):
    if (campaign_id, category) in CATEGORY_CORRECTIONS:
        category = CATEGORY_CORRECTIONS[(campaign_id, category)]

    category = fix_bad_chars(category)
    category = category.replace('&', ' and ')
    category = simplify_whitespace(category)
    category = titlecase(category)

    if category.endswith(' Brands'):
        category = category[:-7]

    if not category or category in BAD_CATEGORIES:
        return None
    else:
        return category


def get_category_map():
    key_to_category = {}

    for campaign_id, campaign_category in select_all_categories():
        category = fix_category(campaign_category, campaign_id)
        if category is None:
            continue

        key_to_category[(campaign_id, campaign_category)] = category

    return key_to_category


def output_category_rows(category_map):
    category_to_keys = defaultdict(set)
    for key, category in category_map.iteritems():
        category_to_keys[category].add(key)

    for category, keys in sorted(category_to_keys.items()):
        campaign_ids = sorted(set(key[0] for key in keys))
        log.info(u'{} ({})'.format(
            category, ', '.join(c for c in campaign_ids)))

        for campaign_id, campaign_category in keys:
            map_row = dict(campaign_id=campaign_id,
                           campaign_category=campaign_category,
                           category=category)

            output_row(map_row, 'campaign_category_map')


def get_company_categories(company, keys, category_map):
    category_rows = []

    for campaign_id, campaign_company in keys:
        category_rows.extend(_map_categories(
            select_company_categories(campaign_id, campaign_company),
            campaign_id, category_map))

    for cr_group in group_by_keys(
            category_rows, keyfunc=lambda cr: [cr['category']]):
        row = merge_dicts(cr_group)
        yield _fix_category_row(row, company)


def get_brand_categories(company, brand, keys, category_map):
    category_rows = []

    for campaign_id, campaign_company, campaign_brand in keys:
        category_rows.extend(_map_categories(
            select_brand_categories(
                campaign_id, campaign_company, campaign_brand),
            campaign_id, category_map))

    for cr_group in group_by_keys(
            category_rows, keyfunc=lambda cr: [cr['category']]):
        row = merge_dicts(cr_group)
        yield _fix_category_row(row, company, brand)


def _map_categories(rows, campaign_id, category_map):
    for row in rows:
        category = category_map.get((campaign_id, row['category']))
        if not category:  # bad category like "Other"
            continue
        row['category'] = category

        yield row

def _fix_category_row(row, company, brand=None):
    del row['campaign_id']

    row['company'] = company
    if brand is not None:
        row['brand'] = brand

    return row
<commit_msg>removed Industry Innovators and Commercial Products categories<commit_after># -*- coding: utf-8 -*-

#   Copyright 2014 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Utilities for category information."""
import logging
from collections import defaultdict

from .vendor.titlecase import titlecase

from .db import output_row
from .db import select_all_categories
from .db import select_brand_categories
from .db import select_company_categories
from .norm import fix_bad_chars
from .norm import group_by_keys
from .norm import merge_dicts
from .norm import simplify_whitespace


BAD_CATEGORIES = {
    'Commercial Products',
    'Industry Innovators',
    'Other',
}

# TODO: push this into rankabrand scraper
CATEGORY_CORRECTIONS = {
    ('rankabrand', 'Luxury brands'): 'Luxury Apparel',
    ('rankabrand', 'Premium brands'): 'Premium Apparel',
}

log = logging.getLogger(__name__)


def fix_category(category, campaign_id):
    if (campaign_id, category) in CATEGORY_CORRECTIONS:
        category = CATEGORY_CORRECTIONS[(campaign_id, category)]

    category = fix_bad_chars(category)
    category = category.replace('&', ' and ')
    category = simplify_whitespace(category)
    category = titlecase(category)

    if category.endswith(' Brands'):
        category = category[:-7]

    if not category or category in BAD_CATEGORIES:
        return None
    else:
        return category


def get_category_map():
    key_to_category = {}

    for campaign_id, campaign_category in select_all_categories():
        category = fix_category(campaign_category, campaign_id)
        if category is None:
            continue

        key_to_category[(campaign_id, campaign_category)] = category

    return key_to_category


def output_category_rows(category_map):
    category_to_keys = defaultdict(set)
    for key, category in category_map.iteritems():
        category_to_keys[category].add(key)

    for category, keys in sorted(category_to_keys.items()):
        campaign_ids = sorted(set(key[0] for key in keys))
        log.info(u'{} ({})'.format(
            category, ', '.join(c for c in campaign_ids)))

        for campaign_id, campaign_category in keys:
            map_row = dict(campaign_id=campaign_id,
                           campaign_category=campaign_category,
                           category=category)

            output_row(map_row, 'campaign_category_map')


def get_company_categories(company, keys, category_map):
    category_rows = []

    for campaign_id, campaign_company in keys:
        category_rows.extend(_map_categories(
            select_company_categories(campaign_id, campaign_company),
            campaign_id, category_map))

    for cr_group in group_by_keys(
            category_rows, keyfunc=lambda cr: [cr['category']]):
        row = merge_dicts(cr_group)
        yield _fix_category_row(row, company)


def get_brand_categories(company, brand, keys, category_map):
    category_rows = []

    for campaign_id, campaign_company, campaign_brand in keys:
        category_rows.extend(_map_categories(
            select_brand_categories(
                campaign_id, campaign_company, campaign_brand),
            campaign_id, category_map))

    for cr_group in group_by_keys(
            category_rows, keyfunc=lambda cr: [cr['category']]):
        row = merge_dicts(cr_group)
        yield _fix_category_row(row, company, brand)


def _map_categories(rows, campaign_id, category_map):
    for row in rows:
        category = category_map.get((campaign_id, row['category']))
        if not category:  # bad category like "Other"
            continue
        row['category'] = category

        yield row

def _fix_category_row(row, company, brand=None):
    del row['campaign_id']

    row['company'] = company
    if brand is not None:
        row['brand'] = brand

    return row

{'fraction_non_alphanumeric': 0.08547108208955224, 'fraction_numerical': 0.0025652985074626866, 'mean_word_length': 3.3100502512562815, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5747561', 'n_tokens_mistral': 2491, 'n_tokens_neox': 2392, 'n_words': 704}