<commit_before># too vague to be useful
BAD_CATEGORIES = {
    'Commercial Products',
    'Industry Innovators',
    'Other',
}

# not useful
BAD_SUFFIXES = [
    ' Brands',
    ' Products',
]

# custom corrections to categories (after normalization)
CATEGORY_ALIASES = {
    'Accessories': 'Fashion Accessories',
    'Bananas and Pineapple': 'Bananas and Pineapples',
    'Food and Beverage': 'Food and Beverages',
    'Food and Drink': 'Food and Beverages',
    'Fun and Games': 'Toys and Games',
    'Misc. Food': 'Food',
    'Occ. Safety and Health Consulting': (
        'Occupational Safety and Health Consulting')
}

# Categories that we can't just split on "and"; e.g.
# Home and Office Furniture, Skin and Hair Care
CATEGORY_SPLITS = {
    'IT Software and Services/Web Design': {
        'IT Software',
        'IT Services',
        'Web Design',
    },
    'Renewable Energy Generation and Installation': {
        'Renewable Energy Generation',
        'Renewable Energy Installation',
    },
    'Sport and Outdoor - Clothing and Shoes': {
        'Sport Clothing',
        'Sport Shoes',
        'Outdoor Clothing',
        'Outdoor Shoes',
    },
    'Home and Office Furniture': {
        'Home Furniture',
        'Office Furniture',
    },
    'Education and Training Services': {
        'Education Services',
        'Training Services',
    },
    'Sports Equipment, Toys and Accessories': {
        'Sports Equipment',
        'Toys',
        'Sports Accessories',
    },
    'Skin and Hair Care': {
        'Skin Care',
        'Hair Care',
    },
    'Sport and Outdoor Clothing': {
        'Sport Clothing',
        'Outdoor Clothing',
    },
    'Surf, Beach and Swimwear': {
        'Surf Wear',
        'Beachwear',
        'Swimwear',
    },
    'Film and Music Production': {
        'Film Production',
        'Music Production',
    },
    'Baby and Children Clothing': {
        'Baby Clothing',
        'Children Clothing',
    },
    'Catering and Meeting/Event Management': {
        'Catering',
        'Meeting Management',
        'Event Management',
    },
    'Waste Reduction Consulting and Services': {
        'Waste Reduction Consulting',
        'Waste Reduction Services',
    },
    'Automotive Sales and Repair': {
        'Automotive Sales',
        'Automotive Repair',
    },
    'Web Design and Development': {
        'Web Design',
        'Web Development',
    },<commit_msg>two more custom category splits<commit_after># too vague to be useful
BAD_CATEGORIES = {
    'Commercial Products',
    'Industry Innovators',
    'Other',
}

# not useful
BAD_SUFFIXES = [
    ' Brands',
    ' Products',
]

# custom corrections to categories (after normalization)
CATEGORY_ALIASES = {
    'Accessories': 'Fashion Accessories',
    'Bananas and Pineapple': 'Bananas and Pineapples',
    'Food and Beverage': 'Food and Beverages',
    'Food and Drink': 'Food and Beverages',
    'Fun and Games': 'Toys and Games',
    'Misc. Food': 'Food',
    'Occ. Safety and Health Consulting': (
        'Occupational Safety and Health Consulting')
}

# Categories that we can't just split on "and"; e.g.
# Home and Office Furniture, Skin and Hair Care
CATEGORY_SPLITS = {
    'IT Software and Services/Web Design': {
        'IT Software',
        'IT Services',
        'Web Design',
    },
    'Renewable Energy Generation and Installation': {
        'Renewable Energy Generation',
        'Renewable Energy Installation',
    },
    'Sport and Outdoor - Clothing and Shoes': {
        'Sport Clothing',
        'Sport Shoes',
        'Outdoor Clothing',
        'Outdoor Shoes',
    },
    'Home and Office Furniture': {
        'Home Furniture',
        'Office Furniture',
    },
    'Education and Training Services': {
        'Education Services',
        'Training Services',
    },
    'Sports Equipment, Toys and Accessories': {
        'Sports Equipment',
        'Toys',
        'Sports Accessories',
    },
    'Skin and Hair Care': {
        'Skin Care',
        'Hair Care',
    },
    'Sport and Outdoor Clothing': {
        'Sport Clothing',
        'Outdoor Clothing',
    },
    'Surf, Beach and Swimwear': {
        'Surf Wear',
        'Beachwear',
        'Swimwear',
    },
    'Film and Music Production': {
        'Film Production',
        'Music Production',
    },
    'Baby and Children Clothing': {
        'Baby Clothing',
        'Children Clothing',
    },
    'Catering and Meeting/Event Management': {
        'Catering',
        'Meeting Management',
        'Event Management',
    },
    'Waste Reduction Consulting and Services': {
        'Waste Reduction Consulting',
        'Waste Reduction Services',
    },
    'Automotive Sales and Repair': {
        'Automotive Sales',
        'Automotive Repair',
    },
    'Web Design and Development': {
        'Web Design',
        'Web Development',
    },
    'Pet Toys, Bedding and Apparel': {
        'Pet Toys',
        'Pet Bedding',
        'Pet Apparel',
    },
    'Housewares, Home Furnishings, and Accessories': {
        'Housewares',
        'Home Furnishings',
        'Home Accessories',
    },
{'fraction_non_alphanumeric': 0.11978963770938839, 'fraction_numerical': 0.0, 'mean_word_length': 2.2541191381495564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1033515', 'n_tokens_mistral': 1604, 'n_tokens_neox': 1510, 'n_words': 465}