<commit_before>import re
import logging
from furl import furl
import requests
import grequests
logger = logging.getLogger("headliner")


class MostPopular(object):
    NUM_CONCURRENT = 1
    SECTIONS = ["arts", "automobiles", "business", "dining", "education", "fashion", "garden", "health", "movies", "music", "politics", "science", "sports", "style", "technology", "television", "travel", "your-money"]

    POPULARITY = ["mostviewed", "mostshared", "mostemailed"]

    WWW_PATH_PATTERN = re.compile(r"/[0-9]{4}/[0-9]{2}/[0-9]{2}/([a-z-]+)/([a-z-]+/)?[a-z0-9-]+\.html")
    BLOG_PATH_PATTERN = re.compile(r"/[0-9]{4}/")

    MAPPINGS = {
            "arts": {
                "__PATH": {
                    "design": ["Design"],
                    "music": ["Music"],
                    "television": ["Television"],
                    "video-games": ["Video-Games"],
                    "__NONE": ["Arts"],
                },
                "__FACET": {
                    "weddings and engagements": ["Weddings"],
                },
            },

            "automobiles": {
                "__ALL": ["Autos"],
            },

            "business day": {
                "__ALL": ["Business"],
                "__PATH": {
                    "smallbusiness": ["Entrepreneur"],
                },
            },

            "dining & wine": {
                "__ALL": ["Cooking"],
            },

            "education": {
                "__ALL": ["Ideas"],
            },

            "fashion & style": {
                "__ALL": ["Fashion-Men", "Fashion-Women"],
                "__PATH": {
                    "weddings": ["Weddings"],
                },
            },

            "health": {
                "__ALL": ["Health-Men", "Health-Women"],
            },

            "home & garden": {
                "__ALL": ["Do-It-Yourself", "Home-Design"],
            },

            "movies": {
                "__ALL": ["Movies"],
            },

            "science": {
                "__ALL": ["Science"],
            },

            "sports": {
                "__ALL": ["Sports"],
                "__PATH": {
                    "baseball": ["Baseball"],
                    "basketball": ["Basketball"],
                    "football": ["Football"],
                    "golf": ["Golf"],
                    "hockey": ["Hockey"],
                    "soccer": ["Soccer"],
                    "tennis": ["Tennis"],
                },
                "__KEYWORD": {
                    "boxing": ["Boxing"],
                },
                "__COLUMN": {
                    "On Boxing": ["Boxing"],
                },
            },

            "style": {
                "__KEYWORD": {
                    "parenting": ["Parenting"]
                },
                "__FACET": {
                    "parenting": ["Parenting"]
                },
            },

            "technology": {
                "__ALL": ["Programming", "Technology"],
                "__PATH": {
                    "personaltech": ["Android", "Apple"],
                },
                "__FACET": {
                    "computer and video games": ["Video-Games"]
                },
            },

            "travel": {
                "__ALL": ["Travel"],
            },

            "u.s.": {
                "__ALL": ["Politics"],
            },

            "your money": {
                "__ALL": ["Business"],
            },
    }

    def __init__(self, config):
        self.config = config
        self._url_index = 0<commit_msg>Closes #3 Look for politics under /us/ for politics-specific articles Avoid classifying general us articles as politics.<commit_after>import re
import logging
from furl import furl
import requests
import grequests
logger = logging.getLogger("headliner")


class MostPopular(object):
    NUM_CONCURRENT = 1
    SECTIONS = ["arts", "automobiles", "business", "dining", "education", "fashion", "garden", "health", "movies", "music", "politics", "science", "sports", "style", "technology", "television", "travel", "your-money"]

    POPULARITY = ["mostviewed", "mostshared", "mostemailed"]

    WWW_PATH_PATTERN = re.compile(r"/[0-9]{4}/[0-9]{2}/[0-9]{2}/([a-z-]+)/([a-z-]+/)?[a-z0-9-]+\.html")
    BLOG_PATH_PATTERN = re.compile(r"/[0-9]{4}/")

    MAPPINGS = {
            "arts": {
                "__PATH": {
                    "design": ["Design"],
                    "music": ["Music"],
                    "television": ["Television"],
                    "video-games": ["Video-Games"],
                    "__NONE": ["Arts"],
                },
                "__FACET": {
                    "weddings and engagements": ["Weddings"],
                },
            },

            "automobiles": {
                "__ALL": ["Autos"],
            },

            "business day": {
                "__ALL": ["Business"],
                "__PATH": {
                    "smallbusiness": ["Entrepreneur"],
                },
            },

            "dining & wine": {
                "__ALL": ["Cooking"],
            },

            "education": {
                "__ALL": ["Ideas"],
            },

            "fashion & style": {
                "__ALL": ["Fashion-Men", "Fashion-Women"],
                "__PATH": {
                    "weddings": ["Weddings"],
                },
            },

            "health": {
                "__ALL": ["Health-Men", "Health-Women"],
            },

            "home & garden": {
                "__ALL": ["Do-It-Yourself", "Home-Design"],
            },

            "movies": {
                "__ALL": ["Movies"],
            },

            "science": {
                "__ALL": ["Science"],
            },

            "sports": {
                "__ALL": ["Sports"],
                "__PATH": {
                    "baseball": ["Baseball"],
                    "basketball": ["Basketball"],
                    "football": ["Football"],
                    "golf": ["Golf"],
                    "hockey": ["Hockey"],
                    "soccer": ["Soccer"],
                    "tennis": ["Tennis"],
                },
                "__KEYWORD": {
                    "boxing": ["Boxing"],
                },
                "__COLUMN": {
                    "On Boxing": ["Boxing"],
                },
            },

            "style": {
                "__KEYWORD": {
                    "parenting": ["Parenting"]
                },
                "__FACET": {
                    "parenting": ["Parenting"]
                },
            },

            "technology": {
                "__ALL": ["Programming", "Technology"],
                "__PATH": {
                    "personaltech": ["Android", "Apple"],
                },
                "__FACET": {
                    "computer and video games": ["Video-Games"]
                },
            },

            "travel": {
                "__ALL": ["Travel"],
            },

            "u.s.": {
                "__PATH": {
                    "politics": ["Politics"],
                },
            },

            "your money": {
                "__ALL": ["Business"],
            },
    }

    def __init__(self, config):
        self.config = config
        self._url_index = 0
{'fraction_non_alphanumeric': 0.17296331913724414, 'fraction_numerical': 0.004533589778815772, 'mean_word_length': 1.0788121073672188, 'pattern_counts': {'":': 125, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2474225', 'n_tokens_mistral': 2090, 'n_tokens_neox': 1890, 'n_words': 348}