<commit_before>"""
Main module for running "new and improved" python package searches with better metrics.

"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from datetime import datetime, time as dt_time
from lxml.html import etree, HTMLParser
from tempfile import NamedTemporaryFile
from total_ordering import total_ordering
import dateutil.parser
import logging
import os
import re
import requests
import sh
import sys

logging.getLogger().setLevel(logging.DEBUG)

ARIA2_DOWNLOAD_COMPLETE_RGX = re.compile("Download complete:\\s+(?P<path>.*)$")

ARIA2C_OPTIONS = {"no-conf": True,
                  "timeout": 30,
                  "summary-interval": 2,
                  "max-connection-per-server": 1,
                  "connect-timeout": 30,
                  "max-tries": 16,
                  "max-file-not-found": 4,
                  "max-resume-failure-tries": 4,
                  "retry-wait": 8,
                  "deferred-input": True,
                  "max-concurrent-downloads": 33,
                  "enable-mmap": True,
                  "auto-file-renaming": False,
                  "allow-overwrite": True,
                  "async-dns": True,
                  "conditional-get": True,
                  "remote-time": True,
                  "http-accept-gzip": True,
                  "enable-http-pipelining": True,
                  "enable-http-keep-alive": False}

DOWNLOAD_COUNT_XPATH = "//ul[@class=\"nodot\"][li[strong[starts-with(text(), \"Downloads\")]]]/li/span/text()"
LAST_UPDATE_XPATH = "//table[@class=\"list\"]/tr[@class]/td[4]/text()"
PYPI_BASE_URL = "https://pypi.python.org/pypi"
SEARCH_RESULTS_XPATH = "//table[@class=\"list\"]/tr[@class][td]"


class NamedObject(object):
    """
    Abstract class to represent a named and/or downloadable object.
    """

    @property
    def name(self):
        """
        The name of this object.
        """
        raise NotImplementedError

    def to_aria2_input_entry(self):
        """
        Generate an input file entry such that aria2c can read and download this object.

        @return: The text to insert into the input file
        @rtype: str|unicode
        """
        raise NotImplementedError

    def apply_update(self, new_content):
        """
        Update this object with the information contained within C{new_content}.

        @param new_content: The content to update this object with
        @type new_content: str|unicode
        """
        raise NotImplementedError

    def run_backup_update(self):
        """
        Run a "backup" update (in case the primary update failed).
        """
        pass


@total_ordering
class PypiSearchResult(NamedObject):
    """
    A named object representing a search result.
    """

    __slots__ = ("link", "weight", "summary", "download_counts", "last_update")

    def __repr__(self):
        repr_fmt = "<{0.name}/{0.version}, weight={0.weight}, rate={0.download_rate:.2f}, age={0.age}>"
        return repr_fmt.format(self)

    def __hash__(self):
        return hash((self.link, self.weight))

    def __eq__(self, other):
        return (self.link, self.weight) == (other.link, other.weight)

    def __lt__(self, other):
        return (self.weight, self.link) < (other.weight, other.link)

    def __init__(self, link, weight, summary, download_counts=None, last_update=None):
        self.link = link
        self.weight = weight
        self.summary = summary
        self.download_counts = download_counts or []
        self.last_update = last_update or None

    @classmethod
    def from_dict(cls, data_dict):
        return PypiSearchResult(link=data_dict["link"],
                                weight=data_dict["weight"],
                                summary=data_dict["summary"],
                                download_counts=data_dict.get("download_counts"),
                                last_update=data_dict.get("last_update"))

    def to_dict(self):
        return {"link": self.link,
                "weight": self.weight,
                "summary": self.summary,
                "download_rate": self.download_rate,
                "last_update": self.last_update.isoformat() if self.last_update is not None else None}

    @property
    def version(self):
        """
        @return: The package version
        @rtype: basestring
        """
        return self.link.split("/")[-1]

    @property
    def name(self):
        """
        @return: The name of this named object
        @rtype: basestring
        """
        return self.link.split("/")[-2]

    @property
    def ftp_page_url(self):
        return "https://pypi.python.org/packages/source/{0[0]}/{0}/".format(self.name)

    @property
    def age(self):
        if self.last_update is None:
            return 3488
        return (datetime.now().date() - self.last_update.date()).days

    @property
    def scaled_age(self):
        """
        The age, scaled by rank, is 116.8 - 45506 / (397.7 + age).
        """
        return 116.8 - (45506.0 / (self.age + 397.7))
    
    @property
    def download_rate(self):
        return max(self.download_counts[1] / 7.0, self.download_counts[2] / 30.0)
    
    @property
    def scaled_download_rate(self):
        """
        The overall download rate, scaled by rank, is 98.21 - 742 / (6.404 + rate).
        """
        return 98.21 - (742 / (self.download_rate + 6.404))<commit_msg>Update pypi_pip_search.py with the features developed locally in pypi_pip_search2.py<commit_after>"""
Main module for running "new and improved" python package searches with better metrics.

"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from datetime import datetime, time as dt_time
from lxml.html import etree, HTMLParser
from tempfile import NamedTemporaryFile
from total_ordering import total_ordering
import dateutil.parser
import logging
import os
import re
import requests
import sh
import sys

logging.getLogger().setLevel(logging.DEBUG)

ARIA2_DOWNLOAD_COMPLETE_RGX = re.compile("Download complete:\\s+(?P<path>.*)$")

ARIA2C_OPTIONS = {"no-conf": True,
                  "timeout": 30,
                  "summary-interval": 2,
                  "max-connection-per-server": 1,
                  "connect-timeout": 30,
                  "max-tries": 16,
                  "max-file-not-found": 4,
                  "max-resume-failure-tries": 4,
                  "retry-wait": 8,
                  "deferred-input": True,
                  "max-concurrent-downloads": 33,
                  "enable-mmap": True,
                  "auto-file-renaming": False,
                  "allow-overwrite": True,
                  "async-dns": True,
                  "conditional-get": True,
                  "remote-time": True,
                  "http-accept-gzip": True,
                  "enable-http-pipelining": True,
                  "enable-http-keep-alive": False}

DOWNLOAD_COUNT_XPATH = "//ul[@class=\"nodot\"][li[strong[starts-with(text(), \"Downloads\")]]]/li/span/text()"
LAST_UPDATE_XPATH = "//table[@class=\"list\"]/tr[@class]/td[4]/text()"
PYPI_BASE_URL = "https://pypi.python.org/pypi"
SEARCH_RESULTS_XPATH = "//table[@class=\"list\"]/tr[@class][td]"


class NamedObject(object):
    """
    Abstract class to represent a named and/or downloadable object.
    """

    @property
    def name(self):
        """
        The name of this object.
        """
        raise NotImplementedError

    def to_aria2_input_entry(self):
        """
        Generate an input file entry such that aria2c can read and download this object.

        @return: The text to insert into the input file
        @rtype: str|unicode
        """
        raise NotImplementedError

    def apply_update(self, new_content):
        """
        Update this object with the information contained within C{new_content}.

        @param new_content: The content to update this object with
        @type new_content: str|unicode
        """
        raise NotImplementedError

    def run_backup_update(self):
        """
        Run a "backup" update (in case the primary update failed).
        """
        pass


@total_ordering
class PypiSearchResult(NamedObject):
    """
    A named object representing a search result.
    """

    __slots__ = ("link", "weight", "summary", "download_counts", "last_update")

    def __repr__(self):
        repr_fmt = "<{0.name}/{0.version}, weight={0.weight}, rate={0.download_rate:.2f}, age={0.age}>"
        return repr_fmt.format(self)

    def __hash__(self):
        return hash((self.link, self.weight))

    def __eq__(self, other):
        return (self.link, self.weight) == (other.link, other.weight)

    def __lt__(self, other):
        return (self.weight, self.link) < (other.weight, other.link)

    def __init__(self, link, weight, summary, download_counts=None, last_update=None):
        self.link = link
        self.weight = weight
        self.summary = summary
        self.download_counts = download_counts or []
        self.last_update = last_update or None

    @classmethod
    def from_dict(cls, data_dict):
        return PypiSearchResult(link=data_dict["link"],
                                weight=data_dict["weight"],
                                summary=data_dict["summary"],
                                download_counts=data_dict.get("download_counts"),
                                last_update=data_dict.get("last_update"))

    def to_dict(self):
        return {"link": self.link,
                "weight": self.weight,
                "summary": self.summary,
                "download_rate": self.download_rate,
                "last_update": self.last_update.isoformat() if self.last_update is not None else None}

    @property
    def version(self):
        """
        @return: The package version
        @rtype: basestring
        """
        return self.link.split("/")[-1]

    @property
    def name(self):
        """
        @return: The name of this named object
        @rtype: basestring
        """
        return self.link.split("/")[-2]

    @property
    def ftp_page_url(self):
        return "https://pypi.python.org/packages/source/{0[0]}/{0}/".format(self.name)

    @property
    def age(self):
        if self.last_update is None:
            return 3488
        return (datetime.now().date() - self.last_update.date()).days

    @property
    def scaled_age(self):
        """
        The age, scaled by rank, is 116.8 - 45506 / (397.7 + age).
        """
        return 116.8 - (45506.0 / (self.age + 397.7))

    @property
    def download_rate(self):
        return max(self.download_counts[1] / 7.0, self.download_counts[2] / 30.0)

    @property
    def scaled_download_rate(self):
        """
        The overall download rate, scaled by rank, is 98.21 - 742 / (6.404 + rate).
        """
        return 98.21 - (742 / (self.download_rate + 6.404))
{'fraction_non_alphanumeric': 0.12450719721279913, 'fraction_numerical': 0.0164114788667828, 'mean_word_length': 2.126397248495271, 'pattern_counts': {'":': 50, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2179828', 'n_tokens_mistral': 3386, 'n_tokens_neox': 3137, 'n_words': 767}