<commit_before>import suggestive.mstat as mstat
from suggestive.model import Album, Track, Scrobble, LastfmTrackInfo

import logging
import re
import sys
from sqlalchemy import func, Integer, distinct
from itertools import repeat
from collections import defaultdict
from operator import itemgetter
from datetime import datetime


logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())


class Suggestion(object):

    def __init__(self, album, order=None):
        self.album = album
        self.order = order
        #self.loved = [
        #    track for track in album.tracks
        #    if track.lastfm_info and track.lastfm_info.loved
        #]


def choose(N, k):
    if (k > N) or (N < 0) or (k < 0):
        return 0
    N, k = (int(N), int(k))
    top = N
    val = 1
    while (top > (N - k)):
        val *= top
        top -= 1
    n = 1
    while (n < k + 1):
        val /= n
        n += 1
    return val


class OrderDecorator(object):

    TRUTHY = (True, 'True', 'TRUE', 'true', 1, 'yes')
    NONE = (None, 'None', 'null', '')

    def __repr__(self):
        return '<{}()>'.format(self.__class__.__name__)

    def order(self, albums, session, mpd):
        raise NotImplementedError


class BaseOrder(OrderDecorator):

    """Initialize all albums with unity order"""

    def order(self, albums, session, mpd):
        return defaultdict(
            lambda: 1.0,
            zip(session.query(Album).all(), repeat(1.0))
        )


class AlbumFilter(OrderDecorator):
    def __init__(self, *name_pieces):
        name = ' '.join(name_pieces)
        self.name = name
        self.name_rgx = re.compile(name, re.I)

    def __repr__(self):
        return '<AlbumFilter({})>'.format(self.name)

    def order(self, albums, session, mpd):
        return {
            album: order for album, order in albums.items()
            if re.search(self.name_rgx, album.name) is not None
        }


class ArtistFilter(OrderDecorator):
    def __init__(self, *name_pieces):
        name = ' '.join(name_pieces)
        self.name = name
        self.name_rgx = re.compile(name, re.I)

    def __repr__(self):
        return '<ArtistFilter({})>'.format(self.name)

    def order(self, albums, session, mpd):
        return {
            album: order for album, order in albums.items()
            if re.search(self.name_rgx, album.artist.name) is not None
        }


class SortOrder(OrderDecorator):

    """Sort by 'Artist - Album'"""

    def __init__(self, ignore_artist_the=True, reverse=False):
        self.ignore_artist_the = bool(ignore_artist_the)
        self.reverse = (not bool(reverse))

    def _format(self, album):
        artist = album.artist.name
        if self.ignore_artist_the and artist.lower().startswith('the '):
            artist = artist[4:] + ', ' + artist[:3]

        return '{} - {}'.format(artist, album.name)

    def order(self, albums, session, mpd):
        sorted_albums = sorted(albums, key=self._format, reverse=self.reverse)
        return {album: i for i, album in enumerate(sorted_albums, 1)}


class ModifiedOrder(OrderDecorator):

    """Sort by modified date"""

    FMT = '%Y-%m-%dT%H:%M:%SZ'

    def __init__(self, reverse=False):
        self.reverse = bool(reverse)

    @classmethod
    def get_date(cls, album, mpd):
        track_info = mpd.search('album', album.name)
        if not track_info:
            return None

        dates = (datetime.strptime(info['last-modified'], cls.FMT)
                 for info in track_info)
        return max(dates)

    def order(self, albums, session, mpd):
        sorted_albums = sorted(
            albums,
            key=lambda album: self.get_date(album, mpd),
            reverse=self.reverse)
        return {album: i for i, album in enumerate(sorted_albums, 1)}


class BannedOrder(OrderDecorator):

    """Remove or demote albums with banned tracks"""

    def __init__(self, remove_banned=True):
        self.remove = remove_banned in self.TRUTHY

    def __repr__(self):
        return '<BannedOrder({})>'.format(self.remove)

    def order(self, albums, session, mpd):
        results = session.query(Album).\
            join(Track).\
            outerjoin(LastfmTrackInfo).\
            add_columns(func.count(Track.id),
                        func.sum(LastfmTrackInfo.banned, type_=Integer)).\
            group_by(Album.id).\
            all()

        neworder = defaultdict(lambda: 1.0, albums.items())

        for album, n_tracks, n_banned in results:
            if album not in neworder or n_tracks == 0:
                continue

            if n_banned and self.remove:
                del neworder[album]
            elif n_banned:
                # Banned albums are equally worthless
                neworder[album] *= 0.0

        return neworder


class FractionLovedOrder(OrderDecorator):

    """Order by fraction of tracks loved"""

    def __init__(self, reverse=False, penalize_unloved=False, **kwArgs):
        super(FractionLovedOrder, self).__init__()
        self.reverse = bool(reverse)

        maximum = kwArgs.pop('max', None)
        minimum = kwArgs.pop('min', None)

        if kwArgs:
            arg = list(kwArgs.keys())[0]
            raise TypeError("FractionLovedOrder got an unexpected keyword "
                            "argument '{}'".format(arg))

        self.penalize = penalize_unloved

        if minimum in self.NONE:
            minimum = 0
        if maximum in self.NONE:
            maximum = 1

        try:
            minimum, maximum = float(minimum), float(maximum)<commit_msg>Added docstrings<commit_after>import suggestive.mstat as mstat
from suggestive.model import Album, Track, Scrobble, LastfmTrackInfo

import logging
import re
import sys
from sqlalchemy import func, Integer, distinct
from itertools import repeat
from collections import defaultdict
from operator import itemgetter
from datetime import datetime


logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())


class Suggestion(object):

    def __init__(self, album, order=None):
        self.album = album
        self.order = order


class OrderDecorator(object):

    TRUTHY = (True, 'True', 'TRUE', 'true', 1, 'yes')
    NONE = (None, 'None', 'null', '')

    def __repr__(self):
        return '<{}()>'.format(self.__class__.__name__)

    def order(self, albums, session, mpd):
        raise NotImplementedError


class BaseOrder(OrderDecorator):
    """Initialize all albums with unity order"""

    def order(self, albums, session, mpd):
        return defaultdict(
            lambda: 1.0,
            zip(session.query(Album).all(), repeat(1.0))
        )


class AlbumFilter(OrderDecorator):
    """Show albums whose name contains a string"""

    def __init__(self, *name_pieces):
        name = ' '.join(name_pieces)
        self.name = name
        self.name_rgx = re.compile(name, re.I)

    def __repr__(self):
        return '<AlbumFilter({})>'.format(self.name)

    def order(self, albums, session, mpd):
        return {
            album: order for album, order in albums.items()
            if re.search(self.name_rgx, album.name) is not None
        }


class ArtistFilter(OrderDecorator):
    """Show albums for which the artist name contains a string"""

    def __init__(self, *name_pieces):
        name = ' '.join(name_pieces)
        self.name = name
        self.name_rgx = re.compile(name, re.I)

    def __repr__(self):
        return '<ArtistFilter({})>'.format(self.name)

    def order(self, albums, session, mpd):
        return {
            album: order for album, order in albums.items()
            if re.search(self.name_rgx, album.artist.name) is not None
        }


class SortOrder(OrderDecorator):
    """Sort by 'Artist - Album'"""

    def __init__(self, ignore_artist_the=True, reverse=False):
        self.ignore_artist_the = bool(ignore_artist_the)
        self.reverse = (not bool(reverse))

    def _format(self, album):
        artist = album.artist.name
        if self.ignore_artist_the and artist.lower().startswith('the '):
            artist = artist[4:] + ', ' + artist[:3]

        return '{} - {}'.format(artist, album.name)

    def order(self, albums, session, mpd):
        sorted_albums = sorted(albums, key=self._format, reverse=self.reverse)
        return {album: i for i, album in enumerate(sorted_albums, 1)}


class ModifiedOrder(OrderDecorator):

    """Sort by modified date"""

    FMT = '%Y-%m-%dT%H:%M:%SZ'

    def __init__(self, reverse=False):
        self.reverse = bool(reverse)

    @classmethod
    def get_date(cls, album, mpd):
        track_info = mpd.search('album', album.name)
        if not track_info:
            return None

        dates = (datetime.strptime(info['last-modified'], cls.FMT)
                 for info in track_info)
        return max(dates)

    def order(self, albums, session, mpd):
        sorted_albums = sorted(
            albums,
            key=lambda album: self.get_date(album, mpd),
            reverse=self.reverse)
        return {album: i for i, album in enumerate(sorted_albums, 1)}


class BannedOrder(OrderDecorator):
    """Remove or demote albums with banned tracks"""

    def __init__(self, remove_banned=True):
        self.remove = remove_banned in self.TRUTHY

    def __repr__(self):
        return '<BannedOrder({})>'.format(self.remove)

    def _query_albums(self, session):
        return session.query(Album).\
            join(Track).\
            outerjoin(LastfmTrackInfo).\
            add_columns(func.count(Track.id),
                        func.sum(LastfmTrackInfo.banned, type_=Integer)).\
            group_by(Album.id).\
            all()

    def order(self, albums, session, mpd):
        results = self._query_albums(session)
        neworder = defaultdict(lambda: 1.0, albums.items())

        for album, n_tracks, n_banned in results:
            if album not in neworder or n_tracks == 0:
                continue

            if n_banned and self.remove:
                del neworder[album]
            elif n_banned:
                # Banned albums are equally worthless
                neworder[album] *= 0.0

        return neworder


class FractionLovedOrder(OrderDecorator):
    """Order by fraction of tracks loved"""

    def __init__(self, reverse=False, penalize_unloved=False, **kwArgs):
        super(FractionLovedOrder, self).__init__()
        self.reverse = bool(reverse)

        maximum = kwArgs.pop('max', None)
        minimum = kwArgs.pop('min', None)

        if kwArgs:
            arg = list(kwArgs.keys())[0]
            raise TypeError("FractionLovedOrder got an unexpected keyword "
                            "argument '{}'".format(arg))

        self.penalize = penalize_unloved

        if minimum in self.NONE:
            minimum = 0
        if maximum in self.NONE:
            maximum = 1

        try:
            minimum, maximum = float(minimum), float(maximum)
{'fraction_non_alphanumeric': 0.11528982199908717, 'fraction_numerical': 0.0038338658146964857, 'mean_word_length': 2.3700399876960936, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7285137', 'n_tokens_mistral': 3403, 'n_tokens_neox': 3239, 'n_words': 862}