<commit_before>#!/usr/bin/env python

# The MIT License (MIT)

# Copyright (c) 2013 <NAME>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""  """

from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

import time
from urllib import quote
from Queue import Queue, Empty

from lxml.html import fromstring, tostring

from calibre import as_unicode
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source
from calibre.utils.icu import lower
from calibre.utils.cleantext import clean_ascii_chars

__author__ = "<NAME>"
__copyright__ = "Copyright 2013"
__credits__ = ["<NAME> <<EMAIL>>"]

__license__ = "MIT"
__version__ = ""
__maintainer__ = "<NAME>"
__email__ = ""
__url__ = "http://github.com/beeftornado/"


class Shelfari(Source):

    name = 'Shelfari'
    description = _('Downloads metadata and covers from Shelfari')
    author = '<NAME>'
    version = (0, 0, 1)
    minimum_calibre_version = (0, 8, 0)

    capabilities = frozenset(['identify', 'cover'])
    touched_fields = frozenset(['title', 'authors', 'identifier:shelfari',
        'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate',
        'tags', 'series', 'languages'])
    has_html_comments = True
    supports_gzip_transfer_encoding = True

    BASE_URL = 'http://www.shelfari.com'
    MAX_EDITIONS = 5

    def config_widget(self):
        '''
        Overriding the default configuration screen for our own custom configuration
        '''
        from calibre_plugins.shelfari.config import ConfigWidget
        return ConfigWidget(self)

    def get_book_url(self, identifiers):
        shelfari_id = identifiers.get('shelfari', None)
        if shelfari_id:
            return ('shelfari', shelfari_id,
                    '%s/books/%s' % (Shelfari.BASE_URL, shelfari_id))

    def create_query(self, log, title=None, authors=None, identifiers={}):

        isbn = check_isbn(identifiers.get('isbn', None))
        q = ''
        if isbn is not None:
            q = 'search_type=books&search[query]=' + isbn
        elif title or authors:
            tokens = []
            title_tokens = list(self.get_title_tokens(title,
                                strip_joiners=False, strip_subtitle=True))
            tokens += title_tokens
            author_tokens = self.get_author_tokens(authors,
                    only_first_author=True)
            tokens += author_tokens
            tokens = [quote(t.encode('utf-8') if isinstance(t, unicode) else t) for t in tokens]
            q = '+'.join(tokens)
            q = 'search_type=books&search[query]=' + q

        if not q:
            return None
        if isinstance(q, unicode):
            q = q.encode('utf-8')
        return Shelfari.BASE_URL + '/search?' + q

    def get_cached_cover_url(self, identifiers):
        url = None
        shelfari_id = identifiers.get('shelfari', None)
        if shelfari_id is None:
            isbn = identifiers.get('isbn', None)
            if isbn is not None:
                shelfari_id = self.cached_isbn_to_identifier(isbn)
        if shelfari_id is not None:
            url = self.cached_identifier_to_cover_url(shelfari_id)

        return url

    def identify(self, log, result_queue, abort, title=None, authors=None,
            identifiers={}, timeout=30):
        '''
        Note this method will retry without identifiers automatically if no
        match is found with identifiers.
        '''
        matches = []
        # Unlike the other metadata sources, if we have a shelfari id then we
        # do not need to fire a "search" at Shelfari.com. Instead we will be
        # able to go straight to the URL for that book.
        shelfari_id = identifiers.get('shelfari', None)
        isbn = check_isbn(identifiers.get('isbn', None))
        br = self.browser
        if shelfari_id:
            matches.append('%s/book/show/%s' % (Shelfari.BASE_URL, shelfari_id))
        else:
            query = self.create_query(log, title=title, authors=authors,
                    identifiers=identifiers)
            if query is None:
                log.error('Insufficient metadata to construct query')
                return
            try:
                log.info('Querying: %s' % query)
                response = br.open_novisit(query, timeout=timeout)
                if isbn:
                    # Check whether we got redirected to a book page for ISBN searches.
                    # If we did, will use the url.
                    # If we didn't then treat it as no matches on Shelfari
                    location = response.geturl()
                    if '/book/show/' in location:
                        log.info('ISBN match location: %r' % location)
                        matches.append(location)
            except Exception as e:
                err = 'Failed to make identify query: %r' % query
                log.exception(err)
                return as_unicode(e)

            # For ISBN based searches we have already done everything we need to
            # So anything from this point below is for title/author based searches.
            if not isbn:
                try:
                    raw = response.read().strip()
                    #open('E:\\t.html', 'wb').write(raw)
                    raw = raw.decode('utf-8', errors='replace')
                    if not raw:
                        log.error('Failed to get raw result for query: %r' % query)
                        return
                    root = fromstring(clean_ascii_chars(raw))
                except:
                    msg = 'Failed to parse shelfari page for query: %r' % query
                    log.exception(msg)
                    return msg
                # Now grab the first value from the search results, provided the
                # title and authors appear to be for the same book
                self._parse_search_results(log, title, authors, root, matches, timeout)

        if abort.is_set():
            return

        if not matches:
            if identifiers and title and authors:
                log.info('No matches found with identifiers, retrying using only'
                        ' title and authors')
                return self.identify(log, result_queue, abort, title=title,
                        authors=authors, timeout=timeout)
            log.error('No matches found with query: %r' % query)
            return

        from calibre_plugins.shelfari.worker import Worker
        workers = [Worker(url, result_queue, br, log, i, self) for i, url in
                enumerate(matches)]

        for w in workers:
            w.start()
            # Don't send all requests at the same time
            time.sleep(0.1)

        while not abort.is_set():
            a_worker_is_alive = False
            for w in workers:
                w.join(0.2)
                if abort.is_set():
                    break
                if w.is_alive():
                    a_worker_is_alive = True
            if not a_worker_is_alive:
                break

        return None

    def _parse_search_results(self, log, orig_title, orig_authors, root, matches, timeout):
        first_result = root.xpath('//table[@class="tableList"]/tr/td[2]')
        if not first_result:
            return<commit_msg>Update the book search for shelfari<commit_after>#!/usr/bin/env python

# The MIT License (MIT)

# Copyright (c) 2013 <NAME>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""  """

from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

import time
from urllib import quote
from Queue import Queue, Empty

from lxml.html import fromstring, tostring

from calibre import as_unicode
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source
from calibre.utils.icu import lower
from calibre.utils.cleantext import clean_ascii_chars

from calibre_plugins.shelfari.worker import Worker

__author__ = "<NAME>"
__copyright__ = "Copyright 2013"
__credits__ = ["<NAME> <<EMAIL>>"]

__license__ = "MIT"
__version__ = ""
__maintainer__ = "<NAME>"
__email__ = ""
__url__ = "http://github.com/beeftornado/"


class Shelfari(Source):

    name = 'Shelfari'
    description = _('Downloads metadata and covers from Shelfari')
    author = '<NAME>'
    version = (0, 0, 1)
    minimum_calibre_version = (0, 8, 0)

    capabilities = frozenset(['identify', 'cover'])
    touched_fields = frozenset(['title', 'authors', 'identifier:shelfari',
        'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate',
        'tags', 'series', 'languages'])
    has_html_comments = True
    supports_gzip_transfer_encoding = True

    BASE_URL = 'http://www.shelfari.com'
    MAX_EDITIONS = 5

    def config_widget(self):
        '''
        Overriding the default configuration screen for our own custom configuration
        '''
        from calibre_plugins.shelfari.config import ConfigWidget
        return ConfigWidget(self)

    def get_book_url(self, identifiers):
        shelfari_id = identifiers.get('shelfari', None)
        if shelfari_id:
            return ('shelfari', shelfari_id,
                    '%s/books/%s' % (Shelfari.BASE_URL, shelfari_id))

    def _create_query(self, log, title=None, authors=None, identifiers={}):
        """ Generates the search url to use to find the book """
        isbn = check_isbn(identifiers.get('isbn', None))
        q = []
        if isbn is not None:
            # do isbn search
            q.append('Isbn=' + isbn)
            
        if title or authors:
            # do title and or author based search
            
            # tokenize the author and title fields from the current metadata
            title_tokens = list(self.get_title_tokens(title,
                                strip_joiners=False, strip_subtitle=True))
            author_tokens = self.get_author_tokens(authors,
                    only_first_author=True)
            
            # sanitize the title and author info before sending
            title_tokens = [quote(t.encode('utf-8') if isinstance(t, unicode) else t) for t in title_tokens]
            author_tokens = [quote(t.encode('utf-8') if isinstance(t, unicode) else t) for t in author_tokens]
            
            # build the query from the tokens
            if len(title_tokens):
                q.append("Title={0}".format('+'.join(title_tokens)))
            if len(author_tokens):
                q.append("Author={0}".format('+'.join(author_tokens)))
            
            q = '&'.join(q)

        if not q:
            return None
        if isinstance(q, unicode):
            q = q.encode('utf-8')
        return Shelfari.BASE_URL + '/search/books?' + q

    def get_cached_cover_url(self, identifiers):
        url = None
        shelfari_id = identifiers.get('shelfari', None)
        if shelfari_id is None:
            isbn = identifiers.get('isbn', None)
            if isbn is not None:
                shelfari_id = self.cached_isbn_to_identifier(isbn)
        if shelfari_id is not None:
            url = self.cached_identifier_to_cover_url(shelfari_id)

        return url

    def identify(self, log, result_queue, abort, title=None, authors=None,
            identifiers={}, timeout=30):
        '''
        .. note::
            this method will retry without identifiers automatically if no
            match is found with identifiers.
        '''
        matches = []
        # Unlike the other metadata sources, if we have a shelfari id then we
        # do not need to fire a "search" at Shelfari.com. Instead we will be
        # able to go straight to the URL for that book.
        shelfari_id = identifiers.get('shelfari', None)
        isbn = check_isbn(identifiers.get('isbn', None))
        br = self.browser
        if shelfari_id:
            matches.append('%s/books/%s' % (Shelfari.BASE_URL, shelfari_id))
        else:
            query = self._create_query(log, title=title, authors=authors,
                    identifiers=identifiers)
            if query is None:
                log.error('Insufficient metadata to construct query')
                return
            try:
                log.info('Querying: %s' % query)
                response = br.open_novisit(query, timeout=timeout)
                if isbn:
                    # Check whether we got redirected to a book page for ISBN searches.
                    # If we did, will use the url.
                    # If we didn't then treat it as no matches on Shelfari
                    location = response.geturl()
                    if '/search/' not in location:
                        log.info('ISBN match location: %r' % location)
                        matches.append(location)
            except Exception as e:
                err = 'Failed to make identify query: %r' % query
                log.exception(err)
                return as_unicode(e)

            # For ISBN based searches we have already done everything we need to
            # So anything from this point below is for title/author based searches.
            if not isbn:
                try:
                    raw = response.read().strip()
                    #open('E:\\t.html', 'wb').write(raw)
                    raw = raw.decode('utf-8', errors='replace')
                    if not raw:
                        log.error('Failed to get raw result for query: %r' % query)
                        return
                    root = fromstring(clean_ascii_chars(raw))
                except:
                    msg = 'Failed to parse shelfari page for query: %r' % query
                    log.exception(msg)
                    return msg
                # Now grab the first value from the search results, provided the
                # title and authors appear to be for the same book
                self._parse_search_results(log, title, authors, root, matches, timeout)

        if abort.is_set():
            return

        if not matches:
            # If there's no matches, normally we would try to query with less info, but shelfari's search is already fuzzy
            log.error('No matches found with query: %r' % query)
            return

        # Setup worker threads to look more thoroughly at matching books to extract information
        workers = [Worker(url, result_queue, br, log, i, self) for i, url in
                enumerate(matches)]

        # Start the workers and stagger them so we don't hammer shelfari :)
        for w in workers:
            w.start()
            time.sleep(0.1)

        while not abort.is_set():
            a_worker_is_alive = False
            for w in workers:
                w.join(0.2)
                if abort.is_set():
                    break
                if w.is_alive():
                    a_worker_is_alive = True
            if not a_worker_is_alive:
                break

        return None

    def _parse_search_results(self, log, orig_title, orig_authors, root, matches, timeout):
        first_result = root.xpath('//table[@class="tableList"]/tr/td[2]')
        if not first_result:
            return
{'fraction_non_alphanumeric': 0.08547207060736267, 'fraction_numerical': 0.0030774590639879224, 'mean_word_length': 2.068412613575628, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4955051', 'n_tokens_mistral': 4783, 'n_tokens_neox': 4520, 'n_words': 1574}