<commit_before>import time
import os.path

import bencode
from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from pyquery.pyquery import PyQuery

from bibliotik.settings import BIBLIOTIK_GET_TORRENT_URL
from home.models import TransTorrentBase
from what_transcode.utils import get_info_hash_from_data


EBOOK_FORMATS = ['EPUB', 'PDF', 'MOBI', 'AZW3', 'DJVU', 'CBR', 'CHM', 'TXT']
LANGUAGES = ['English', 'Irish', 'German', 'French', 'Spanish', 'Italian', 'Latin', 'Japanese',
             'Danish', 'Swedish', 'Norwegian', 'Dutch', 'Russian', 'Polish', 'Portuguese', 'Greek',
             'Turkish', 'Hungarian', 'Korean', 'Chinese', 'Thai', 'Indonesian', 'Arabic']


def load_bibliotik_data(bibliotik_client, torrent_id):
    exception = None
    for i in xrange(3):
        try:
            response = bibliotik_client.session.get(
                BIBLIOTIK_GET_TORRENT_URL.format(torrent_id), allow_redirects=False)
            if response.status_code != 200:
                raise Exception('Getting bibliotik data returned HTTP {0}'
                                .format(response.status_code))
            return response.text
        except Exception as ex:
            print u'Error while retrieving bibliotik data. Will retry: {0}'.format(ex)
            time.sleep(2)
            exception = ex
    raise exception


class BibliotikTorrent(models.Model):
    info_hash = models.CharField(max_length=40, db_index=True)
    retrieved = models.DateTimeField()
    category = models.CharField(max_length=32)
    format = models.CharField(max_length=16)
    retail = models.BooleanField(default=False)
    pages = models.IntegerField()
    language = models.CharField(max_length=32)
    isbn = models.CharField(max_length=16)
    cover_url = models.TextField()
    tags = models.TextField()
    publisher = models.TextField()
    year = models.IntegerField()
    author = models.TextField()
    title = models.TextField()
    html_page = models.TextField()
    torrent_filename = models.TextField(null=True)
    torrent_file = models.BinaryField(null=True)

    @cached_property
    def publisher_list(self):
        return self.publisher.split(';')

    def __unicode__(self):
        return u'BibliotikTorrent id={0} hash={1}'.format(self.id, self.info_hash)

    def import_bibliotik_data(self, bibliotik_client):
        self.html_page = load_bibliotik_data(bibliotik_client, self.id)
        self.parse_html_page()

    def parse_html_page(self):
        pq = PyQuery(self.html_page)
        authors = []
        for author in pq('p#creatorlist a').items():
            authors.append(author.text())
        self.author = ', '.join(authors)
        self.title = pq('h1#title').text()
        if not self.title:
            raise Exception(u'Title should not be empty.')
        self.category = pq('h1#title > img:first-child').attr('title')
        details = pq('p#details_content_info').text().split(', ')
        self.format = details[0]
        details = details[1:]
        if self.category == u'Ebooks':
            assert self.format in EBOOK_FORMATS, u'Unknown eBook format {0}'.format(self.format)
        elif self.category == u'Applications':
            pass
        elif self.category == u'Articles':
            pass
        elif self.category == u'Audiobooks':
            pass
        elif self.category == u'Comics':
            pass
        elif self.category == u'Journals':
            pass
        elif self.category == u'Magazines':
            pass
        else:
            raise Exception(u'Unknown category {0}'.format(self.category))
        if details[0] == u'Retail':
            self.retail = True
            details = details[1:]
        else:
            self.retail = False
        if details[0].endswith(u'pages'):
            self.pages = int(details[0][:-len(u'pages') - 1])
            details = details[1:]
        else:
            self.pages = 0
        if details[0].split(' ')[0] in LANGUAGES:
            parts = details[0].split(' ')
            details = details[1:]
            self.language = parts[0]
            parts = parts[1:]
            if len(parts):
                assert parts[0][0] == '(' and parts[0][-1] == ')', u'Unknown string after language'
                self.isbn = parts[0][1:-1]
                parts = parts[1:]
            else:
                self.isbn = ''
        else:
            self.language = ''
        assert len(details) == 0, u'All details must be parsed: {0}'.format(', '.join(details))
        self.cover_url = pq('div#sidebar > a[rel="lightbox"] > img').attr('src') or ''
        self.tags = ', '.join(i.text() for i in pq('span.taglist > a').items())
        publisher_year = pq('p#published').text()
        if publisher_year:
            assert publisher_year.startswith('Published '), \<commit_msg>Add compatibility for audiobooks<commit_after>import time
import os.path

import bencode
from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from pyquery.pyquery import PyQuery

from bibliotik.settings import BIBLIOTIK_GET_TORRENT_URL
from home.models import TransTorrentBase
from what_transcode.utils import get_info_hash_from_data


EBOOK_FORMATS = ['EPUB', 'PDF', 'MOBI', 'AZW3', 'DJVU', 'CBR', 'CHM', 'TXT']
LANGUAGES = ['English', 'Irish', 'German', 'French', 'Spanish', 'Italian', 'Latin', 'Japanese',
             'Danish', 'Swedish', 'Norwegian', 'Dutch', 'Russian', 'Polish', 'Portuguese', 'Greek',
             'Turkish', 'Hungarian', 'Korean', 'Chinese', 'Thai', 'Indonesian', 'Arabic']


def load_bibliotik_data(bibliotik_client, torrent_id):
    exception = None
    for i in xrange(3):
        try:
            response = bibliotik_client.session.get(
                BIBLIOTIK_GET_TORRENT_URL.format(torrent_id), allow_redirects=False)
            if response.status_code != 200:
                raise Exception('Getting bibliotik data returned HTTP {0}'
                                .format(response.status_code))
            return response.text
        except Exception as ex:
            print u'Error while retrieving bibliotik data. Will retry: {0}'.format(ex)
            time.sleep(2)
            exception = ex
    raise exception


class BibliotikTorrent(models.Model):
    info_hash = models.CharField(max_length=40, db_index=True)
    retrieved = models.DateTimeField()
    category = models.CharField(max_length=32)
    format = models.CharField(max_length=16)
    retail = models.BooleanField(default=False)
    pages = models.IntegerField()
    language = models.CharField(max_length=32)
    isbn = models.CharField(max_length=16)
    cover_url = models.TextField()
    tags = models.TextField()
    publisher = models.TextField()
    year = models.IntegerField()
    author = models.TextField()
    title = models.TextField()
    html_page = models.TextField()
    torrent_filename = models.TextField(null=True)
    torrent_file = models.BinaryField(null=True)

    @cached_property
    def publisher_list(self):
        return self.publisher.split(';')

    def __unicode__(self):
        return u'BibliotikTorrent id={0} hash={1}'.format(self.id, self.info_hash)

    def import_bibliotik_data(self, bibliotik_client):
        self.html_page = load_bibliotik_data(bibliotik_client, self.id)
        self.parse_html_page()

    def parse_html_page(self):
        pq = PyQuery(self.html_page)
        authors = []
        for author in pq('p#creatorlist a').items():
            authors.append(author.text())
        self.author = ', '.join(authors)
        self.title = pq('h1#title').text()
        if not self.title:
            raise Exception(u'Title should not be empty.')
        self.category = pq('h1#title > img:first-child').attr('title')
        details = pq('p#details_content_info').text().split(', ')
        self.format = details[0]
        details = details[1:]
        if self.category == u'Ebooks':
            assert self.format in EBOOK_FORMATS, u'Unknown eBook format {0}'.format(self.format)
        elif self.category == u'Applications':
            pass
        elif self.category == u'Articles':
            pass
        elif self.category == u'Audiobooks':
            pass
        elif self.category == u'Comics':
            pass
        elif self.category == u'Journals':
            pass
        elif self.category == u'Magazines':
            pass
        else:
            raise Exception(u'Unknown category {0}'.format(self.category))
        if details[0] == u'Retail':
            self.retail = True
            details = details[1:]
        else:
            self.retail = False
        if details[0].endswith(u'pages'):
            self.pages = int(details[0][:-len(u'pages') - 1])
            details = details[1:]
        else:
            self.pages = 0
        if details[0] == u'Unabridged' or details[0] == u'Abridged':
            details = details[1:]
        if details[0].split(' ')[0] in LANGUAGES:
            parts = details[0].split(' ')
            details = details[1:]
            self.language = parts[0]
            parts = parts[1:]
            if len(parts):
                assert parts[0][0] == '(' and parts[0][-1] == ')', u'Unknown string after language'
                self.isbn = parts[0][1:-1]
                parts = parts[1:]
            else:
                self.isbn = ''
        else:
            self.language = ''
        assert len(details) == 0, u'All details must be parsed: {0}'.format(', '.join(details))
        self.cover_url = pq('div#sidebar > a[rel="lightbox"] > img').attr('src') or ''
        self.tags = ', '.join(i.text() for i in pq('span.taglist > a').items())
        publisher_year = pq('p#published').text()
        if publisher_year:
            assert publisher_year.startswith('Published '), \
{'fraction_non_alphanumeric': 0.12982135606983353, 'fraction_numerical': 0.010251725537961835, 'mean_word_length': 2.4705882352941178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2181164', 'n_tokens_mistral': 3082, 'n_tokens_neox': 2942, 'n_words': 651}