<commit_before>import logging
import re
import urllib2
import urlparse
import time
import booksearch
import pyisbn
import settings
from cgi import parse_qsl
from collections import defaultdict
from datetime import datetime
from email.utils import formatdate
from hashlib import sha1
from google.appengine.api import memcache, urlfetch
from google.appengine.ext import db
from google.appengine.ext.deferred import deferred
from zaim import urlnorm


class NormalizedLinkProperty(db.LinkProperty):
    def validate(self, value):
        value = super(NormalizedLinkProperty, self).validate(value)
        if value is not None:
            # urltuple = scheme, netloc, path, query, fragment
            urltuple = urlparse.urlparse(value)
            if not urltuple.scheme or not urltuple.netloc:
                raise BadValueError('Property %s must be a full URL (\'%s\')' %
                                    (self.name, value))
            else:
                urltuple = urlnorm.norm(urltuple)
                value = urlparse.urlunparse(urltuple)
        return value


class Book(db.Model):
    MEMCACHE_TTL = 604800  # 7 days
    ISBN13_REGEX = re.compile(r'^([0-9]{13})$')
    ISBN10_REGEX = re.compile(r'^([0-9]{10})$')

    isbn13 = db.StringProperty(required=True)
    isbn10 = db.StringProperty()
    title = db.StringProperty()
    authors = db.StringListProperty()
    publishers = db.StringListProperty()
    published_date = db.DateProperty()
    rating = db.RatingProperty()
    description = db.StringProperty(multiline=True)
    google_id = db.StringProperty()
    amazon_id = db.StringProperty()
    amazon_salesrank = db.IntegerProperty()
    mph_bestseller = db.BooleanProperty()

    def __init__(self, *args, **kwargs):
        # automatically generate key_name
        isbn10 = kwargs.get('isbn10')
        isbn13 = kwargs.get('isbn13', pyisbn.convert(isbn10))
        if isbn13:
            kwargs['key_name'] = 'book:%s' % isbn13
        super(Book, self).__init__(*args, **kwargs)
        self.links = LinkAttribute(self)

    def __cmp__(self, other):
        if not self or not other:
            return -1
        return cmp(self.isbn13, other.isbn13)

    def __hash__(self):
        return hash(self.isbn13)

    def put(self):
        memcache.delete(self.memkey)
        return super(Book, self).put()

    def delete(self):
        memcache.delete(self.memkey)
        return super(Book, self).delete()

    @property
    def memkey(self):
        return 'book:%s' % self.isbn13

    @property<commit_msg>Fixed bug in models.py in ISBN13 conversion from None<commit_after>import logging
import re
import urllib2
import urlparse
import time
import booksearch
import pyisbn
import settings
from cgi import parse_qsl
from collections import defaultdict
from datetime import datetime
from email.utils import formatdate
from hashlib import sha1
from google.appengine.api import memcache, urlfetch
from google.appengine.ext import db
from google.appengine.ext.deferred import deferred
from zaim import urlnorm


class NormalizedLinkProperty(db.LinkProperty):
    def validate(self, value):
        value = super(NormalizedLinkProperty, self).validate(value)
        if value is not None:
            # urltuple = scheme, netloc, path, query, fragment
            urltuple = urlparse.urlparse(value)
            if not urltuple.scheme or not urltuple.netloc:
                raise BadValueError('Property %s must be a full URL (\'%s\')' %
                                    (self.name, value))
            else:
                urltuple = urlnorm.norm(urltuple)
                value = urlparse.urlunparse(urltuple)
        return value


class Book(db.Model):
    MEMCACHE_TTL = 604800  # 7 days
    ISBN13_REGEX = re.compile(r'^([0-9]{13})$')
    ISBN10_REGEX = re.compile(r'^([0-9]{10})$')

    isbn13 = db.StringProperty(required=True)
    isbn10 = db.StringProperty()
    title = db.StringProperty()
    authors = db.StringListProperty()
    publishers = db.StringListProperty()
    published_date = db.DateProperty()
    rating = db.RatingProperty()
    description = db.StringProperty(multiline=True)
    google_id = db.StringProperty()
    amazon_id = db.StringProperty()
    amazon_salesrank = db.IntegerProperty()
    mph_bestseller = db.BooleanProperty()

    def __init__(self, *args, **kwargs):
        # automatically generate key_name
        isbn10 = kwargs.get('isbn10')
        isbn13 = kwargs.get('isbn13', (pyisbn.convert(isbn10) if isbn10 else None))
        if isbn13:
            kwargs['key_name'] = 'book:%s' % isbn13
        super(Book, self).__init__(*args, **kwargs)
        self.links = LinkAttribute(self)

    def __cmp__(self, other):
        if not self or not other:
            return -1
        return cmp(self.isbn13, other.isbn13)

    def __hash__(self):
        return hash(self.isbn13)

    def put(self):
        memcache.delete(self.memkey)
        return super(Book, self).put()

    def delete(self):
        memcache.delete(self.memkey)
        return super(Book, self).delete()

    @property
    def memkey(self):
        return 'book:%s' % self.isbn13

    @property
{'fraction_non_alphanumeric': 0.10046638165565487, 'fraction_numerical': 0.0194325689856199, 'mean_word_length': 2.7901325478645065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4761432', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1532, 'n_words': 393}