<commit_before>import requests
import logging
import isodate
import importlib
from peewee import CharField, IntegerField, TextField, fn
from src.utils.auth import Auth
from src.models.base import BaseModel
from tornado.options import options

YOUTUBE = "youtube"
SPOTIFY = "spotify"
SOUNDCLOUD = "soundcloud"

VOTE_LIMIT = -2
DURATION_LIMIT_MAP = {
    YOUTUBE: 60*60*3,  # 3 hour
    SPOTIFY: 60*60,  # 1 hour
    SOUNDCLOUD: 60*60,  # 1 hour
}

YOUTUBE_URL = "https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&id=%s&fields=items&key="
SOUNDCLOUD_URL = "http://api.soundcloud.com/tracks/%s.json?client_id="

URL_MAP = {
    YOUTUBE: "%s%s" % (YOUTUBE_URL, options.youtube_key),
    SPOTIFY: "http://ws.spotify.com/lookup/1/.json?uri=spotify:track:%s",
    SOUNDCLOUD: "%s%s" % (SOUNDCLOUD_URL, options.soundcloud_key)
}


class MediaItemError(Exception):
    pass


class MediaItem(BaseModel):

    title = CharField(default="")
    author = CharField(default="")
    description = TextField(default="")
    thumbnail = CharField(default="")
    cid = CharField()
    nick = CharField()
    type = CharField()
    external_id = CharField()
    duration = IntegerField()
    album = CharField(null=True)
    permalink_url = CharField(null=True)

    def exists(self):
        return MediaItem.fetch().where(
            MediaItem.external_id == self.external_id,
            MediaItem.type == self.type
        ).exists()

    def save(self, *args, **kwargs):
        allowed_duration = DURATION_LIMIT_MAP.get(self.type)
        if self.duration > allowed_duration:
            raise MediaItemError("duration %d is longer then allowed %d" % (self.duration, allowed_duration))
<commit_msg>Allow fields to be null<commit_after>import requests
import logging
import isodate
import importlib
from peewee import CharField, IntegerField, TextField, fn
from src.utils.auth import Auth
from src.models.base import BaseModel
from tornado.options import options

YOUTUBE = "youtube"
SPOTIFY = "spotify"
SOUNDCLOUD = "soundcloud"

VOTE_LIMIT = -2
DURATION_LIMIT_MAP = {
    YOUTUBE: 60*60*3,  # 3 hour
    SPOTIFY: 60*60,  # 1 hour
    SOUNDCLOUD: 60*60,  # 1 hour
}

YOUTUBE_URL = "https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&id=%s&fields=items&key="
SOUNDCLOUD_URL = "http://api.soundcloud.com/tracks/%s.json?client_id="

URL_MAP = {
    YOUTUBE: "%s%s" % (YOUTUBE_URL, options.youtube_key),
    SPOTIFY: "http://ws.spotify.com/lookup/1/.json?uri=spotify:track:%s",
    SOUNDCLOUD: "%s%s" % (SOUNDCLOUD_URL, options.soundcloud_key)
}


class MediaItemError(Exception):
    pass


class MediaItem(BaseModel):

    title = CharField(default="")
    author = CharField(default="", null=True)
    description = TextField(default="", null=True)
    thumbnail = CharField(default="", null=True)
    cid = CharField()
    nick = CharField(null=True)
    type = CharField()
    external_id = CharField()
    duration = IntegerField(null=True)
    album = CharField(null=True)
    permalink_url = CharField(null=True)

    def exists(self):
        return MediaItem.fetch().where(
            MediaItem.external_id == self.external_id,
            MediaItem.type == self.type
        ).exists()

    def save(self, *args, **kwargs):
        allowed_duration = DURATION_LIMIT_MAP.get(self.type)
        if self.duration > allowed_duration:
            raise MediaItemError("duration %d is longer then allowed %d" % (self.duration, allowed_duration))

{'fraction_non_alphanumeric': 0.12632787826586275, 'fraction_numerical': 0.010910134941142693, 'mean_word_length': 3.8659217877094973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4506400', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1186, 'n_words': 243}