<commit_before>
import re

# We try and exclusively query google to avoid thrashing animemusicvideos.org
GOOGLE_JSON_AMVS = 'http://ajax.googleapis.com/ajax/services/search/web'\
    '?v=1.0&rsz=large&q="site:animemusicvideos.org"+video+information+%s'


def Start():
    pass


class AMVAgent(Agent.Movies):
    name = 'Anime Music Videos'
    languages = [Locale.Language.English]

    def search(self, results, media, lang, manual):
        try:
            google_results = JSON.ObjectFromURL(GOOGLE_JSON_AMVS % media.name)[
                'responseData']['results']
        except:
            return None
        for result in google_results:
            url = result['url']
            if not url.startswith('http://www.animemusicvideos.org'
                                  '/members/members_videoinfo.php'):
                continue
            vid = None
            try:
                params = url.split('%3F', 2)[1].split('%26')
                for param in params:
                    key, value = param.split('%3D')
                    if key == 'v':
                        vid = int(value)
            except:
                continue
            if vid is None:
                continue
            try:
                title = result['titleNoFormatting'].split('Information: ')[1]
            except:
                title = result['titleNoFormatting']
            results.Append(MetadataSearchResult(
                id=str(vid),
                name=title,
                year=2000,
                score=Util.LevenshteinDistance(title.lower(),
                                               media.name.lower()),
                lang=lang
            ))

    def update(self, metadata, media, lang):
        pass<commit_msg>Better google searching<commit_after>
import re
import urllib

# We try and exclusively query google to avoid thrashing animemusicvideos.org
GOOGLE_JSON_AMVS = 'http://ajax.googleapis.com/ajax/services/search/web'\
    '?v=1.0&rsz=large&q="site:animemusicvideos.org"+video+information+%s'


def Start():
    pass


class AMVAgent(Agent.Movies):
    name = 'Anime Music Videos'
    languages = [Locale.Language.English]

    def search(self, results, media, lang, manual):
        sanitized_name = media.name.lower()
        if sanitized_name.startswith('amv'):
            sanitized_name = sanitized_name[3:]
        sanitized_name = sanitized_name.replace('-', ' ')
        sanitized_name = sanitized_name.strip()
        try:
            safe_name = urllib.quote_plus(sanitized_name)
            google_results = JSON.ObjectFromURL(GOOGLE_JSON_AMVS % safe_name)[
                'responseData']['results']
        except:
            return None
        for result in google_results:
            url = result['url']
            vid = None
            if url.startswith('http://www.animemusicvideos.org'
                              '/members/members_videoinfo.php'):
                try:
                    params = url.split('%3F', 2)[1].split('%26')
                    for param in params:
                        key, value = param.split('%3D')
                        if key == 'v':
                            vid = int(value)
                except:
                    continue
            elif url.startswith('http://www.animemusicvideos.org'
                                '/video/'):
                try:
                    vid = int(url.split('/')[-1])
                except:
                    continue
            if vid is None:
                continue
            try:
                title = result['titleNoFormatting'].split('Information: ')[1]
            except:
                title = result['titleNoFormatting']
            match = re.match(r'.+Video Information: (.*?)(?: - AnimeMusicVideos.org)?$', result['titleNoFormatting'])
            try:
                title = match.group(1)
            except:
                pass
            Log([title, match.groups() if match else None])
            results.Append(MetadataSearchResult(
                id=str(vid),
                name=title,
                year=None,
                score=100-Util.LevenshteinDistance(title.lower(),
                                                   sanitized_name),
                lang=lang
            ))

    def update(self, metadata, media, lang):
        pass
{'fraction_non_alphanumeric': 0.10821412090447624, 'fraction_numerical': 0.006460544531610522, 'mean_word_length': 1.4757281553398058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2327090', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1128, 'n_words': 231}