<commit_before>'''Slickbird scrapper'''

import logging
import os
import re
import io
import tornado.gen
import tornado.escape
import tornado.httpclient

try:
    from urllib.parse import quote
except ImportError:
    from urllib import quote
from lxml import etree

import slickbird.orm as orm
from slickbird import filenames

pjoin = os.path.join


def _log():
    if not _log.logger:
        _log.logger = logging.getLogger(__name__)
    return _log.logger


_log.logger = None


# Scrapper: ##################################################################

class Scrapper(object):
    FIELDMAP = {
        'title': './GameTitle',
        'year': './ReleaseDate',
        'publisher': './Publisher',
        'platform': './Platform',
        'genre': './Genres/genre',
        'plot': './Overview',
    }

    def __init__(self, session, home):
        self.session = session
        self.home = home

    @tornado.gen.coroutine
    def scrap_missing(self):
        l = []
        for v in self.session.query(orm.Variant):
            nfofile = filenames.nfo(self.home,
                                    v)
            if os.path.exists(nfofile):
                _log().debug('scrapper skipping {}, found {}'
                             .format(v.name, nfofile))
                continue
            romfile = filenames.variant(self.home, v)
            if not os.path.exists(romfile):
                _log().debug('scrapper skipping {}, rom not found {}'
                             .format(v.name, romfile))
                continue
            l.append(self.scrap(v, nfofile))
            if len(l) > 5:
                yield l
                l = []
        if len(l) > 0:
            yield l

    @tornado.gen.coroutine
    def scrap(self, v, nfofile):
        _log().info('scrapping {}'.format(v.name))
        url = 'http://thegamesdb.net/api/GetGame.php?exactname=' + \
            quote(v.game.name)
        http = tornado.httpclient.AsyncHTTPClient()
        response = yield http.fetch(url)
        if response.code != 200:
            _log().warn('error scrapping {}: {}'
                        .format(v.game.name, str(response)))
            raise tornado.gen.Return()
        try:<commit_msg>Fix "ambiguous var name" static analysis issue<commit_after>'''Slickbird scrapper'''

import logging
import os
import re
import io
import tornado.gen
import tornado.escape
import tornado.httpclient

try:
    from urllib.parse import quote
except ImportError:
    from urllib import quote
from lxml import etree

import slickbird.orm as orm
from slickbird import filenames

pjoin = os.path.join


def _log():
    if not _log.logger:
        _log.logger = logging.getLogger(__name__)
    return _log.logger


_log.logger = None


# Scrapper: ##################################################################

class Scrapper(object):
    FIELDMAP = {
        'title': './GameTitle',
        'year': './ReleaseDate',
        'publisher': './Publisher',
        'platform': './Platform',
        'genre': './Genres/genre',
        'plot': './Overview',
    }

    def __init__(self, session, home):
        self.session = session
        self.home = home

    @tornado.gen.coroutine
    def scrap_missing(self):
        lines = []
        for v in self.session.query(orm.Variant):
            nfofile = filenames.nfo(self.home,
                                    v)
            if os.path.exists(nfofile):
                _log().debug('scrapper skipping {}, found {}'
                             .format(v.name, nfofile))
                continue
            romfile = filenames.variant(self.home, v)
            if not os.path.exists(romfile):
                _log().debug('scrapper skipping {}, rom not found {}'
                             .format(v.name, romfile))
                continue
            lines.append(self.scrap(v, nfofile))
            if len(lines) > 5:
                yield lines
                lines = []
        if len(lines) > 0:
            yield lines

    @tornado.gen.coroutine
    def scrap(self, v, nfofile):
        _log().info('scrapping {}'.format(v.name))
        url = 'http://thegamesdb.net/api/GetGame.php?exactname=' + \
            quote(v.game.name)
        http = tornado.httpclient.AsyncHTTPClient()
        response = yield http.fetch(url)
        if response.code != 200:
            _log().warn('error scrapping {}: {}'
                        .format(v.game.name, str(response)))
            raise tornado.gen.Return()
        try:
{'fraction_non_alphanumeric': 0.14727474972191323, 'fraction_numerical': 0.002224694104560623, 'mean_word_length': 1.8546031746031746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3586460', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1285, 'n_words': 299}