<commit_before>from __future__ import unicode_literals

import hashlib
import logging
import operator
import os
import os.path
import sqlite3
import sys
import uritools

from mopidy import local
from mopidy.exceptions import ExtensionError
from mopidy.local import translator
from mopidy.models import Ref, SearchResult
from mopidy.utils.path import path_to_uri, uri_to_path

from . import Extension, schema

URI_PREFIX = 'local:'

logger = logging.getLogger(__name__)


# missing from mopidy.local.translator
def local_directory_uri_to_path(uri, media_dir):
    if not uri.startswith('local:directory:'):
        raise ValueError('Invalid URI.')
    file_path = uri_to_path(uri).split(b':', 1)[1]
    return os.path.join(media_dir, file_path)


class SQLiteLibrary(local.Library):

    ROOT_PATH_URI = 'local:directory:'

    name = 'sqlite'

    def __init__(self, config):
        self._config = ext_config = config[Extension.ext_name]
        self._data_dir = Extension.get_or_create_data_dir(config)
        try:
            self._media_dir = config['local']['media_dir']
            excluded = config['local']['excluded_file_extensions']
            self._excluded_ext = tuple(bytes(e.lower()) for e in excluded)
        except KeyError:
            raise ExtensionError('Mopidy-Local not enabled')
        self._directories = []
        for line in ext_config['directories']:
            name, uri = line.rsplit(None, 1)
            ref = Ref.directory(uri=uri, name=name)
            self._directories.append(ref)
        self._dbpath = os.path.join(self._data_dir, b'library.db')
        self._connection = None

    def load(self):
        with self._connect() as connection:
            version = schema.load(connection)
            logger.debug('Using SQLite database schema v%s', version)
            return schema.count_tracks(connection)

    def lookup(self, uri):
        if uri.startswith('local:album'):
            return list(schema.lookup(self._connect(), Ref.ALBUM, uri))
        elif uri.startswith('local:artist'):
            return list(schema.lookup(self._connect(), Ref.ARTIST, uri))
        elif uri.startswith('local:track'):
            return list(schema.lookup(self._connect(), Ref.TRACK, uri))
        else:
            logger.error('Invalid lookup URI %s', uri)
            return []

    def browse(self, uri):
        try:
            if uri == self.ROOT_DIRECTORY_URI:
                return self._directories
            elif uri.startswith(self.ROOT_PATH_URI):
                return self._browse_path(uri)
            elif uri.startswith('local:directory'):
                return self._browse_directory(uri)
            elif uri.startswith('local:artist'):
                return self._browse_artist(uri)
            elif uri.startswith('local:album'):
                return self._browse_album(uri)
            else:
                raise ValueError('Invalid browse URI')
        except Exception as e:
            logger.error('Error browsing %s: %s', uri, e)
            return []

    def search(self, query=None, limit=100, offset=0, uris=None, exact=False):
        q = []
        for field, values in (query.items() if query else []):
            if isinstance(values, basestring):
                q.append((field, values))
            else:
                q.extend((field, value) for value in values)
        # temporary workaround until Mopidy core sets limit
        if self._config['search_limit'] is not None:
            limit = self._config['search_limit']
        filters = [f for uri in uris or [] for f in self._filters(uri) if f]
        with self._connect() as c:
            tracks = schema.search_tracks(c, q, limit, offset, exact, filters)
        uri = uritools.uricompose('local', path='search', query=q)
        return SearchResult(uri=uri, tracks=tracks)

    def get_distinct(self, field, query=None):
        q = []
        for key, values in (query.items() if query else []):
            if isinstance(values, basestring):
                q.append((key, values))
            else:
                q.extend((key, value) for value in values)
        return set(schema.list_distinct(self._connect(), field, q))

    def begin(self):
        return schema.tracks(self._connect())

    def add(self, track):
        try:
            track = self._validate_track(track)
            schema.insert_track(self._connect(), track)
        except Exception as e:
            logger.warn('Skipped %s: %s', track.uri, e)

    def remove(self, uri):
        schema.delete_track(self._connect(), uri)
<commit_msg>Fix #57: Remove search query normalization.<commit_after>from __future__ import unicode_literals

import hashlib
import logging
import operator
import os
import os.path
import sqlite3
import sys
import uritools

from mopidy import local
from mopidy.exceptions import ExtensionError
from mopidy.local import translator
from mopidy.models import Ref, SearchResult
from mopidy.utils.path import path_to_uri, uri_to_path

from . import Extension, schema

URI_PREFIX = 'local:'

logger = logging.getLogger(__name__)


# missing from mopidy.local.translator
def local_directory_uri_to_path(uri, media_dir):
    if not uri.startswith('local:directory:'):
        raise ValueError('Invalid URI.')
    file_path = uri_to_path(uri).split(b':', 1)[1]
    return os.path.join(media_dir, file_path)


class SQLiteLibrary(local.Library):

    ROOT_PATH_URI = 'local:directory:'

    name = 'sqlite'

    def __init__(self, config):
        self._config = ext_config = config[Extension.ext_name]
        self._data_dir = Extension.get_or_create_data_dir(config)
        try:
            self._media_dir = config['local']['media_dir']
            excluded = config['local']['excluded_file_extensions']
            self._excluded_ext = tuple(bytes(e.lower()) for e in excluded)
        except KeyError:
            raise ExtensionError('Mopidy-Local not enabled')
        self._directories = []
        for line in ext_config['directories']:
            name, uri = line.rsplit(None, 1)
            ref = Ref.directory(uri=uri, name=name)
            self._directories.append(ref)
        self._dbpath = os.path.join(self._data_dir, b'library.db')
        self._connection = None

    def load(self):
        with self._connect() as connection:
            version = schema.load(connection)
            logger.debug('Using SQLite database schema v%s', version)
            return schema.count_tracks(connection)

    def lookup(self, uri):
        if uri.startswith('local:album'):
            return list(schema.lookup(self._connect(), Ref.ALBUM, uri))
        elif uri.startswith('local:artist'):
            return list(schema.lookup(self._connect(), Ref.ARTIST, uri))
        elif uri.startswith('local:track'):
            return list(schema.lookup(self._connect(), Ref.TRACK, uri))
        else:
            logger.error('Invalid lookup URI %s', uri)
            return []

    def browse(self, uri):
        try:
            if uri == self.ROOT_DIRECTORY_URI:
                return self._directories
            elif uri.startswith(self.ROOT_PATH_URI):
                return self._browse_path(uri)
            elif uri.startswith('local:directory'):
                return self._browse_directory(uri)
            elif uri.startswith('local:artist'):
                return self._browse_artist(uri)
            elif uri.startswith('local:album'):
                return self._browse_album(uri)
            else:
                raise ValueError('Invalid browse URI')
        except Exception as e:
            logger.error('Error browsing %s: %s', uri, e)
            return []

    def search(self, query=None, limit=100, offset=0, uris=None, exact=False):
        q = []
        for field, values in (query.items() if query else []):
            q.extend((field, value) for value in values)
        # temporary workaround until Mopidy core sets limit
        if self._config['search_limit'] is not None:
            limit = self._config['search_limit']
        filters = [f for uri in uris or [] for f in self._filters(uri) if f]
        with self._connect() as c:
            tracks = schema.search_tracks(c, q, limit, offset, exact, filters)
        uri = uritools.uricompose('local', path='search', query=q)
        return SearchResult(uri=uri, tracks=tracks)

    def get_distinct(self, field, query=None):
        q = []
        for key, values in (query.items() if query else []):
            q.extend((key, value) for value in values)
        return set(schema.list_distinct(self._connect(), field, q))

    def begin(self):
        return schema.tracks(self._connect())

    def add(self, track):
        try:
            track = self._validate_track(track)
            schema.insert_track(self._connect(), track)
        except Exception as e:
            logger.warn('Skipped %s: %s', track.uri, e)

    def remove(self, uri):
        schema.delete_track(self._connect(), uri)

{'fraction_non_alphanumeric': 0.10960893854748603, 'fraction_numerical': 0.002011173184357542, 'mean_word_length': 2.514330585001963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4097707', 'n_tokens_mistral': 2612, 'n_tokens_neox': 2531, 'n_words': 653}