<commit_before>;(function( _, $, App ){

    /// Constants: -------------------------------------------------------------

    var NO_RESULTS = '<li class="warning">No stations found matching your criteria. Please change your search parameters and try again.</li>';

    /// Variables: -------------------------------------------------------------

    var $form;
    var $results;
    var stationList =   App.StationList();
    var genres =        App.Db.genres;
    var words =         App.Db.words;
    var stations =      App.Db.stations;
    var X =             window.goodJob;
    var onChangeTh =    _.throttle( onChange, 1000, { leading: false, trailing: true });

    /// Exports: ---------------------------------------------------------------

    App.Search = {
        search:     search,
    };

    /// Init: ------------------------------------------------------------------

    $( init );

    /// Functions: -------------------------------------------------------------

    function init() {

        $form =     $( "#search-form" );
        $results =  $( "#search-results" ).append( stationList.getView() );

        $form.on( "change", "select", onChangeTh );
        $form.on( "keydown", "input", onChangeTh );
        $form.on( "change", "input", onChangeTh );
    };

    function onChange() {

        var genre = $form.find( "#search-genre" ).val();
        var query = $form.find( "#search-query" ).val();

        search( query, genre );
    };

    function search( query, genre ) {
        App.debug( "search", query, genre );

        stationList.anounceUpdate();

        if ( !query && !genre ) {
            updateResults( null, [] );
            return;
        } else {
            X.run({
                genre:      X.call( okCb( genres.read, {}), genre ),
                word:       X.call( okCb( words.read, {}), query ),
                ids:        X.callSync( getIds, genre && X.get( "genre" ), query && X.get( "word" )),
                stations:   X.call( stations.listByIds, X.get( "ids" )),
                update:     X.call( updateResults, null, X.get( "stations" )),
            });
        }

        function getIds( genre, word ) {
           
            var no_genre =  _.isEmpty( genre );
            var no_word =   _.isEmpty( word );
            App.debug( "search/getIds", no_genre, genre, no_word, word );

            if ( no_genre && no_word ) {
                return [];
            } else if ( no_word ) {
                return _( genre ).pairs().sortBy( 1 ).pluck( 0 ).value().reverse();
            } else if ( no_genre ) {
                return _( word ).pairs().sortBy( 1 ).pluck( 0 ).value().reverse();
            } else {
                var ids = _.intersection( _.keys( genre ), _.keys( word ));
                App.debug( "search/getIds/both", ids );
                return _.sortBy( ids, countScore ).reverse();
            }

            function countScore( id ) {
                return word[id] + ( 0.1 * genre[id] );
            };
        };
    };

    function updateResults( err, results ) {

        if ( !err && ( !results || !results.length )) {
            stationList.getView().html( NO_RESULTS );
            stationList.endUpdate();
        } else {
            stationList.update( err, results.slice( 0, 20 ));
        }
    };

    /// Utilities: -------------------------------------------------------------

    function okCb( fn, default_value ) {
        return function() {

            var args =  _.toArray( arguments );
            var cb =    args.pop();
            args.push( onFn );
            
            fn.apply( null, args );
            
            function onFn( err, result ) {
                cb( null, result || default_value );
            };
        };
    };


})( window._, window.$, window.App );<commit_msg>Implemented a smarter search algorithm.<commit_after>;(function( _, $, App ){

    /// Constants: -------------------------------------------------------------

    var NO_RESULTS = '<li class="warning">No stations found matching your criteria. Please change your search parameters and try again.</li>';

    /// Variables: -------------------------------------------------------------

    var $form;
    var $results;
    var stationList =   App.StationList();
    var genres =        App.Db.genres;
    var words =         App.Db.words;
    var stations =      App.Db.stations;
    var X =             window.goodJob;
    var onChangeTh =    _.throttle( onChange, 1000, { leading: false, trailing: true });

    /// Exports: ---------------------------------------------------------------

    App.Search = {
        search:         search,
        tokenize:       tokenize,
        filterWords:    filterWords,
        getKeywords:    getKeywords,
    };

    /// Init: ------------------------------------------------------------------

    $( init );

    /// Functions: -------------------------------------------------------------

    function init() {

        $form =     $( "#search-form" );
        $results =  $( "#search-results" ).append( stationList.getView() );

        $form.on( "change", "select", onChangeTh );
        $form.on( "keydown", "input", onChangeTh );
        $form.on( "change", "input", onChangeTh );
    };

    function onChange() {

        var genre = $form.find( "#search-genre" ).val();
        var query = $form.find( "#search-query" ).val();

        search( query, genre );
    };

    function search( query, genre_name ) {
        App.debug( "search", query, genre_name );

        stationList.anounceUpdate();

        if ( !query && !genre_name ) {
            updateResults( null, [] );
            return;
        } else {
            X.run(
                X.options({
                    id:         "search",
                    log:        App.DEBUG,
                    timeout:    8096
                }), {
                    genre:      X.call( okCb( genres.read, {}), genre_name ),
                    keywords:   X.call( getKeywords, query, genre_name ),
                    ids:        X.callSync( getIds, X.get( "genre" ), X.get( "keywords" )),
                    stations:   X.call( stations.listByIds, X.get( "ids" )),
                }, function( e, r ){
                
                    e && console.error( e, r );   
                    updateResults( e, r.stations );
                });
        }

        function getIds( genre, keywords ) {
           
            var no_genre =  _.isEmpty( genre );
            var kwids =     kwIds( keywords );
            var no_words =  _.isEmpty( kwids );
            App.debug( "search/getIds", no_genre, genre, no_words, kwids );

            if ( no_genre && no_words ) {
                return [];
            } else if ( no_words ) {
                return _( genre ).pairs().sortBy( 1 ).pluck( 0 ).value().reverse();
            } else if ( no_genre ) {
                return _( kwids ).sortBy( countScore ).value().reverse();
            } else {
                var ids = _.intersection( _.keys( genre ), kwids );
                App.debug( "search/getIds/both", ids );
                return _.sortBy( ids, countScore ).reverse();
            }

            function countScore( id ) {
                var score = (
                        (keywords.words[id] || 0)
                        + 0.4 * ( keywords.mwords[id] || 0 )
                        + 0.2 * ( keywords.genres[id] || 0 )
                        + 0.1 * ( keywords.mgenres[id] || 0 ));
                App.debug( "countScore", id, score, keywords.words[id], keywords.genres[id], keywords.mwords[id], keywords.mgenres[id] );
                return score;
            };
            function kwIds( keywords ) {

                return _( keywords ).values().map( Object.keys ).flatten().union().value();
            };
        };
    };


    function getKeywords( query, genre_name, cb ) {

        var qwords =    tokenize( query || genre_name );
        App.debug( "getKeywords", query, genre_name, qwords );

        X.run(
            X.options({
                id:         "getKeywords",
                log:        App.DEBUG,
                timeout:    8096
            }), {
                wscores:    X.call( words.listByIds, qwords),
                gscores:    X.call( genres.listByIds, qwords),
                awords:     X.call( words.listIds ),
                mwords:     X.callSync( filterWords, qwords, X.get( "awords")),
                mwscores:   X.call( words.listByIds, X.get( "mwords" )),
                agenres:    X.call( genres.listIds ),
                mgenres:    X.callSync( filterWords, qwords, X.get( "agenres" )),
                mgscores:   X.call( genres.listByIds, X.get( "mgenres" )),

                keywords:   X.callSync( combineResults,
                                X.get( "wscores" ),
                                X.get( "gscores" ),
                                X.get( "mwscores" ),
                                X.get( "mgscores" )),
            }, function( e, r ) {
            
                App.debug( "getKeywords/job-done", e, Object.keys( r ));
                e && console.error( e );
                cb( e, r.keywords );
            });

        function combineResults( words, genres, mwords, mgenres ) {
            var result = {
                words:      toObject( words ), //_.extend.apply( _, words.reverse() ),
                genres:     toObject( genres ), //_.extend.apply( _, genres.reverse() ),
                mwords:     toObject( mwords ), //_.extend.apply( _, mwords.reverse() ),
                mgenres:    toObject( mgenres ),
            };
            App.debug( "getKeywords/combineResults", result );
            return result;

            function toObject( arr ) {
                if ( _.isArray( arr ) && arr.length ) {
                    return _.extend.apply( _, arr.reverse() );
                } else {
                    return {};
                }
            };
        };
    };

    function updateResults( err, results ) {

        if ( !err && ( !results || !results.length )) {
            stationList.getView().html( NO_RESULTS );
            stationList.endUpdate();
        } else {
            stationList.update( err, results.slice( 0, 20 ));
        }
    };

    /// Utilities: -------------------------------------------------------------

    function tokenize( str ) {

        var result = _.filter( str.toLowerCase().split( /[^a-z0-9]+/g ));
        App.debug( "tokenize", str, result );
        return result;
    };

    /**
     *  Returns a list of words that begin with one of qwords.
     */
    function filterWords( qwords, words ) {
        
        var result = _.union.apply( _, qwords.map( getMatching ));
        App.debug( "filterWords", qwords, words.length, result );
        return result;

        function getMatching( qw ) {
            return _.reject( words, function( w ){
                return w.indexOf( qw );
            });
        };
    };
           
    function okCb( fn, default_value ) {
        return function() {

            var args =  _.toArray( arguments );
            var cb =    args.pop();
            args.push( onFn );
            
            fn.apply( null, args );
            
            function onFn( err, result ) {
                cb( null, result || default_value );
            };
        };
    };


})( window._, window.$, window.App );
{'fraction_non_alphanumeric': 0.19648938957296305, 'fraction_numerical': 0.0036678019386953103, 'mean_word_length': 1.170172446465795, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4511684', 'n_tokens_mistral': 3168, 'n_tokens_neox': 2981, 'n_words': 726}