<commit_before>class ArticleSearch < Search
    require 'erb'
    include ERB::Util
    attr_reader :articles, :facets, :search_opts

    def enough_data_exists_in(record)
        begin
            if record['PLink'] and record['RecordInfo']['BibRecord']['BibEntity']['Titles'].first['TitleFull'] and record['Header']['DbId'] and record['Header']['An']
                return true
            else
                return false
            end
        rescue NoMethodError
            return false
        end
    end


    def initialize(q, page, requested_facets = [], api_connection)
        @q = q
        @page = page
        @api_connection = api_connection
        @articles = Array.new
        @requested_facets = requested_facets.to_a
        @facets = Array.new
    end

    def search_opts
        search_opts = String.new
        search_opts.concat('query-1=' + url_encode(@q))
        search_opts.concat('&limiter=FT:y')
        search_opts.concat('&resultsperpage=10')
        search_opts.concat('&view=detailed')
        search_opts.concat('&facetfilter=' + url_encode('1, SourceType:Academic Journals,SourceType:News'))
        i = 2
        @requested_facets.each do |facet|
            search_opts.concat('&facetfilter=' + url_encode(i.to_s + ', ' + facet))
            i = i + 1
        end
        search_opts.concat('&pagenumber=' + @page.to_s)
        search_opts.concat('&highlight=n')
        return search_opts
    end

    def send_search
        records = Array.new

        if @api_connection.show_session_token and @api_connection.show_auth_token
            begin
                results = @api_connection.search search_opts, @api_connection.show_session_token, @api_connection.show_auth_token
            rescue ActionView::Template::Error, Net::ReadTimeout
                logger.debug "EDS timed out"
            end
            if results.any?
                begin
                    total_results_count = results['SearchResult']['Statistics']['TotalHits']
                    list_of_records = results['SearchResult']['Data']['Records']
                rescue ActionView::Template::Error
                    total_results_count = 0
                end
                unless list_of_records.nil? or total_results_count < 1
                    list_of_records.each do |record|
                        if enough_data_exists_in record
                            current_article = Article.new
                            current_article.extract_data_from_api_response record
                            records.push current_article
                        end
                    end
                end
            else
                total_records_count = 0
            end
            @articles = Kaminari.paginate_array(records, total_count: total_results_count).page(@page).per(10)

	    if results['SearchResult']['AvailableFacets'].respond_to? :each
                results['SearchResult']['AvailableFacets'].each do |facet|
                    tmp = ArticleFacet.new facet['Label']
                    facet['AvailableFacetValues'].each do |value|
                        tmp.add_value value['Value'], value['AddAction'].sub('addfacetfilter(', '').chop, value['Count']
                    end
                    @facets.push(tmp)
                end
	    end
        end
    end

end<commit_msg>handling incomplete data from EDS<commit_after>class ArticleSearch < Search
    require 'erb'
    include ERB::Util
    attr_reader :articles, :facets, :search_opts

    def enough_data_exists_in(record)
        begin
            if record['PLink'] and record['RecordInfo']['BibRecord']['BibEntity']['Titles'].first['TitleFull'] and record['Header']['DbId'] and record['Header']['An']
                return true
            else
                return false
            end
        rescue NoMethodError
            return false
        end
    end


    def initialize(q, page, requested_facets = [], api_connection)
        @q = q
        @page = page
        @api_connection = api_connection
        @articles = Array.new
        @requested_facets = requested_facets.to_a
        @facets = Array.new
    end

    def search_opts
        search_opts = String.new
        search_opts.concat('query-1=' + url_encode(@q))
        search_opts.concat('&limiter=FT:y')
        search_opts.concat('&resultsperpage=10')
        search_opts.concat('&view=detailed')
        search_opts.concat('&facetfilter=' + url_encode('1, SourceType:Academic Journals,SourceType:News'))
        i = 2
        @requested_facets.each do |facet|
            search_opts.concat('&facetfilter=' + url_encode(i.to_s + ', ' + facet))
            i = i + 1
        end
        search_opts.concat('&pagenumber=' + @page.to_s)
        search_opts.concat('&highlight=n')
        return search_opts
    end

    def send_search
        records = Array.new

        if @api_connection.show_session_token and @api_connection.show_auth_token
            begin
                results = @api_connection.search search_opts, @api_connection.show_session_token, @api_connection.show_auth_token
            rescue ActionView::Template::Error, Net::ReadTimeout
                logger.debug "EDS timed out"
            end
            if results.any?
                begin
                    total_results_count = results['SearchResult']['Statistics']['TotalHits']
                    list_of_records = results['SearchResult']['Data']['Records']
                rescue 
                    total_results_count = 0
                end
                unless list_of_records.nil? or total_results_count < 1
                    list_of_records.each do |record|
                        if enough_data_exists_in record
                            current_article = Article.new
                            current_article.extract_data_from_api_response record
                            records.push current_article
                        end
                    end
                end
            else
                total_records_count = 0
            end
            @articles = Kaminari.paginate_array(records, total_count: total_results_count).page(@page).per(10)

	    begin
	        if results['SearchResult']['AvailableFacets'].respond_to? :each
                    results['SearchResult']['AvailableFacets'].each do |facet|
                        tmp = ArticleFacet.new facet['Label']
                        facet['AvailableFacetValues'].each do |value|
                            tmp.add_value value['Value'], value['AddAction'].sub('addfacetfilter(', '').chop, value['Count']
                        end
                        @facets.push(tmp)
                    end
	        end
	    rescue
                logger.debug "EDS returned no facets"
	    end
        end
    end

end
{'fraction_non_alphanumeric': 0.09321160359299073, 'fraction_numerical': 0.0032395817994404358, 'mean_word_length': 1.7870332375871973, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5258427', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1740, 'n_words': 348}