<commit_before>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2011, <NAME>, <NAME>.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package org.carrot2.source.boss;

import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.carrot2.core.Document;
import org.carrot2.core.LanguageCode;
import org.carrot2.source.SearchEngineResponse;
import org.simpleframework.xml.*;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

/**
 * Search response model for Yahoo Boss.
 */
@Root(name = "ysearchresponse", strict = false)
final class YSearchResponse
{
    @Attribute(name = "responsecode", required = false)
    public Integer responseCode;

    @Element(name = "nextpage", required = false)
    public String nextPageURI;

    @Element(name = "resultset_web", required = false)
    public WebResultSet webResultSet;

    @Element(name = "resultset_news", required = false)
    public NewsResultSet newsResultSet;

    @Element(name = "resultset_images", required = false)
    public ImagesResultSet imagesResultSet;
    
    @Element(name = "language", required = false)
    public String language;

    /**
     * Populate {@link SearchEngineResponse} depending on the type of the search result
     * returned.
     * 
     * @param response
     * @param requestedLanguage the language requested by the user, mapped from
     *            {@link BossLanguageCodes} to {@link LanguageCode}.
     */
    public void populate(SearchEngineResponse response, LanguageCode requestedLanguage)
    {
        if (webResultSet != null)
        {
            response.metadata.put(SearchEngineResponse.RESULTS_TOTAL_KEY,
                webResultSet.deephits);

            if (webResultSet.results != null)
            {
                for (WebResult result : webResultSet.results)
                {
                    final Document document = new Document(result.title, result.summary,
                        result.url);

                    document.setField(Document.CLICK_URL, result.clickURL);

                    try
                    {
                        document.setField(Document.SIZE, Long.parseLong(result.size));
                    }
                    catch (NumberFormatException e)
                    {
                        // Ignore if cannot parse.
                    }

                    response.results.add(document);
                }
            }
        }
        else if (newsResultSet != null)
        {
            response.metadata.put(SearchEngineResponse.RESULTS_TOTAL_KEY,
                newsResultSet.deephits);

            if (newsResultSet.results != null)
            {
                final Set<String> unknownLanguages = Sets.newHashSet();
                for (NewsResult result : newsResultSet.results)
                {
                    final Document document = new Document(result.title, result.summary,
                        result.url);

                    document.setField(Document.CLICK_URL, result.clickURL);
                    if (StringUtils.isNotBlank(result.source))
                    {
                        document.setField(Document.SOURCES, Lists.newArrayList(result.source));
                    }
                    
                    // BOSS news returns language name as a string, but there is no list
                    // of supported values in the documentation. It seems that the strings
                    // are parallel to LanguageCode enum names, so we use them here.
                    if (StringUtils.isNotBlank(result.language))
                    {
                        try
                        {
                            document.setLanguage(LanguageCode.valueOf(result.language));
                        }
                        catch (IllegalArgumentException ignored)
                        {
                            unknownLanguages.add(result.language);
                        }
                    }
                    
                    response.results.add(document);
                }
                <commit_msg>carrot2-source-boss: CARROT-812: fixed, looks like BOSS news API changed the case of language strings<commit_after>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2011, <NAME>, <NAME>.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package org.carrot2.source.boss;

import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.carrot2.core.Document;
import org.carrot2.core.LanguageCode;
import org.carrot2.source.SearchEngineResponse;
import org.simpleframework.xml.*;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

/**
 * Search response model for Yahoo Boss.
 */
@Root(name = "ysearchresponse", strict = false)
final class YSearchResponse
{
    @Attribute(name = "responsecode", required = false)
    public Integer responseCode;

    @Element(name = "nextpage", required = false)
    public String nextPageURI;

    @Element(name = "resultset_web", required = false)
    public WebResultSet webResultSet;

    @Element(name = "resultset_news", required = false)
    public NewsResultSet newsResultSet;

    @Element(name = "resultset_images", required = false)
    public ImagesResultSet imagesResultSet;
    
    @Element(name = "language", required = false)
    public String language;

    /**
     * Populate {@link SearchEngineResponse} depending on the type of the search result
     * returned.
     * 
     * @param response
     * @param requestedLanguage the language requested by the user, mapped from
     *            {@link BossLanguageCodes} to {@link LanguageCode}.
     */
    public void populate(SearchEngineResponse response, LanguageCode requestedLanguage)
    {
        if (webResultSet != null)
        {
            response.metadata.put(SearchEngineResponse.RESULTS_TOTAL_KEY,
                webResultSet.deephits);

            if (webResultSet.results != null)
            {
                for (WebResult result : webResultSet.results)
                {
                    final Document document = new Document(result.title, result.summary,
                        result.url);

                    document.setField(Document.CLICK_URL, result.clickURL);

                    try
                    {
                        document.setField(Document.SIZE, Long.parseLong(result.size));
                    }
                    catch (NumberFormatException e)
                    {
                        // Ignore if cannot parse.
                    }

                    response.results.add(document);
                }
            }
        }
        else if (newsResultSet != null)
        {
            response.metadata.put(SearchEngineResponse.RESULTS_TOTAL_KEY,
                newsResultSet.deephits);

            if (newsResultSet.results != null)
            {
                final Set<String> unknownLanguages = Sets.newHashSet();
                for (NewsResult result : newsResultSet.results)
                {
                    final Document document = new Document(result.title, result.summary,
                        result.url);

                    document.setField(Document.CLICK_URL, result.clickURL);
                    if (StringUtils.isNotBlank(result.source))
                    {
                        document.setField(Document.SOURCES, Lists.newArrayList(result.source));
                    }
                    
                    // BOSS news returns language name as a string, but there is no list
                    // of supported values in the documentation. It seems that the strings
                    // are parallel to LanguageCode enum names, so we use them here.
                    if (StringUtils.isNotBlank(result.language))
                    {
                        try
                        {
                            document.setLanguage(LanguageCode.valueOf(result.language.toUpperCase()));
                        }
                        catch (IllegalArgumentException ignored)
                        {
                            unknownLanguages.add(result.language);
                        }
                    }
                    
                    response.results.add(document);
                }
                
{'fraction_non_alphanumeric': 0.0830607476635514, 'fraction_numerical': 0.004205607476635514, 'mean_word_length': 1.6180428134556575, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1794794', 'n_tokens_mistral': 2159, 'n_tokens_neox': 2014, 'n_words': 501}