<commit_before>package com.gofish.sentiment.newslinker;

import io.vertx.core.Future;
import io.vertx.core.http.HttpClientOptions;
import io.vertx.core.http.HttpMethod;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.core.logging.Logger;
import io.vertx.core.logging.LoggerFactory;
import io.vertx.rx.java.ObservableFuture;
import io.vertx.rx.java.RxHelper;
import io.vertx.rxjava.core.AbstractVerticle;
import io.vertx.rxjava.core.eventbus.MessageConsumer;
import io.vertx.rxjava.core.http.HttpClient;
import io.vertx.rxjava.core.http.HttpClientRequest;
import io.vertx.rxjava.core.http.HttpClientResponse;
import rx.Observable;

import java.util.Optional;
import java.util.concurrent.TimeUnit;

/**
 * @author <NAME>
 */
public class NewsLinkerWorker  extends AbstractVerticle {

    static final String ADDRESS = "sentiment.linker.worker";
    private static final Logger LOG = LoggerFactory.getLogger(NewsLinkerWorker.class);

    private HttpClient httpClient;
    private MessageConsumer<JsonObject> messageConsumer;

    private String apiKey;
    private String baseUrl;
    private String urlPath;
    private Integer port;

    public NewsLinkerWorker() {
        // Vertx requires a default constructor
    }

    public NewsLinkerWorker(HttpClient httpClient) {
        this.httpClient = httpClient;
    }

    @Override
    public void start() throws Exception {
        LOG.info("Bringing up News Linker Worker");

        JsonObject apiConfig = Optional.ofNullable(config().getJsonObject("api"))
                .orElseThrow(() -> new RuntimeException("Could not load linker configuration"));

        apiKey = apiConfig.getString("key", "");
        baseUrl = apiConfig.getString("base.url", "");
        urlPath = apiConfig.getString("url.path", "");
        port = apiConfig.getInteger("port", 443);
        httpClient = Optional.ofNullable(httpClient).orElseGet(() -> vertx.createHttpClient(getHttpClientOptions()));

        messageConsumer = vertx.eventBus().localConsumer(ADDRESS, messageHandler -> {
            try {
                final JsonObject article = messageHandler.body().getJsonObject("article");
                final String text = String.join(". ", article.getString("name"), article.getString("description"));

                LOG.info("Calling Entity Linking API");

                HttpClientRequest request = httpClient.request(HttpMethod.POST, port, baseUrl, urlPath)
                        .putHeader("Content-Type", "text/plain; charset=UTF-8")
                        .putHeader("Content-Length", String.valueOf(text.length()))
                        .putHeader("Ocp-Apim-Subscription-Key", apiKey);

                linkNewsArticleEntities(request)
                        .flatMap(result -> this.addNewEntities(article, result))
                        .subscribe(
                                result -> messageHandler.reply(result),
                                failure -> messageHandler.fail(1, failure.getMessage()),
                                () -> request.end()
                        );

                request.write(text);
            }
            catch (Throwable t) {
                messageHandler.fail(2, "Invalid Request");
            }
        });
    }

    private Observable<JsonObject> linkNewsArticleEntities(HttpClientRequest request) {
        return request.toObservable()
                .flatMap(this::bodyHandlerObservable)
                .switchMap(json -> {
                    switch(("" + json.getInteger("statusCode", 0)).charAt(0)) {
                        case '4':
                        case '5':
                            return Observable.error(new Throwable(json.getString("message")));
                        default:
                            return Observable.just(json);
                    }
                })<commit_msg>Offloaded content-length calculations to vertx for NewsLinkerWorker http client requests.<commit_after>package com.gofish.sentiment.newslinker;

import io.vertx.core.Future;
import io.vertx.core.http.HttpClientOptions;
import io.vertx.core.http.HttpMethod;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.core.logging.Logger;
import io.vertx.core.logging.LoggerFactory;
import io.vertx.rx.java.ObservableFuture;
import io.vertx.rx.java.RxHelper;
import io.vertx.rxjava.core.AbstractVerticle;
import io.vertx.rxjava.core.eventbus.MessageConsumer;
import io.vertx.rxjava.core.http.HttpClient;
import io.vertx.rxjava.core.http.HttpClientRequest;
import io.vertx.rxjava.core.http.HttpClientResponse;
import rx.Observable;

import java.util.Optional;
import java.util.concurrent.TimeUnit;

/**
 * @author <NAME>
 */
public class NewsLinkerWorker  extends AbstractVerticle {

    static final String ADDRESS = "sentiment.linker.worker";
    private static final Logger LOG = LoggerFactory.getLogger(NewsLinkerWorker.class);

    private HttpClient httpClient;
    private MessageConsumer<JsonObject> messageConsumer;

    private String apiKey;
    private String baseUrl;
    private String urlPath;
    private Integer port;

    public NewsLinkerWorker() {
        // Vertx requires a default constructor
    }

    public NewsLinkerWorker(HttpClient httpClient) {
        this.httpClient = httpClient;
    }

    @Override
    public void start() throws Exception {
        LOG.info("Bringing up News Linker Worker");

        JsonObject apiConfig = Optional.ofNullable(config().getJsonObject("api"))
                .orElseThrow(() -> new RuntimeException("Could not load linker configuration"));

        apiKey = apiConfig.getString("key", "");
        baseUrl = apiConfig.getString("base.url", "");
        urlPath = apiConfig.getString("url.path", "");
        port = apiConfig.getInteger("port", 443);
        httpClient = Optional.ofNullable(httpClient).orElseGet(() -> vertx.createHttpClient(getHttpClientOptions()));

        messageConsumer = vertx.eventBus().localConsumer(ADDRESS, messageHandler -> {
            try {
                final JsonObject article = messageHandler.body().getJsonObject("article");
                final String text = String.join(". ", article.getString("name"), article.getString("description"));

                LOG.info("Calling Entity Linking API");

                HttpClientRequest request = httpClient.request(HttpMethod.POST, port, baseUrl, urlPath)
                        .putHeader("Content-Type", "text/plain; charset=UTF-8")
                        .putHeader("Ocp-Apim-Subscription-Key", apiKey);

                linkNewsArticleEntities(request)
                        .flatMap(result -> this.addNewEntities(article, result))
                        .subscribe(
                                result -> messageHandler.reply(result),
                                failure -> messageHandler.fail(1, failure.getMessage()),
                                () -> LOG.info("Finished News Linking")
                        );

                request.end(text);
            }
            catch (Throwable t) {
                messageHandler.fail(2, "Invalid Request");
            }
        });
    }

    private Observable<JsonObject> linkNewsArticleEntities(HttpClientRequest request) {
        return request.toObservable()
                .flatMap(this::bodyHandlerObservable)
                .switchMap(json -> {
                    switch(("" + json.getInteger("statusCode", 0)).charAt(0)) {
                        case '4':
                        case '5':
                            return Observable.error(new Throwable(json.getString("message")));
                        default:
                            return Observable.just(json);
                    }
                })
{'fraction_non_alphanumeric': 0.109837754313675, 'fraction_numerical': 0.0025753283543651817, 'mean_word_length': 2.2966893039049237, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5015256', 'n_tokens_mistral': 2046, 'n_tokens_neox': 1966, 'n_words': 383}