<commit_before>/*
 * Scalyr client library
 * Copyright 2012 Scalyr, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.scalyr.api.tests;

import com.scalyr.api.LogHook;
import com.scalyr.api.internal.Logging;
import com.scalyr.api.internal.ScalyrUtil;
import com.scalyr.api.internal.SimpleRateLimiter;
import com.scalyr.api.knobs.ConfigurationFile;
import com.scalyr.api.knobs.ConfigurationFileFactory;
import com.scalyr.api.knobs.Knob;
import com.scalyr.api.knobs.LocalConfigurationFile;
import com.scalyr.api.logs.Severity;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.HashSet;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;

import java.util.HashMap;
import com.scalyr.api.Converter;

import static org.junit.Assert.*;


/**
 * Tests for Knob.
 */
public class KnobTest extends KnobTestBase {
  private File paramDir;
  private ConfigurationFileFactory paramFactory;

  @Before @Override public void setup() {
    super.setup();

    paramDir = TestUtils.createTemporaryDirectory();
    paramFactory = ConfigurationFile.makeLocalFileFactory(paramDir, 100);
  }

  @After public void teardownKnobTest() {
    LocalConfigurationFile.useLastKnownGoodJson = true;

    ScalyrUtil.recreateAsyncApiExecutor();

    TestUtils.recursiveDelete(paramDir);
  }

  /**
   * Simple test of reading a knob file. Exercises some of our extensions to the JSON format --
   * comments, optional quoting on attributenames, etc.
   */
  @Test public void testJsonExtensions() {
    // Publish, and verify, a simple file.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt'}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 1, 'createDate': 1000, 'modDate': 2000," +
            "'content': '{foo: \\'abc\\', /*comment*/ \\'bar\\': \\'xyz\\'}'}");

    ConfigurationFile paramFile = factory.getFile("/foo.txt");
    Knob.String valueFoo = new Knob.String("foo", "fooDefault", paramFile);
    Knob.String valueBar = new Knob.String("bar", "barDefault", paramFile);
    Knob.String valueBaz = new Knob.String("baz", "bazDefault", paramFile);

    assertEquals("abc",        valueFoo.get());
    assertEquals("xyz",        valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    assertRequestQueueEmpty();
  }

  /**
   * Straightforward test where we read values from a file and update the file.
   */
  @Test public void testUpdates() {
    // Publish, and verify, an initial version of the file.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt'}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 1, 'createDate': 1000, 'modDate': 2000," +
            "'content': '{\\'foo\\': \\'abc\\', \\'bar\\': \\'xyz\\'}'}");

    ConfigurationFile paramFile = factory.getFile("/foo.txt");
    Knob.String valueFoo = new Knob.String("foo", "fooDefault", paramFile);
    Knob.String valueBar = new Knob.String("bar", "barDefault", paramFile);
    Knob.String valueBaz = new Knob.String("baz", "bazDefault", paramFile);

    assertEquals("abc", valueFoo.get());
    assertEquals("xyz", valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    TestListener fooListener = new TestListener();
    valueFoo.addUpdateListener(fooListener);
    assertNull(fooListener.value);

    TestListener barListener = new TestListener();
    valueBar.addUpdateListener(barListener);
    assertNull(barListener.value);

    TestListener bazListener = new TestListener();
    valueBaz.addUpdateListener(bazListener);
    assertNull(bazListener.value);

    // Publish a second version, pause to ensure it's picked up, and then verify it.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 1}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 2, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'abc2\\', \\'bar\\': \\'xyz\\'}'}");

    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    assertEquals("abc2",       valueFoo.get());
    assertEquals("xyz",        valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    assertEquals("abc2", fooListener.value);
    assertEquals(null,   barListener.value);
    assertEquals(null,   bazListener.value);
    fooListener.value = null;

    valueFoo.removeUpdateListener(fooListener);

    // Publish a third version.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 2}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 3, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'abc3\\', \\'baz\\': \\'pdq\\'}'}");

    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    assertEquals("abc3",       valueFoo.get());
    assertEquals("barDefault", valueBar.get());
    assertEquals("pdq",        valueBaz.get());

    assertEquals(null,         fooListener.value);
    assertEquals("barDefault", barListener.value);
    assertEquals("pdq",        bazListener.value);
    barListener.value = null;
    bazListener.value = null;

    // Publish a fourth, non-parseable version.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 3}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 4, 'createDate': 1000, 'modDate': 3000," +
            "'content': 'blah'}");

    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    assertEquals("fooDefault", valueFoo.get());
    assertEquals("barDefault", valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    assertEquals(null,         fooListener.value);
    assertEquals(null,         barListener.value);
    assertEquals("bazDefault", bazListener.value);
    bazListener.value = null;

    assertRequestQueueEmpty();
  }

  /**
   * Test that Converter.java calls are only made for SI Knobs when files are updated, not on every get().
   */

  public static int converterAccessCount;
  @Test public void testConverterCalls() {
    converterAccessCount = 0;
    // Initial file
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt'}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 1, 'createDate': 1000, 'modDate': 2000," +
            "'content': '{\\'foo\\': \\'100K\\'}'}");

    ConfigurationFile paramFile = factory.getFile("/foo.txt");
    Knob fooKnob = new TestLongKnob("foo", 404L, paramFile);

    // Dummy calls
    assertEquals(100000L, (long) fooKnob.get());
    for (int i = 0; i < 200; i++) {
      fooKnob.get();
    }

    // Update the file
    expectRequest(
        "getFile",
        "{'token': 'dummy<PASSWORD>', 'path': '/foo.txt', 'expectedVersion': 1}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 2, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'200K\\'}'}");
    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    // Dummy calls
    assertEquals(200000L, (long) fooKnob.get());
    for (int i = 0; i < 10; i++) {
      fooKnob.get();
    }

    // Update the file again
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 2}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 3, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'300K\\'}'}");
    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    // Dummy calls
    assertEquals(300000L, (long) fooKnob.get());
    for (int i = 0; i < 5; i++) {
      fooKnob.get();
    }

    // Ensure that after the 100-call threshold for caching, we only called Converter on the 2 updates.
    assertEquals(102, converterAccessCount);
  }

  public class TestLongKnob extends Knob {
    public TestLongKnob(java.lang.String valueKey, java.lang.Long defaultValue, ConfigurationFile ... files) {
      super(valueKey, defaultValue, KnobTest::parseNanosForTests, files);
    }
  }

  public static java.lang.Long parseNanosForTests(Object value) {
    converterAccessCount++;
    return Converter.toLongWithSI(value);
  }

  /**
   * Test of layering multiple files.
   */
  @Test public void testLayering() throws IOException, InterruptedException {
    LocalConfigurationFile.useLastKnownGoodJson = false;

    Knob.setDefaultFiles(new ConfigurationFile[0]);

    // Set up a pair of parameter files.
    LocalConfigurationFileTest.createOrUpdateFile(paramDir, "foo", "{\"x\": 10}");
    LocalConfigurationFileTest.createOrUpdateFile(paramDir, "bar", "{\"x\": 20, \"y\": 100}");


    ConfigurationFile fileFoo = paramFactory.getFile("/foo");
    ConfigurationFile fileBar = paramFactory.getFile("/bar");
    ConfigurationFile fileBaz = paramFactory.getFile("/baz");

    Knob.Integer value1 = new Knob.Integer("x", 1);<commit_msg>Changed wrong name of helper method in test<commit_after>/*
 * Scalyr client library
 * Copyright 2012 Scalyr, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.scalyr.api.tests;

import com.scalyr.api.LogHook;
import com.scalyr.api.internal.Logging;
import com.scalyr.api.internal.ScalyrUtil;
import com.scalyr.api.internal.SimpleRateLimiter;
import com.scalyr.api.knobs.ConfigurationFile;
import com.scalyr.api.knobs.ConfigurationFileFactory;
import com.scalyr.api.knobs.Knob;
import com.scalyr.api.knobs.LocalConfigurationFile;
import com.scalyr.api.logs.Severity;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.HashSet;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;

import java.util.HashMap;
import com.scalyr.api.Converter;

import static org.junit.Assert.*;


/**
 * Tests for Knob.
 */
public class KnobTest extends KnobTestBase {
  private File paramDir;
  private ConfigurationFileFactory paramFactory;

  @Before @Override public void setup() {
    super.setup();

    paramDir = TestUtils.createTemporaryDirectory();
    paramFactory = ConfigurationFile.makeLocalFileFactory(paramDir, 100);
  }

  @After public void teardownKnobTest() {
    LocalConfigurationFile.useLastKnownGoodJson = true;

    ScalyrUtil.recreateAsyncApiExecutor();

    TestUtils.recursiveDelete(paramDir);
  }

  /**
   * Simple test of reading a knob file. Exercises some of our extensions to the JSON format --
   * comments, optional quoting on attributenames, etc.
   */
  @Test public void testJsonExtensions() {
    // Publish, and verify, a simple file.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt'}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 1, 'createDate': 1000, 'modDate': 2000," +
            "'content': '{foo: \\'abc\\', /*comment*/ \\'bar\\': \\'xyz\\'}'}");

    ConfigurationFile paramFile = factory.getFile("/foo.txt");
    Knob.String valueFoo = new Knob.String("foo", "fooDefault", paramFile);
    Knob.String valueBar = new Knob.String("bar", "barDefault", paramFile);
    Knob.String valueBaz = new Knob.String("baz", "bazDefault", paramFile);

    assertEquals("abc",        valueFoo.get());
    assertEquals("xyz",        valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    assertRequestQueueEmpty();
  }

  /**
   * Straightforward test where we read values from a file and update the file.
   */
  @Test public void testUpdates() {
    // Publish, and verify, an initial version of the file.
    expectRequest(
        "getFile",
        "{'token': '<PASSWORD>', 'path': '/foo.txt'}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 1, 'createDate': 1000, 'modDate': 2000," +
            "'content': '{\\'foo\\': \\'abc\\', \\'bar\\': \\'xyz\\'}'}");

    ConfigurationFile paramFile = factory.getFile("/foo.txt");
    Knob.String valueFoo = new Knob.String("foo", "fooDefault", paramFile);
    Knob.String valueBar = new Knob.String("bar", "barDefault", paramFile);
    Knob.String valueBaz = new Knob.String("baz", "bazDefault", paramFile);

    assertEquals("abc", valueFoo.get());
    assertEquals("xyz", valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    TestListener fooListener = new TestListener();
    valueFoo.addUpdateListener(fooListener);
    assertNull(fooListener.value);

    TestListener barListener = new TestListener();
    valueBar.addUpdateListener(barListener);
    assertNull(barListener.value);

    TestListener bazListener = new TestListener();
    valueBaz.addUpdateListener(bazListener);
    assertNull(bazListener.value);

    // Publish a second version, pause to ensure it's picked up, and then verify it.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 1}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 2, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'abc2\\', \\'bar\\': \\'xyz\\'}'}");

    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    assertEquals("abc2",       valueFoo.get());
    assertEquals("xyz",        valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    assertEquals("abc2", fooListener.value);
    assertEquals(null,   barListener.value);
    assertEquals(null,   bazListener.value);
    fooListener.value = null;

    valueFoo.removeUpdateListener(fooListener);

    // Publish a third version.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 2}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 3, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'abc3\\', \\'baz\\': \\'pdq\\'}'}");

    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    assertEquals("abc3",       valueFoo.get());
    assertEquals("barDefault", valueBar.get());
    assertEquals("pdq",        valueBaz.get());

    assertEquals(null,         fooListener.value);
    assertEquals("barDefault", barListener.value);
    assertEquals("pdq",        bazListener.value);
    barListener.value = null;
    bazListener.value = null;

    // Publish a fourth, non-parseable version.
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 3}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 4, 'createDate': 1000, 'modDate': 3000," +
            "'content': 'blah'}");

    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    assertEquals("fooDefault", valueFoo.get());
    assertEquals("barDefault", valueBar.get());
    assertEquals("bazDefault", valueBaz.get());

    assertEquals(null,         fooListener.value);
    assertEquals(null,         barListener.value);
    assertEquals("bazDefault", bazListener.value);
    bazListener.value = null;

    assertRequestQueueEmpty();
  }

  /**
   * Test that Converter.java calls are only made for SI Knobs when files are updated, not on every get().
   */

  public static int converterAccessCount;
  @Test public void testConverterCalls() {
    converterAccessCount = 0;
    // Initial file
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt'}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 1, 'createDate': 1000, 'modDate': 2000," +
            "'content': '{\\'foo\\': \\'100K\\'}'}");

    ConfigurationFile paramFile = factory.getFile("/foo.txt");
    Knob fooKnob = new TestLongKnob("foo", 404L, paramFile);

    // Dummy calls
    assertEquals(100000L, (long) fooKnob.get());
    for (int i = 0; i < 200; i++) {
      fooKnob.get();
    }

    // Update the file
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 1}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 2, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'200K\\'}'}");
    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    // Dummy calls
    assertEquals(200000L, (long) fooKnob.get());
    for (int i = 0; i < 10; i++) {
      fooKnob.get();
    }

    // Update the file again
    expectRequest(
        "getFile",
        "{'token': 'dummyToken', 'path': '/foo.txt', 'expectedVersion': 2}",
        "{'status': 'success', 'path': '/foo.txt', 'version': 3, 'createDate': 1000, 'modDate': 3000," +
            "'content': '{\\'foo\\': \\'300K\\'}'}");
    try {
      Thread.sleep(1000); // must sleep for longer than HostedParameterFile's minimum inter-request delay
    } catch (InterruptedException ex) {
      ex.printStackTrace();
    }

    // Dummy calls
    assertEquals(300000L, (long) fooKnob.get());
    for (int i = 0; i < 5; i++) {
      fooKnob.get();
    }

    // Ensure that after the 100-call threshold for caching, we only called Converter on the 2 updates.
    assertEquals(102, converterAccessCount);
  }

  public class TestLongKnob extends Knob {
    public TestLongKnob(java.lang.String valueKey, java.lang.Long defaultValue, ConfigurationFile ... files) {
      super(valueKey, defaultValue, KnobTest::toLongWithSIForTests, files);
    }
  }

  public static java.lang.Long toLongWithSIForTests(Object value) {
    converterAccessCount++;
    return Converter.toLongWithSI(value);
  }

  /**
   * Test of layering multiple files.
   */
  @Test public void testLayering() throws IOException, InterruptedException {
    LocalConfigurationFile.useLastKnownGoodJson = false;

    Knob.setDefaultFiles(new ConfigurationFile[0]);

    // Set up a pair of parameter files.
    LocalConfigurationFileTest.createOrUpdateFile(paramDir, "foo", "{\"x\": 10}");
    LocalConfigurationFileTest.createOrUpdateFile(paramDir, "bar", "{\"x\": 20, \"y\": 100}");


    ConfigurationFile fileFoo = paramFactory.getFile("/foo");
    ConfigurationFile fileBar = paramFactory.getFile("/bar");
    ConfigurationFile fileBaz = paramFactory.getFile("/baz");

    Knob.Integer value1 = new Knob.Integer("x", 1);
{'fraction_non_alphanumeric': 0.15675229446899433, 'fraction_numerical': 0.016510464721021707, 'mean_word_length': 3.359441151566469, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4142496', 'n_tokens_mistral': 6472, 'n_tokens_neox': 6025, 'n_words': 1588}