<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 *
 */

package org.apache.geode.management.internal.cli.commands;

import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.FORMAT;
import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.ONLY_DATE_FORMAT;
import static org.assertj.core.api.Assertions.assertThat;

import org.apache.commons.io.FileUtils;
import org.apache.geode.cache.Cache;
import org.apache.geode.distributed.ConfigurationProperties;
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.logging.LogService;
import org.apache.geode.management.internal.cli.functions.ExportLogsFunction;
import org.apache.geode.management.internal.cli.result.CommandResult;
import org.apache.geode.management.internal.cli.util.CommandStringBuilder;
import org.apache.geode.management.internal.configuration.utils.ZipUtils;
import org.apache.geode.test.dunit.IgnoredException;
import org.apache.geode.test.dunit.rules.GfshShellConnectionRule;
import org.apache.geode.test.dunit.rules.Locator;
import org.apache.geode.test.dunit.rules.LocatorServerStartupRule;
import org.apache.geode.test.dunit.rules.Member;
import org.apache.geode.test.dunit.rules.Server;
import org.apache.geode.test.junit.categories.DistributedTest;
import org.apache.logging.log4j.Logger;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Stream;

@Category(DistributedTest.class)
public class ExportLogsDUnitTest {
  private static final String ERROR_LOG_PREFIX = "[IGNORE]";

  @Rule
  public LocatorServerStartupRule lsRule = new LocatorServerStartupRule();

  @Rule
  public GfshShellConnectionRule gfshConnector = new GfshShellConnectionRule();

  private Locator locator;
  private Server server1;
  private Server server2;

  private Map<Member, List<LogLine>> expectedMessages;

  @Before
  public void setup() throws Exception {
    Properties properties = new Properties();
    properties.setProperty(ConfigurationProperties.LOG_LEVEL, "debug");

    locator = lsRule.startLocatorVM(0, properties);
    server1 = lsRule.startServerVM(1, properties, locator.getPort());
    server2 = lsRule.startServerVM(2, properties, locator.getPort());

    IgnoredException.addIgnoredException(ERROR_LOG_PREFIX);

    expectedMessages = new HashMap<>();
    expectedMessages.put(locator, listOfLogLines(locator, "info", "error", "debug"));
    expectedMessages.put(server1, listOfLogLines(server1, "info", "error", "debug"));
    expectedMessages.put(server2, listOfLogLines(server2, "info", "error", "debug"));

    // log the messages in each of the members
    for (Member member : expectedMessages.keySet()) {
      List<LogLine> logLines = expectedMessages.get(member);

      member.invoke(() -> {
        Logger logger = LogService.getLogger();
        logLines.forEach((LogLine logLine) -> logLine.writeLog(logger));
      });
    }

    gfshConnector.connectAndVerify(locator);
  }

  @Test
  public void startAndEndDateCanIncludeLogs() throws Exception {
    ZonedDateTime now = LocalDateTime.now().atZone(ZoneId.systemDefault());
    ZonedDateTime yesterday = now.minusDays(1);
    ZonedDateTime tomorrow = now.plusDays(1);

    DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(ONLY_DATE_FORMAT);

    CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs");
    commandStringBuilder.addOption("start-time", dateTimeFormatter.format(yesterday));
    commandStringBuilder.addOption("end-time", dateTimeFormatter.format(tomorrow));
    commandStringBuilder.addOption("log-level", "debug");
    commandStringBuilder.addOption("dir", "someDir");

    gfshConnector.executeAndVerifyCommand(commandStringBuilder.toString());

    Set<String> acceptedLogLevels = Stream.of("info", "error", "debug").collect(toSet());
    verifyZipFileContents(acceptedLogLevels);
  }

  @Test
  public void testExportWithStartAndEndDateTimeFiltering() throws Exception {
    ZonedDateTime cutoffTime = LocalDateTime.now().atZone(ZoneId.systemDefault());

    String messageAfterCutoffTime =
        "[this message should not show up since it is after cutoffTime]";
    LogLine logLineAfterCutoffTime = new LogLine(messageAfterCutoffTime, "info", true);
    server1.invoke(() -> {
      Logger logger = LogService.getLogger();
      logLineAfterCutoffTime.writeLog(logger);
    });

    DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(FORMAT);
    String cutoffTimeString = dateTimeFormatter.format(cutoffTime);

    CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs");
    commandStringBuilder.addOption("start-time",
        dateTimeFormatter.format(cutoffTime.minusHours(1)));
    commandStringBuilder.addOption("end-time", cutoffTimeString);
    commandStringBuilder.addOption("log-level", "debug");
    commandStringBuilder.addOption("dir", "someDir");
<commit_msg>GEODE-2267: mark local time sensitive tests as flaky<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 *
 */

package org.apache.geode.management.internal.cli.commands;

import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.FORMAT;
import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.ONLY_DATE_FORMAT;
import static org.assertj.core.api.Assertions.assertThat;

import org.apache.commons.io.FileUtils;
import org.apache.geode.cache.Cache;
import org.apache.geode.distributed.ConfigurationProperties;
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.logging.LogService;
import org.apache.geode.management.internal.cli.functions.ExportLogsFunction;
import org.apache.geode.management.internal.cli.result.CommandResult;
import org.apache.geode.management.internal.cli.util.CommandStringBuilder;
import org.apache.geode.management.internal.configuration.utils.ZipUtils;
import org.apache.geode.test.dunit.IgnoredException;
import org.apache.geode.test.dunit.rules.GfshShellConnectionRule;
import org.apache.geode.test.dunit.rules.Locator;
import org.apache.geode.test.dunit.rules.LocatorServerStartupRule;
import org.apache.geode.test.dunit.rules.Member;
import org.apache.geode.test.dunit.rules.Server;
import org.apache.geode.test.junit.categories.DistributedTest;
import org.apache.geode.test.junit.categories.FlakyTest;
import org.apache.logging.log4j.Logger;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Stream;

@Category(DistributedTest.class)
public class ExportLogsDUnitTest {
  private static final String ERROR_LOG_PREFIX = "[IGNORE]";

  @Rule
  public LocatorServerStartupRule lsRule = new LocatorServerStartupRule();

  @Rule
  public GfshShellConnectionRule gfshConnector = new GfshShellConnectionRule();

  private Locator locator;
  private Server server1;
  private Server server2;

  private Map<Member, List<LogLine>> expectedMessages;

  @Before
  public void setup() throws Exception {
    Properties properties = new Properties();
    properties.setProperty(ConfigurationProperties.LOG_LEVEL, "debug");

    locator = lsRule.startLocatorVM(0, properties);
    server1 = lsRule.startServerVM(1, properties, locator.getPort());
    server2 = lsRule.startServerVM(2, properties, locator.getPort());

    IgnoredException.addIgnoredException(ERROR_LOG_PREFIX);

    expectedMessages = new HashMap<>();
    expectedMessages.put(locator, listOfLogLines(locator, "info", "error", "debug"));
    expectedMessages.put(server1, listOfLogLines(server1, "info", "error", "debug"));
    expectedMessages.put(server2, listOfLogLines(server2, "info", "error", "debug"));

    // log the messages in each of the members
    for (Member member : expectedMessages.keySet()) {
      List<LogLine> logLines = expectedMessages.get(member);

      member.invoke(() -> {
        Logger logger = LogService.getLogger();
        logLines.forEach((LogLine logLine) -> logLine.writeLog(logger));
      });
    }

    gfshConnector.connectAndVerify(locator);
  }

  @Test
  public void startAndEndDateCanIncludeLogs() throws Exception {
    ZonedDateTime now = LocalDateTime.now().atZone(ZoneId.systemDefault());
    ZonedDateTime yesterday = now.minusDays(1);
    ZonedDateTime tomorrow = now.plusDays(1);

    DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(ONLY_DATE_FORMAT);

    CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs");
    commandStringBuilder.addOption("start-time", dateTimeFormatter.format(yesterday));
    commandStringBuilder.addOption("end-time", dateTimeFormatter.format(tomorrow));
    commandStringBuilder.addOption("log-level", "debug");
    commandStringBuilder.addOption("dir", "someDir");

    gfshConnector.executeAndVerifyCommand(commandStringBuilder.toString());

    Set<String> acceptedLogLevels = Stream.of("info", "error", "debug").collect(toSet());
    verifyZipFileContents(acceptedLogLevels);
  }

  @Test
  @Category(FlakyTest.class) // time sensitive
  public void testExportWithStartAndEndDateTimeFiltering() throws Exception {
    ZonedDateTime cutoffTime = LocalDateTime.now().atZone(ZoneId.systemDefault());

    String messageAfterCutoffTime =
        "[this message should not show up since it is after cutoffTime]";
    LogLine logLineAfterCutoffTime = new LogLine(messageAfterCutoffTime, "info", true);
    server1.invoke(() -> {
      Logger logger = LogService.getLogger();
      logLineAfterCutoffTime.writeLog(logger);
    });

    DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(FORMAT);
    String cutoffTimeString = dateTimeFormatter.format(cutoffTime);

    CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs");
    commandStringBuilder.addOption("start-time",
        dateTimeFormatter.format(cutoffTime.minusHours(1)));
    commandStringBuilder.addOption("end-time", cutoffTimeString);
    commandStringBuilder.addOption("log-level", "debug");
    commandStringBuilder.addOption("dir", "someDir");

{'fraction_non_alphanumeric': 0.10468860441372811, 'fraction_numerical': 0.003455587842613681, 'mean_word_length': 5.354291417165669, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1379168', 'n_tokens_mistral': 3608, 'n_tokens_neox': 3533, 'n_words': 802}