<commit_before>/**
 * Licensed to Cloudera, Inc. under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  Cloudera, Inc. licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.cloudera.flume.agent;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import junit.framework.TestCase;

import org.apache.log4j.Logger;
import org.apache.thrift.transport.TTransportException;

import com.cloudera.flume.conf.Context;
import com.cloudera.flume.conf.FlumeBuilder;
import com.cloudera.flume.conf.FlumeConfiguration;
import com.cloudera.flume.conf.SourceFactory;
import com.cloudera.flume.conf.thrift.FlumeConfigData;
import com.cloudera.flume.core.Event;
import com.cloudera.flume.core.EventImpl;
import com.cloudera.flume.core.EventSource;
import com.cloudera.flume.handlers.debug.NullSink;
import com.cloudera.flume.handlers.syslog.SyslogTcpSourceThreads;
import com.cloudera.flume.master.CommandManager;
import com.cloudera.flume.master.ConfigManager;
import com.cloudera.flume.master.ConfigStore;
import com.cloudera.flume.master.FlumeMaster;
import com.cloudera.flume.master.MasterAckManager;
import com.cloudera.flume.master.StatusManager;
import com.cloudera.flume.reporter.ReportEvent;
import com.cloudera.util.Clock;

/**
 * This tests error/exception handling mechanisms in on LogicalNode
 */
public class TestFlumeNode extends TestCase {

  public static Logger LOG = Logger.getLogger(TestFlumeNode.class);

  /**
   * There was a bug -- when source that threw exception when closing twice
   * (should be ok) that prevented progress.
   * 
   * In this, an exception is thrown on any close this would have forced the
   * problem and shows that progress can be made. loadNode only throws if open
   * fails.
   */

  public void testSurviveCloseException() throws IOException {
    LogicalNode node = new LogicalNode(new Context(), "test");

    class CloseExnSource extends EventSource.Base {
      @Override
      public void close() throws IOException {
        throw new IOException("arg!");
      }

      @Override
      public Event next() throws IOException {
        return null;
      }

      @Override
      public void open() throws IOException {
      }
    }
    ;

    node.openLoadNode(new CloseExnSource(), new NullSink());
    node.openLoadNode(new CloseExnSource(), new NullSink());
  }

  public void testRestartNextException() throws Exception {
    LogicalNode node = new LogicalNode(new Context(), "test");

    final AtomicInteger count = new AtomicInteger();

    class NextExnSource extends EventSource.Base {
      @Override
      public void close() throws IOException {
        System.out.println("Next Exn Source closed");
      }

      @Override
      public Event next() throws IOException {
        int i = count.incrementAndGet();
        if (i > 100) {
          return null; // close naturally
        }

        // "fail"
        throw new IOException("ayiii!");
      }

      @Override
      public void open() throws IOException {
        System.out.println("Next Exn Source opened");
      }
    }
    ;

    FlumeBuilder.setSourceFactory(new SourceFactory() {
      public EventSource getSource(String name, String... args) {
        return new NextExnSource();
      }
    });

    // 0 is the first cfg version
    FlumeConfigData cfg = new FlumeConfigData(0, "null", "null", 0, 0,
        "my-test-flow");
    node.loadConfig(cfg); // this will load the NextExnSource and a NullSink

    Map<String, ReportEvent> reports = new HashMap<String, ReportEvent>();
    node.getReports(reports);
    assertEquals(2, reports.size()); // source + sink reports

    // sleep so that we open-append-fail-close, open-append-fail-close
    // multiple times.
    Clock.sleep(5000); // TODO (jon) replace with countdownlatch

    System.out.printf("next called %d times", count.get());
    System.out.flush();
    // node.close(); // this is a differetn testnow

    // successful if open-append-close has been done many times.
    assertTrue(count.get() > 1);<commit_msg>Update the number of reports expected in TestFlumeNode, which has increased by one.<commit_after>/**
 * Licensed to Cloudera, Inc. under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  Cloudera, Inc. licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.cloudera.flume.agent;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import junit.framework.TestCase;

import org.apache.log4j.Logger;
import org.apache.thrift.transport.TTransportException;

import com.cloudera.flume.conf.Context;
import com.cloudera.flume.conf.FlumeBuilder;
import com.cloudera.flume.conf.FlumeConfiguration;
import com.cloudera.flume.conf.SourceFactory;
import com.cloudera.flume.conf.thrift.FlumeConfigData;
import com.cloudera.flume.core.Event;
import com.cloudera.flume.core.EventImpl;
import com.cloudera.flume.core.EventSource;
import com.cloudera.flume.handlers.debug.NullSink;
import com.cloudera.flume.handlers.syslog.SyslogTcpSourceThreads;
import com.cloudera.flume.master.CommandManager;
import com.cloudera.flume.master.ConfigManager;
import com.cloudera.flume.master.ConfigStore;
import com.cloudera.flume.master.FlumeMaster;
import com.cloudera.flume.master.MasterAckManager;
import com.cloudera.flume.master.StatusManager;
import com.cloudera.flume.reporter.ReportEvent;
import com.cloudera.util.Clock;

/**
 * This tests error/exception handling mechanisms in on LogicalNode
 */
public class TestFlumeNode extends TestCase {

  public static Logger LOG = Logger.getLogger(TestFlumeNode.class);

  /**
   * There was a bug -- when source that threw exception when closing twice
   * (should be ok) that prevented progress.
   * 
   * In this, an exception is thrown on any close this would have forced the
   * problem and shows that progress can be made. loadNode only throws if open
   * fails.
   */

  public void testSurviveCloseException() throws IOException {
    LogicalNode node = new LogicalNode(new Context(), "test");

    class CloseExnSource extends EventSource.Base {
      @Override
      public void close() throws IOException {
        throw new IOException("arg!");
      }

      @Override
      public Event next() throws IOException {
        return null;
      }

      @Override
      public void open() throws IOException {
      }
    }
    ;

    node.openLoadNode(new CloseExnSource(), new NullSink());
    node.openLoadNode(new CloseExnSource(), new NullSink());
  }

  public void testRestartNextException() throws Exception {
    LogicalNode node = new LogicalNode(new Context(), "test");

    final AtomicInteger count = new AtomicInteger();

    class NextExnSource extends EventSource.Base {
      @Override
      public void close() throws IOException {
        System.out.println("Next Exn Source closed");
      }

      @Override
      public Event next() throws IOException {
        int i = count.incrementAndGet();
        if (i > 100) {
          return null; // close naturally
        }

        // "fail"
        throw new IOException("ayiii!");
      }

      @Override
      public void open() throws IOException {
        System.out.println("Next Exn Source opened");
      }
    }
    ;

    FlumeBuilder.setSourceFactory(new SourceFactory() {
      public EventSource getSource(String name, String... args) {
        return new NextExnSource();
      }
    });

    // 0 is the first cfg version
    FlumeConfigData cfg = new FlumeConfigData(0, "null", "null", 0, 0,
        "my-test-flow");
    node.loadConfig(cfg); // this will load the NextExnSource and a NullSink

    Map<String, ReportEvent> reports = new HashMap<String, ReportEvent>();
    node.getReports(reports);
    assertEquals(3, reports.size()); // source + sink reports

    // sleep so that we open-append-fail-close, open-append-fail-close
    // multiple times.
    Clock.sleep(5000); // TODO (jon) replace with countdownlatch

    System.out.printf("next called %d times", count.get());
    System.out.flush();
    // node.close(); // this is a differetn testnow

    // successful if open-append-close has been done many times.
    assertTrue(count.get() > 1);
{'fraction_non_alphanumeric': 0.09716684155299056, 'fraction_numerical': 0.003777544596012592, 'mean_word_length': 3.372018348623853, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7586624', 'n_tokens_mistral': 2771, 'n_tokens_neox': 2716, 'n_words': 841}