<commit_before>/*
 * Copyright 2017 StreamSets Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.streamsets.pipeline.lib.parser.log;

import com.streamsets.pipeline.api.Field;
import com.streamsets.pipeline.api.ProtoConfigurableEntity;
import com.streamsets.pipeline.api.Record;
import com.streamsets.pipeline.api.impl.Utils;
import com.streamsets.pipeline.api.ext.io.OverrunReader;
import com.streamsets.pipeline.lib.parser.AbstractDataParser;
import com.streamsets.pipeline.lib.parser.DataParserException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;

public abstract class LogCharDataParser extends AbstractDataParser {

  private static final Logger LOG = LoggerFactory.getLogger(LogCharDataParser.class);

  static final String TEXT_FIELD_NAME = "originalLine";
  static final String TRUNCATED_FIELD_NAME = "truncated";
  static final String PARSED_FIELD_NAME = "parsedLine";

  private final ProtoConfigurableEntity.Context context;
  private final String readerId;
  private final OverrunReader reader;
  private final int maxObjectLen;
  private final StringBuilder currentLine;
  private final StringBuilder previousLine;
  private final boolean retainOriginalText;
  private final Map<String, Field> fieldsFromPrevLine;
  private final int maxStackTraceLines;
  private int previousRead;
  private long currentOffset;
  private final GenericObjectPool<StringBuilder> currentLineBuilderPool;
  private final GenericObjectPool<StringBuilder> previousLineBuilderPool;

  public LogCharDataParser(
      ProtoConfigurableEntity.Context context,
      String readerId,
      OverrunReader reader,
      long readerOffset,
      int maxObjectLen,
      boolean retainOriginalText,
      int maxStackTraceLines,
      GenericObjectPool<StringBuilder> currentLineBuilderPool,
      GenericObjectPool<StringBuilder> previousLineBuilderPool
  ) throws IOException {
    this.context = context;
    this.readerId = readerId;
    this.reader = reader;
    this.maxObjectLen = maxObjectLen;
    this.retainOriginalText = retainOriginalText;
    reader.setEnabled(false);
    IOUtils.skipFully(reader, readerOffset);
    reader.setEnabled(true);
    fieldsFromPrevLine = new LinkedHashMap<>();
    currentOffset = readerOffset;
    this.maxStackTraceLines = maxStackTraceLines;

    this.currentLineBuilderPool = currentLineBuilderPool;
    try {
      this.currentLine = currentLineBuilderPool.borrowObject();
      LOG.debug(
          "Borrowed current line string builder from pool. Num Active {}, Num Idle {}",
          this.currentLineBuilderPool.getNumActive(),
          this.currentLineBuilderPool.getNumIdle()
      );
    } catch (Exception e) {
      throw new IOException(
          Utils.format(
              "Error borrowing current line string builder object from pool : {}",
              e.toString()
          ),
          e
      );
    }

    this.previousLineBuilderPool = previousLineBuilderPool;
    try {
      this.previousLine = previousLineBuilderPool.borrowObject();
      LOG.debug(
        "Borrowed previous line string builder from pool. Num Active {}, Num Idle {}",
        this.previousLineBuilderPool.getNumActive(),
        this.previousLineBuilderPool.getNumIdle()
      );
    } catch (Exception e) {
      throw new IOException(
        Utils.format(
          "Error borrowing previous line string builder object from pool : {}",
          e.toString()
        ),
        e
      );
    }
  }

  private boolean isOverMaxObjectLen(int len) {
    return maxObjectLen > -1 && len > maxObjectLen;
  }

  private boolean isTruncated(int len) {
    return isOverMaxObjectLen(len) || truncated;
  }

  @Override
  public Record parse() throws IOException, DataParserException {
    reader.resetCount();

    //In order to detect stack trace / multi line error messages, the parser reads the next line and attempts<commit_msg>SDC-14475: Reset StringBuffer upon acquisition in log parser<commit_after>/*
 * Copyright 2017 StreamSets Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.streamsets.pipeline.lib.parser.log;

import com.streamsets.pipeline.api.Field;
import com.streamsets.pipeline.api.ProtoConfigurableEntity;
import com.streamsets.pipeline.api.Record;
import com.streamsets.pipeline.api.impl.Utils;
import com.streamsets.pipeline.api.ext.io.OverrunReader;
import com.streamsets.pipeline.lib.parser.AbstractDataParser;
import com.streamsets.pipeline.lib.parser.DataParserException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;

public abstract class LogCharDataParser extends AbstractDataParser {

  private static final Logger LOG = LoggerFactory.getLogger(LogCharDataParser.class);

  static final String TEXT_FIELD_NAME = "originalLine";
  static final String TRUNCATED_FIELD_NAME = "truncated";
  static final String PARSED_FIELD_NAME = "parsedLine";

  private final ProtoConfigurableEntity.Context context;
  private final String readerId;
  private final OverrunReader reader;
  private final int maxObjectLen;
  private final StringBuilder currentLine;
  private final StringBuilder previousLine;
  private final boolean retainOriginalText;
  private final Map<String, Field> fieldsFromPrevLine;
  private final int maxStackTraceLines;
  private int previousRead;
  private long currentOffset;
  private final GenericObjectPool<StringBuilder> currentLineBuilderPool;
  private final GenericObjectPool<StringBuilder> previousLineBuilderPool;

  public LogCharDataParser(
      ProtoConfigurableEntity.Context context,
      String readerId,
      OverrunReader reader,
      long readerOffset,
      int maxObjectLen,
      boolean retainOriginalText,
      int maxStackTraceLines,
      GenericObjectPool<StringBuilder> currentLineBuilderPool,
      GenericObjectPool<StringBuilder> previousLineBuilderPool
  ) throws IOException {
    this.context = context;
    this.readerId = readerId;
    this.reader = reader;
    this.maxObjectLen = maxObjectLen;
    this.retainOriginalText = retainOriginalText;
    reader.setEnabled(false);
    IOUtils.skipFully(reader, readerOffset);
    reader.setEnabled(true);
    fieldsFromPrevLine = new LinkedHashMap<>();
    currentOffset = readerOffset;
    this.maxStackTraceLines = maxStackTraceLines;

    this.currentLineBuilderPool = currentLineBuilderPool;
    try {
      this.currentLine = currentLineBuilderPool.borrowObject();
      this.currentLine.setLength(0);
      LOG.debug(
          "Borrowed current line string builder from pool. Num Active {}, Num Idle {}",
          this.currentLineBuilderPool.getNumActive(),
          this.currentLineBuilderPool.getNumIdle()
      );
    } catch (Exception e) {
      throw new IOException(
          Utils.format(
              "Error borrowing current line string builder object from pool : {}",
              e.toString()
          ),
          e
      );
    }

    this.previousLineBuilderPool = previousLineBuilderPool;
    try {
      this.previousLine = previousLineBuilderPool.borrowObject();
      this.previousLine.setLength(0);
      LOG.debug(
        "Borrowed previous line string builder from pool. Num Active {}, Num Idle {}",
        this.previousLineBuilderPool.getNumActive(),
        this.previousLineBuilderPool.getNumIdle()
      );
    } catch (Exception e) {
      throw new IOException(
        Utils.format(
          "Error borrowing previous line string builder object from pool : {}",
          e.toString()
        ),
        e
      );
    }
  }

  private boolean isOverMaxObjectLen(int len) {
    return maxObjectLen > -1 && len > maxObjectLen;
  }

  private boolean isTruncated(int len) {
    return isOverMaxObjectLen(len) || truncated;
  }

  @Override
  public Record parse() throws IOException, DataParserException {
    reader.resetCount();

    //In order to detect stack trace / multi line error messages, the parser reads the next line and attempts
{'fraction_non_alphanumeric': 0.07720351704911002, 'fraction_numerical': 0.003324040317392237, 'mean_word_length': 3.7129863567458314, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4944258', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2447, 'n_words': 712}