<commit_before>// HdfsIO.java
//
// Copyright (C) 2011 Splunk Inc.
//
// Splunk Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.splunk.shep.s2s.forwarder;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.net.URI;

import javax.management.MBeanServer;
import javax.management.ObjectName;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.LineReader;
import org.apache.log4j.Logger;

import com.splunk.shep.s2s.DataSink;

public class HDFSSink implements DataSink {
    public static final long HadoopFileSize = 63000000;
    public static final int MaxDataLoggingSize = 100000;
    private String ip = new String("localhost");
    private String port = new String("54310");
    private String path = new String("/spl");

    private Configuration conf = null;
    private FileSystem fileSystem = null;
    private Path destination = null;
    private FSDataOutputStream ofstream = null;
    private FSDataInputStream ifstream = null;

    private boolean useAppend = false;
    private boolean reopenedForWriting = false;
    private long totalBytesWritten = 0; // total bytes with current connection.
    private long fileRollingSize = HadoopFileSize;
    private long fileRollingDuration = 30000; // roll to new file every 30 sec.
    private long timeOpened = 0; // milliseconds
    private long maxEventSize = 32000; // not supported for now.
    private String currentFilePath = new String("");
    private String myname;

    private static Logger logger = Logger.getLogger(HDFSSink.class);

    public HDFSSink() {
    }
    
    // only when my name gets set I can init myself
    @Override
    public void setName(String name) {
	this.myname = name;
	try {
	    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

	    // Construct the ObjectName for the MBean we will register
	    ObjectName servername = new ObjectName(
		    "com.splunk.shep.mbeans:type=Server");
	    Object host = mbs.getAttribute(servername, "DefHadoopClusterHost");
	    this.ip = host.toString();
	    Object port = mbs.getAttribute(servername, "DefHadoopClusterPort");
	    this.port = port.toString();
	    ObjectName forwardername = new ObjectName(
		    "com.splunk.shep.mbeans:type=Forwarder");
	    Object params[] = new Object[1];
	    params[0] = new String(name);
	    String signature[] = { "java.lang.String" };
	    Object prefix = mbs.invoke(forwardername, "getHDFSSinkPrefix", params,
		    signature);
	    this.path = prefix.toString();
	    Object appending = mbs.invoke(forwardername,
		    "getHDFSSinkUseAppending", params,
		    signature);
	    this.useAppend = Boolean.getBoolean(appending.toString());
	    Object fileRollingSize = mbs.invoke(forwardername,
		    "getHDFSSinkFileRollingSize", params, signature);
	    this.fileRollingSize = Integer.parseInt(fileRollingSize.toString());
	    logger.debug("filePrefix: " + this.path);
	    logger.debug("useAppending: " + this.useAppend);
	    logger.debug("fileRollingSize: " + this.fileRollingSize);
	    init();
	} catch (Exception e) {
	    // should not happen - so runtime exception
	    logger.error("HDFSSink Init fail in setName" , e);
	    throw new RuntimeException("HDFSSink Init fail in setName");
	}
    }<commit_msg>useAppending not set correctly<commit_after>// HdfsIO.java
//
// Copyright (C) 2011 Splunk Inc.
//
// Splunk Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.splunk.shep.s2s.forwarder;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.net.URI;

import javax.management.MBeanServer;
import javax.management.ObjectName;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.LineReader;
import org.apache.log4j.Logger;

import com.splunk.shep.s2s.DataSink;

public class HDFSSink implements DataSink {
    public static final long HadoopFileSize = 63000000;
    public static final int MaxDataLoggingSize = 100000;
    private String ip = new String("localhost");
    private String port = new String("54310");
    private String path = new String("/spl");

    private Configuration conf = null;
    private FileSystem fileSystem = null;
    private Path destination = null;
    private FSDataOutputStream ofstream = null;
    private FSDataInputStream ifstream = null;

    private boolean useAppend = false;
    private boolean reopenedForWriting = false;
    private long totalBytesWritten = 0; // total bytes with current connection.
    private long fileRollingSize = HadoopFileSize;
    private long fileRollingDuration = 30000; // roll to new file every 30 sec.
    private long timeOpened = 0; // milliseconds
    private long maxEventSize = 32000; // not supported for now.
    private String currentFilePath = new String("");
    private String myname;

    private static Logger logger = Logger.getLogger(HDFSSink.class);

    public HDFSSink() {
    }
    
    // only when my name gets set I can init myself
    @Override
    public void setName(String name) {
	this.myname = name;
	try {
	    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

	    // Construct the ObjectName for the MBean we will register
	    ObjectName servername = new ObjectName(
		    "com.splunk.shep.mbeans:type=Server");
	    Object host = mbs.getAttribute(servername, "DefHadoopClusterHost");
	    this.ip = host.toString();
	    Object port = mbs.getAttribute(servername, "DefHadoopClusterPort");
	    this.port = port.toString();
	    ObjectName forwardername = new ObjectName(
		    "com.splunk.shep.mbeans:type=Forwarder");
	    Object params[] = new Object[1];
	    params[0] = new String(name);
	    String signature[] = { "java.lang.String" };
	    Object prefix = mbs.invoke(forwardername, "getHDFSSinkPrefix", params,
		    signature);
	    this.path = prefix.toString();
	    Object appending = mbs.invoke(forwardername,
		    "getHDFSSinkUseAppending", params,
		    signature);
	    if (appending.toString().equals("true")) {
		this.useAppend = true;
	    } else {
		this.useAppend = false;
	    }
	    Object fileRollingSize = mbs.invoke(forwardername,
		    "getHDFSSinkFileRollingSize", params, signature);
	    this.fileRollingSize = Integer.parseInt(fileRollingSize.toString());
	    logger.debug("filePrefix: " + this.path);
	    logger.debug("useAppending: " + this.useAppend);
	    logger.debug("fileRollingSize: " + this.fileRollingSize);
	    init();
	} catch (Exception e) {
	    // should not happen - so runtime exception
	    logger.error("HDFSSink Init fail in setName" , e);
	    throw new RuntimeException("HDFSSink Init fail in setName");
	}
    }
{'fraction_non_alphanumeric': 0.09967221075634333, 'fraction_numerical': 0.011169114969042127, 'mean_word_length': 3.66213921901528, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3221811', 'n_tokens_mistral': 2483, 'n_tokens_neox': 2319, 'n_words': 712}