<commit_before>/*
 * Copyright 2010-2011 Ning, Inc.
 *
 * Ning licenses this file to you under the Apache License, version 2.0
 * (the "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.ning.metrics.collector.events.hadoop.writer;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;

import java.io.IOException;

class HadoopOutputChunk
{
    private final Path sourcePath;
    private final Path desintationPath;
    private final SequenceFile.Writer writer;
    private boolean isClosed = false;

    HadoopOutputChunk(Path sourcePath, Path desintationPath, SequenceFile.Writer writer)
    {
        this.sourcePath = sourcePath;
        this.desintationPath = desintationPath;
        this.writer = writer;
    }

    public void close() throws IOException
    {
        if (!isClosed) {
            isClosed = true;
            writer.close();
        }
    }

    public void commit(FileSystem fileSystem) throws IOException
    {
        Path destinationDir = desintationPath.getParent();

        // parent directory has to exist for a hdfs rename to succeed
        if (!fileSystem.exists(destinationDir) && !fileSystem.mkdirs(desintationPath.getParent())) {
            throw new IOException(String.format("Unable to make destination directory %s (does the parent directory exist?)", destinationDir));
        }
        if (!fileSystem.rename(sourcePath, desintationPath)) {
            throw new IOException(String.format("Unable to rename %s to %s", sourcePath, desintationPath));
        }
    }

    public void rollback(FileSystem fileSystem) throws IOException
    {
        deleteIfExists(sourcePath, fileSystem);
        deleteIfExists(desintationPath, fileSystem);
    }

    private void deleteIfExists(Path path, FileSystem fileSystem) throws IOException
    {
        if (fileSystem.exists(path) && !fileSystem.delete(path, false)) {
            throw new IOException(String.format("unable to delete %s", path));
        }
    }

    public SequenceFile.Writer getWriter()
    {
        return writer;
    }

    @Override
    public String toString()
    {
        return String.format("%s : %s -> %s", HadoopOutputChunk.class, sourcePath, desintationPath);
    }
}
<commit_msg>hadoop: fix typo in HadoopOutputChunk<commit_after>/*
 * Copyright 2010-2011 Ning, Inc.
 *
 * Ning licenses this file to you under the Apache License, version 2.0
 * (the "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.ning.metrics.collector.events.hadoop.writer;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;

import java.io.IOException;

class HadoopOutputChunk
{
    private final Path sourcePath;
    private final Path destinationPath;
    private final SequenceFile.Writer writer;
    private boolean isClosed = false;

    HadoopOutputChunk(Path sourcePath, Path destinationPath, SequenceFile.Writer writer)
    {
        this.sourcePath = sourcePath;
        this.destinationPath = destinationPath;
        this.writer = writer;
    }

    public void close() throws IOException
    {
        if (!isClosed) {
            isClosed = true;
            writer.close();
        }
    }

    public void commit(FileSystem fileSystem) throws IOException
    {
        Path destinationDir = destinationPath.getParent();

        // parent directory has to exist for a hdfs rename to succeed
        if (!fileSystem.exists(destinationDir) && !fileSystem.mkdirs(destinationPath.getParent())) {
            throw new IOException(String.format("Unable to make destination directory %s (does the parent directory exist?)", destinationDir));
        }
        if (!fileSystem.rename(sourcePath, destinationPath)) {
            throw new IOException(String.format("Unable to rename %s to %s", sourcePath, destinationPath));
        }
    }

    public void rollback(FileSystem fileSystem) throws IOException
    {
        deleteIfExists(sourcePath, fileSystem);
        deleteIfExists(destinationPath, fileSystem);
    }

    private void deleteIfExists(Path path, FileSystem fileSystem) throws IOException
    {
        if (fileSystem.exists(path) && !fileSystem.delete(path, false)) {
            throw new IOException(String.format("unable to delete %s", path));
        }
    }

    public SequenceFile.Writer getWriter()
    {
        return writer;
    }

    @Override
    public String toString()
    {
        return String.format("%s : %s -> %s", HadoopOutputChunk.class, sourcePath, destinationPath);
    }
}

{'fraction_non_alphanumeric': 0.08843909555069293, 'fraction_numerical': 0.00437636761487965, 'mean_word_length': 3.0720118782479586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4770009', 'n_tokens_mistral': 1477, 'n_tokens_neox': 1423, 'n_words': 470}