<commit_before>/**
 * Copyright 2003-2013 MarkLogic Corporation
 * 
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 * 
 *       http://www.apache.org/licenses/LICENSE-2.0
 * 
 *    Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 * 
 */
package com.marklogic.mongo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import com.marklogic.mongo.BSON.AttributeNames;
import com.marklogic.mongo.BSON.ElementNamesBS;
import com.marklogic.xcc.Content;
import com.marklogic.xcc.ContentCreateOptions;
import com.marklogic.xcc.ContentFactory;
import com.marklogic.xcc.ContentSource;
import com.marklogic.xcc.ContentSourceFactory;
import com.marklogic.xcc.SecurityOptions;
import com.marklogic.xcc.Session;
import com.marklogic.xcc.exceptions.RequestException;
import com.marklogic.xcc.exceptions.XccConfigException;

public class FileImporter extends Importer {
    
    private String outdir;
    
	void run(String[] args) throws XccConfigException, Exception {

		
		

		String input = getArg(args, "input", null);
		outdir = getArg( args , "directory" , ".");
		
		InputStream is = new BufferedInputStream(input == null ? System.in : new FileInputStream(input));

		BSONInputStream bos = new BSONInputStream(is);

        long tm_start = System.currentTimeMillis();
		
		int files = 0;
		while (is.available() > 0) {
			
			XMLWriter bxw = getWriter(args);
			bxw.writeDocumentRoot(bos);
			
			put(  bxw.close() );

			files++;
		}
		
		long tm_stop = System.currentTimeMillis();
        long ms = tm_stop - tm_start ; 
        
        Importer.completed(files);
        
		
		
	}
    private  void put( byte[] bytes) throws IOException 
	{
		File f = new File( getUri() );
		FileOutputStream fos = new FileOutputStream( f );
		fos.write(bytes);
		fos.close();
		
		
		
	}
	
	private String getUri() {
		return outdir + "/" + getRandom() + ".xml";
	}
	
	}
<commit_msg>append .json extension for -writer json even when writing to disk<commit_after>/**
 * Copyright 2003-2013 MarkLogic Corporation
 * 
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 * 
 *       http://www.apache.org/licenses/LICENSE-2.0
 * 
 *    Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 * 
 */
package com.marklogic.mongo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import com.marklogic.mongo.BSON.AttributeNames;
import com.marklogic.mongo.BSON.ElementNamesBS;
import com.marklogic.xcc.Content;
import com.marklogic.xcc.ContentCreateOptions;
import com.marklogic.xcc.ContentFactory;
import com.marklogic.xcc.ContentSource;
import com.marklogic.xcc.ContentSourceFactory;
import com.marklogic.xcc.SecurityOptions;
import com.marklogic.xcc.Session;
import com.marklogic.xcc.exceptions.RequestException;
import com.marklogic.xcc.exceptions.XccConfigException;

public class FileImporter extends Importer {
    
    private String outdir;
    private XMLWriter mWriter;
    
	void run(String[] args) throws XccConfigException, Exception {

		
		

		String input = getArg(args, "input", null);
		outdir = getArg( args , "directory" , ".");
		
		InputStream is = new BufferedInputStream(input == null ? System.in : new FileInputStream(input));

		BSONInputStream bos = new BSONInputStream(is);

		
		int files = 0;
		while (is.available() > 0) {
			
			mWriter = getWriter(args);
			mWriter.writeDocumentRoot(bos);
			
			put(  mWriter.close() );

			files++;
		}
		
        Importer.completed(files);
        
		
		
	}
    private  void put( byte[] bytes) throws IOException 
	{
		File f = new File( getUri() );
		FileOutputStream fos = new FileOutputStream( f );
		fos.write(bytes);
		fos.close();
		
		
		
	}
	
	private String getUri() {
		return outdir + "/" + getRandom() + mWriter.getSuffix();
	}
	
	}

{'fraction_non_alphanumeric': 0.09811320754716982, 'fraction_numerical': 0.00686106346483705, 'mean_word_length': 3.6022099447513813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6427195', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1734, 'n_words': 462}