<commit_before>/*
 * Copyright (C) 2012,2013 <NAME>.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package com.renard.ocr;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import android.app.AlertDialog;
import android.app.AlertDialog.Builder;
import android.app.Dialog;
import android.app.ProgressDialog;
import android.content.ActivityNotFoundException;
import android.content.ContentProviderClient;
import android.content.ContentValues;
import android.content.DialogInterface;
import android.content.DialogInterface.OnClickListener;
import android.content.Intent;
import android.database.Cursor;
import android.graphics.Bitmap;
import android.net.Uri;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.RemoteException;
import android.provider.MediaStore;
import android.text.Html;
import android.text.Spanned;
import android.util.Pair;
import android.view.View;
import android.widget.Button;
import android.widget.EditText;
import android.widget.TextView;
import android.widget.Toast;

import com.actionbarsherlock.view.Menu;
import com.actionbarsherlock.view.MenuItem;
import com.googlecode.leptonica.android.Pix;
import com.googlecode.leptonica.android.ReadFile;
import com.renard.documentview.DocumentActivity;
import com.renard.ocr.DocumentContentProvider.Columns;
import com.renard.ocr.cropimage.CropImage;
import com.renard.ocr.cropimage.MonitoredActivity;
import com.renard.pdf.Hocr2Pdf;
import com.renard.pdf.Hocr2Pdf.PDFProgressListener;
import com.renard.util.Util;

/**
 * activities which extend this activity can create a new document. this class
 * also containes the code for functionality which is shared by
 * {@link DocumentGridActivity} and {@link DocumentActivity}
 * 
 * @author renard
 * 
 */
public abstract class BaseDocumentActivitiy extends MonitoredActivity {

	public final static String EXTRA_NATIVE_PIX = "pix_pointer";
	public final static String EXTRA_IMAGE_URI = "image_uri";
	public final static String EXTRA_ROTATION = "rotation";

	private static final int PDF_PROGRESS_DIALOG_ID = 0;
	private static final int DELETE_PROGRESS_DIALOG_ID = 1;
	protected static final int HINT_DIALOG_ID = 2;
	private static final int EDIT_TITLE_DIALOG_ID = 3;

	private static final String DIALOG_ARG_MAX = "max";
	private static final String DIALOG_ARG_MESSAGE = "message";
	private static final String DIALOG_ARG_PROGRESS = "progress";
	private static final String DIALOG_ARG_SECONDARY_PROGRESS = "secondary_progress";
	private static final String DIALOG_ARG_TITLE = "title";
	private static final String DIALOG_ARG_DOCUMENT_URI = "document_uri";

	private final static int REQUEST_CODE_MAKE_PHOTO = 0;
	private final static int REQUEST_CODE_PICK_PHOTO = 1;
	private final static int REQUEST_CODE_CROP_PHOTO = 2;
	protected final static int REQUEST_CODE_OCR = 3;

	private static final String DATE_CAMERA_INTENT_STARTED_STATE = "your.package.android.photo.TakePhotoActivity.dateCameraIntentStarted";
	private static Date dateCameraIntentStarted = null;
	private static final String CAMERA_PIC_URI_STATE = "your.package.android.photo.TakePhotoActivity.CAMERA_PIC_URI_STATE";
	private static Uri cameraPicUri = null;
	private static final String ROTATE_X_DEGREES_STATE = "your.package.android.photo.TakePhotoActivity.ROTATE_X_DEGREES_STATE";
	private static int rotateXDegrees = 0;

	private static class CameraResult {
		public CameraResult(int requestCode, int resultCode, Intent data) {
			mRequestCode = requestCode;
			mResultCode = resultCode;
			mData = data;
		}

		private int mRequestCode;
		private int mResultCode;
		private Intent mData;
	}

	protected abstract int getParentId();

	ProgressDialog pdfProgressDialog;
	ProgressDialog deleteProgressdialog;
	private AsyncTask<Void, Void, Pair<Pix, PixLoadStatus>> mBitmapLoadTask;
	private CameraResult mCameraResult;

	protected void startGallery() {
		cameraPicUri = null;
		Intent i = new Intent(Intent.ACTION_GET_CONTENT, null);
		i.setType("image/*");
		// File photo = getTmpPhotoFile();
		// i.putExtra(MediaStore.EXTRA_OUTPUT, Uri.fromFile(photo));
		startActivityForResult(i, REQUEST_CODE_PICK_PHOTO);
	}

	protected void startCamera() {
		try {
			cameraPicUri = null;
			dateCameraIntentStarted = new Date();
			Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
			// NOTE: Do NOT SET: intent.putExtra(MediaStore.EXTRA_OUTPUT,
			// cameraPicUri) on Samsung Galaxy S2/S3/.. for the following
			// reasons:
			// 1.) it will break the correct picture orientation
			// 2.) the photo will be stored in two locations (the given path and
			// additionally in the MediaStore)
			String manufacturer = android.os.Build.MANUFACTURER.toLowerCase();
			if (!(manufacturer.contains("samsung")) && !(manufacturer.contains("sony"))) {
				String filename = System.currentTimeMillis() + ".jpg";
				ContentValues values = new ContentValues();
				values.put(MediaStore.Images.Media.TITLE, filename);
				cameraPicUri = getContentResolver().insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, values);
				intent.putExtra(MediaStore.EXTRA_OUTPUT, cameraPicUri);
			}
			startActivityForResult(intent, REQUEST_CODE_MAKE_PHOTO);
		} catch (ActivityNotFoundException e) {
			showFileError(PixLoadStatus.CAMERA_APP_NOT_FOUND);
		}
	}

	@Override
	protected void onSaveInstanceState(Bundle savedInstanceState) {
		super.onSaveInstanceState(savedInstanceState);
		if (dateCameraIntentStarted != null) {
			savedInstanceState.putLong(DATE_CAMERA_INTENT_STARTED_STATE, dateCameraIntentStarted.getTime());
		}
		if (cameraPicUri != null) {
			savedInstanceState.putString(CAMERA_PIC_URI_STATE, cameraPicUri.toString());
		}
		savedInstanceState.putInt(ROTATE_X_DEGREES_STATE, rotateXDegrees);
	}

	@Override
	protected void onRestoreInstanceState(Bundle savedInstanceState) {
		super.onRestoreInstanceState(savedInstanceState);
		if (savedInstanceState.containsKey(DATE_CAMERA_INTENT_STARTED_STATE)) {
			dateCameraIntentStarted = new Date(savedInstanceState.getLong(DATE_CAMERA_INTENT_STARTED_STATE));
		}
		if (savedInstanceState.containsKey(CAMERA_PIC_URI_STATE)) {
			cameraPicUri = Uri.parse(savedInstanceState.getString(CAMERA_PIC_URI_STATE));
		}
		rotateXDegrees = savedInstanceState.getInt(ROTATE_X_DEGREES_STATE);
	}

	@Override
	public boolean onOptionsItemSelected(MenuItem item) {
		int itemId = item.getItemId();
		if (itemId == R.id.item_camera) {
			startCamera();
			return true;
		} else if (itemId == R.id.item_gallery) {
			startGallery();
			return true;
		}

		return super.onOptionsItemSelected(item);
	}

	@Override
	public boolean onCreateOptionsMenu(Menu menu) {
		getSupportMenuInflater().inflate(R.menu.base_document_activity_options, menu);
		return true;
	}

	private void onTakePhotoActivityResult(int requestCode, int resultCode, Intent intent) {
		if (resultCode == RESULT_OK) {
			if (requestCode == REQUEST_CODE_MAKE_PHOTO) {
				Cursor myCursor = null;
				Date dateOfPicture = null;
				try {
					// Create a Cursor to obtain the file Path for the large
					// image
					String[] largeFileProjection = { MediaStore.Images.ImageColumns._ID, MediaStore.Images.ImageColumns.DATA, MediaStore.Images.ImageColumns.ORIENTATION,
							MediaStore.Images.ImageColumns.DATE_TAKEN };
					String largeFileSort = MediaStore.Images.ImageColumns._ID + " DESC";
					myCursor = getContentResolver().query(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, largeFileProjection, null, null, largeFileSort);
					myCursor.moveToFirst();
					// This will actually give you the file path location of the
					// image.
					String largeImagePath = myCursor.getString(myCursor.getColumnIndexOrThrow(MediaStore.Images.ImageColumns.DATA));
					Uri tempCameraPicUri = Uri.fromFile(new File(largeImagePath));
					if (tempCameraPicUri != null) {
						dateOfPicture = new Date(myCursor.getLong(myCursor.getColumnIndexOrThrow(MediaStore.Images.ImageColumns.DATE_TAKEN)));
						if (dateOfPicture != null && dateOfPicture.after(dateCameraIntentStarted)) {
							cameraPicUri = tempCameraPicUri;
							rotateXDegrees = myCursor.getInt(myCursor.getColumnIndexOrThrow(MediaStore.Images.ImageColumns.ORIENTATION));
						}
					}
				} catch (Exception e) {
					// Log.w("TAG",
					// "Exception - optaining the picture's uri failed: " +
					// e.toString());
				} finally {
					if (myCursor != null) {
						myCursor.close();
					}
				}
			}

			if (cameraPicUri == null) {
				try {
					cameraPicUri = intent.getData();
				} catch (Exception e) {
					showFileError(PixLoadStatus.CAMERA_APP_ERROR);
				}
			}

			if (cameraPicUri != null) {
				loadBitmapFromContentUri(cameraPicUri);
				return;
			} else {
				showFileError(PixLoadStatus.CAMERA_NO_IMAGE_RETURNED);
			}
		}
	}

	private enum PixLoadStatus {
		MEDIA_STORE_RETURNED_NULL, IMAGE_DOES_NOT_EXIST, SUCCESS, IO_ERROR, CAMERA_APP_NOT_FOUND, CAMERA_APP_ERROR, CAMERA_NO_IMAGE_RETURNED

	}

	private void loadBitmapFromContentUri(final Uri cameraPicUri) {
		if (mBitmapLoadTask != null) {
			mBitmapLoadTask.cancel(true);
		}
		mBitmapLoadTask = new AsyncTask<Void, Void, Pair<Pix, PixLoadStatus>>() {
			ProgressDialogFragment progressDialog;

			protected void onPreExecute() {
				progressDialog = ProgressDialogFragment.newInstance(R.string.please_wait, R.string.loading_image);
				progressDialog.show(getSupportFragmentManager(), "load_image_progress");
			};

			protected void onPostExecute(Pair<Pix, PixLoadStatus> p) {
				if (progressDialog !=null && !progressDialog.isDetached()) {
					try {
						progressDialog.dismiss();
					} catch (NullPointerException e) {
						// workaround strange playstore crash

					} catch(IllegalStateException e){
						// workaround strange playstore crash
						
					}
				}
				if (p.second == PixLoadStatus.SUCCESS) {
					Intent actionIntent = new Intent(BaseDocumentActivitiy.this, CropImage.class);
					actionIntent.putExtra(EXTRA_NATIVE_PIX, p.first.getNativePix());
					actionIntent.putExtra(EXTRA_ROTATION, rotateXDegrees);
					startActivityForResult(actionIntent, REQUEST_CODE_CROP_PHOTO);
				} else {
					showFileError(p.second);
				}
			};

			@Override<commit_msg>dismiss dialog fragment with fragment transaction<commit_after>/*
 * Copyright (C) 2012,2013 <NAME>.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package com.renard.ocr;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import android.app.AlertDialog;
import android.app.AlertDialog.Builder;
import android.app.Dialog;
import android.app.ProgressDialog;
import android.content.ActivityNotFoundException;
import android.content.ContentProviderClient;
import android.content.ContentValues;
import android.content.DialogInterface;
import android.content.DialogInterface.OnClickListener;
import android.content.Intent;
import android.database.Cursor;
import android.graphics.Bitmap;
import android.net.Uri;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.RemoteException;
import android.provider.MediaStore;
import android.text.Html;
import android.text.Spanned;
import android.util.Pair;
import android.view.View;
import android.widget.Button;
import android.widget.EditText;
import android.widget.TextView;
import android.widget.Toast;

import com.actionbarsherlock.view.Menu;
import com.actionbarsherlock.view.MenuItem;
import com.googlecode.leptonica.android.Pix;
import com.googlecode.leptonica.android.ReadFile;
import com.renard.documentview.DocumentActivity;
import com.renard.ocr.DocumentContentProvider.Columns;
import com.renard.ocr.cropimage.CropImage;
import com.renard.ocr.cropimage.MonitoredActivity;
import com.renard.pdf.Hocr2Pdf;
import com.renard.pdf.Hocr2Pdf.PDFProgressListener;
import com.renard.util.Util;

/**
 * activities which extend this activity can create a new document. this class
 * also containes the code for functionality which is shared by
 * {@link DocumentGridActivity} and {@link DocumentActivity}
 * 
 * @author renard
 * 
 */
public abstract class BaseDocumentActivitiy extends MonitoredActivity {

	public final static String EXTRA_NATIVE_PIX = "pix_pointer";
	public final static String EXTRA_IMAGE_URI = "image_uri";
	public final static String EXTRA_ROTATION = "rotation";

	private static final int PDF_PROGRESS_DIALOG_ID = 0;
	private static final int DELETE_PROGRESS_DIALOG_ID = 1;
	protected static final int HINT_DIALOG_ID = 2;
	private static final int EDIT_TITLE_DIALOG_ID = 3;

	private static final String DIALOG_ARG_MAX = "max";
	private static final String DIALOG_ARG_MESSAGE = "message";
	private static final String DIALOG_ARG_PROGRESS = "progress";
	private static final String DIALOG_ARG_SECONDARY_PROGRESS = "secondary_progress";
	private static final String DIALOG_ARG_TITLE = "title";
	private static final String DIALOG_ARG_DOCUMENT_URI = "document_uri";

	private final static int REQUEST_CODE_MAKE_PHOTO = 0;
	private final static int REQUEST_CODE_PICK_PHOTO = 1;
	private final static int REQUEST_CODE_CROP_PHOTO = 2;
	protected final static int REQUEST_CODE_OCR = 3;

	private static final String DATE_CAMERA_INTENT_STARTED_STATE = "your.package.android.photo.TakePhotoActivity.dateCameraIntentStarted";
	private static Date dateCameraIntentStarted = null;
	private static final String CAMERA_PIC_URI_STATE = "your.package.android.photo.TakePhotoActivity.CAMERA_PIC_URI_STATE";
	private static Uri cameraPicUri = null;
	private static final String ROTATE_X_DEGREES_STATE = "your.package.android.photo.TakePhotoActivity.ROTATE_X_DEGREES_STATE";
	private static int rotateXDegrees = 0;

	private static class CameraResult {
		public CameraResult(int requestCode, int resultCode, Intent data) {
			mRequestCode = requestCode;
			mResultCode = resultCode;
			mData = data;
		}

		private int mRequestCode;
		private int mResultCode;
		private Intent mData;
	}

	protected abstract int getParentId();

	ProgressDialog pdfProgressDialog;
	ProgressDialog deleteProgressdialog;
	private AsyncTask<Void, Void, Pair<Pix, PixLoadStatus>> mBitmapLoadTask;
	private CameraResult mCameraResult;

	protected void startGallery() {
		cameraPicUri = null;
		Intent i = new Intent(Intent.ACTION_GET_CONTENT, null);
		i.setType("image/*");
		// File photo = getTmpPhotoFile();
		// i.putExtra(MediaStore.EXTRA_OUTPUT, Uri.fromFile(photo));
		startActivityForResult(i, REQUEST_CODE_PICK_PHOTO);
	}

	protected void startCamera() {
		try {
			cameraPicUri = null;
			dateCameraIntentStarted = new Date();
			Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
			// NOTE: Do NOT SET: intent.putExtra(MediaStore.EXTRA_OUTPUT,
			// cameraPicUri) on Samsung Galaxy S2/S3/.. for the following
			// reasons:
			// 1.) it will break the correct picture orientation
			// 2.) the photo will be stored in two locations (the given path and
			// additionally in the MediaStore)
			String manufacturer = android.os.Build.MANUFACTURER.toLowerCase();
			if (!(manufacturer.contains("samsung")) && !(manufacturer.contains("sony"))) {
				String filename = System.currentTimeMillis() + ".jpg";
				ContentValues values = new ContentValues();
				values.put(MediaStore.Images.Media.TITLE, filename);
				cameraPicUri = getContentResolver().insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, values);
				intent.putExtra(MediaStore.EXTRA_OUTPUT, cameraPicUri);
			}
			startActivityForResult(intent, REQUEST_CODE_MAKE_PHOTO);
		} catch (ActivityNotFoundException e) {
			showFileError(PixLoadStatus.CAMERA_APP_NOT_FOUND);
		}
	}

	@Override
	protected void onSaveInstanceState(Bundle savedInstanceState) {
		super.onSaveInstanceState(savedInstanceState);
		if (dateCameraIntentStarted != null) {
			savedInstanceState.putLong(DATE_CAMERA_INTENT_STARTED_STATE, dateCameraIntentStarted.getTime());
		}
		if (cameraPicUri != null) {
			savedInstanceState.putString(CAMERA_PIC_URI_STATE, cameraPicUri.toString());
		}
		savedInstanceState.putInt(ROTATE_X_DEGREES_STATE, rotateXDegrees);
	}

	@Override
	protected void onRestoreInstanceState(Bundle savedInstanceState) {
		super.onRestoreInstanceState(savedInstanceState);
		if (savedInstanceState.containsKey(DATE_CAMERA_INTENT_STARTED_STATE)) {
			dateCameraIntentStarted = new Date(savedInstanceState.getLong(DATE_CAMERA_INTENT_STARTED_STATE));
		}
		if (savedInstanceState.containsKey(CAMERA_PIC_URI_STATE)) {
			cameraPicUri = Uri.parse(savedInstanceState.getString(CAMERA_PIC_URI_STATE));
		}
		rotateXDegrees = savedInstanceState.getInt(ROTATE_X_DEGREES_STATE);
	}

	@Override
	public boolean onOptionsItemSelected(MenuItem item) {
		int itemId = item.getItemId();
		if (itemId == R.id.item_camera) {
			startCamera();
			return true;
		} else if (itemId == R.id.item_gallery) {
			startGallery();
			return true;
		}

		return super.onOptionsItemSelected(item);
	}

	@Override
	public boolean onCreateOptionsMenu(Menu menu) {
		getSupportMenuInflater().inflate(R.menu.base_document_activity_options, menu);
		return true;
	}

	private void onTakePhotoActivityResult(int requestCode, int resultCode, Intent intent) {
		if (resultCode == RESULT_OK) {
			if (requestCode == REQUEST_CODE_MAKE_PHOTO) {
				Cursor myCursor = null;
				Date dateOfPicture = null;
				try {
					// Create a Cursor to obtain the file Path for the large
					// image
					String[] largeFileProjection = { MediaStore.Images.ImageColumns._ID, MediaStore.Images.ImageColumns.DATA, MediaStore.Images.ImageColumns.ORIENTATION,
							MediaStore.Images.ImageColumns.DATE_TAKEN };
					String largeFileSort = MediaStore.Images.ImageColumns._ID + " DESC";
					myCursor = getContentResolver().query(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, largeFileProjection, null, null, largeFileSort);
					myCursor.moveToFirst();
					// This will actually give you the file path location of the
					// image.
					String largeImagePath = myCursor.getString(myCursor.getColumnIndexOrThrow(MediaStore.Images.ImageColumns.DATA));
					Uri tempCameraPicUri = Uri.fromFile(new File(largeImagePath));
					if (tempCameraPicUri != null) {
						dateOfPicture = new Date(myCursor.getLong(myCursor.getColumnIndexOrThrow(MediaStore.Images.ImageColumns.DATE_TAKEN)));
						if (dateOfPicture != null && dateOfPicture.after(dateCameraIntentStarted)) {
							cameraPicUri = tempCameraPicUri;
							rotateXDegrees = myCursor.getInt(myCursor.getColumnIndexOrThrow(MediaStore.Images.ImageColumns.ORIENTATION));
						}
					}
				} catch (Exception e) {
					// Log.w("TAG",
					// "Exception - optaining the picture's uri failed: " +
					// e.toString());
				} finally {
					if (myCursor != null) {
						myCursor.close();
					}
				}
			}

			if (cameraPicUri == null) {
				try {
					cameraPicUri = intent.getData();
				} catch (Exception e) {
					showFileError(PixLoadStatus.CAMERA_APP_ERROR);
				}
			}

			if (cameraPicUri != null) {
				loadBitmapFromContentUri(cameraPicUri);
				return;
			} else {
				showFileError(PixLoadStatus.CAMERA_NO_IMAGE_RETURNED);
			}
		}
	}

	private enum PixLoadStatus {
		MEDIA_STORE_RETURNED_NULL, IMAGE_DOES_NOT_EXIST, SUCCESS, IO_ERROR, CAMERA_APP_NOT_FOUND, CAMERA_APP_ERROR, CAMERA_NO_IMAGE_RETURNED

	}

	private void loadBitmapFromContentUri(final Uri cameraPicUri) {
		if (mBitmapLoadTask != null) {
			mBitmapLoadTask.cancel(true);
		}
		mBitmapLoadTask = new AsyncTask<Void, Void, Pair<Pix, PixLoadStatus>>() {
			ProgressDialogFragment progressDialog;

			protected void onPreExecute() {
				progressDialog = ProgressDialogFragment.newInstance(R.string.please_wait, R.string.loading_image);
				progressDialog.show(getSupportFragmentManager(), "load_image_progress");
			};

			protected void onPostExecute(Pair<Pix, PixLoadStatus> p) {
				if (progressDialog !=null && !progressDialog.isDetached()) {
					try {
						getSupportFragmentManager().beginTransaction().remove(progressDialog).commitAllowingStateLoss();
					} catch (NullPointerException e) {
						// workaround strange playstore crash

					} catch(IllegalStateException e){
						// workaround strange playstore crash
						
					}
				}
				if (p.second == PixLoadStatus.SUCCESS) {
					Intent actionIntent = new Intent(BaseDocumentActivitiy.this, CropImage.class);
					actionIntent.putExtra(EXTRA_NATIVE_PIX, p.first.getNativePix());
					actionIntent.putExtra(EXTRA_ROTATION, rotateXDegrees);
					startActivityForResult(actionIntent, REQUEST_CODE_CROP_PHOTO);
				} else {
					showFileError(p.second);
				}
			};

			@Override
{'fraction_non_alphanumeric': 0.0923006205470007, 'fraction_numerical': 0.0024821880027579867, 'mean_word_length': 4.867313915857605, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6885512', 'n_tokens_mistral': 7549, 'n_tokens_neox': 6523, 'n_words': 1492}