<commit_before>package com.google.dotorg.crisisresponse.translationcards;

import android.app.AlertDialog;
import android.content.DialogInterface;
import android.content.Intent;
import android.net.Uri;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Scanner;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

public class ImportActivity extends AppCompatActivity {

    private static final String TAG = "ImportActivity";

    private static final String INDEX_FILENAME = "card_deck.csv";
    private static final int BUFFER_SIZE = 2048;

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        importData(getIntent().getData());
    }

    private void importData(Uri source) {
        ZipInputStream zip = getZip(source);
        if (zip == null) {
            alertUserOfFailure();
            return;
        }
        String filename = source.getLastPathSegment();
        File targetDir = getTargetDirectory(filename);
        if (!readFiles(zip, targetDir)) {
            alertUserOfFailure();
            return;
        }
        List<ImportItem> index = getIndex(targetDir);
        if (index == null) {
            alertUserOfFailure();
            return;
        }
        confirmAndLoadData(targetDir, index);
    }

    private ZipInputStream getZip(Uri source) {
        InputStream inputStream = null;
        try {
            inputStream = getContentResolver().openInputStream(source);
        } catch (FileNotFoundException e) {
            return null;
        }
        return new ZipInputStream(inputStream);
    }

    private File getTargetDirectory(String filename) {
        File recordingsDir = new File(getFilesDir(), "recordings");
        File targetDir = new File(recordingsDir, String.format("%s-%d",
                filename, (new Random()).nextInt()));
        targetDir.mkdirs();
        return targetDir;
    }

    private boolean readFiles(ZipInputStream zip, File targetDir) {
        boolean foundIndex = false;
        FileOutputStream outputStream = null;
        try {
            ZipEntry zipEntry = null;
            while ((zipEntry = zip.getNextEntry()) != null) {
                String name = zipEntry.getName();
                if (INDEX_FILENAME.equals(name)) {
                    foundIndex = true;
                }
                outputStream = new FileOutputStream(new File(targetDir, name));
                byte[] buffer = new byte[BUFFER_SIZE];
                int read;
                while ((read = zip.read(buffer)) != -1) {
                    outputStream.write(buffer, 0, read);
                }
                outputStream.flush();
                outputStream.close();
                outputStream = null;
            }
        } catch (IOException e) {
        } finally {
            try {
                zip.close();
            } catch (IOException e) {
                return false;
            }
            if (outputStream != null) {
                try {
                    outputStream.close();
                } catch (IOException e) {
                    return false;
                }
            }
        }
        if (foundIndex) {
            return true;
        } else {
            targetDir.delete();
            Log.d(TAG, "Failed to find index.");
            return false;
        }
    }

    private List<ImportItem> getIndex(File dir) {
        List<ImportItem> results = new ArrayList<>();
        Scanner s;
        try {
            s = new Scanner(new File(dir, INDEX_FILENAME));
        } catch (FileNotFoundException e) {
            return null;
        }
        while (s.hasNextLine()) {
            String line = s.nextLine();<commit_msg>Accept index files with .txt extension.<commit_after>package com.google.dotorg.crisisresponse.translationcards;

import android.app.AlertDialog;
import android.content.DialogInterface;
import android.content.Intent;
import android.net.Uri;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Scanner;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

public class ImportActivity extends AppCompatActivity {

    private static final String TAG = "ImportActivity";

    private static final String INDEX_FILENAME_1 = "card_deck.csv";
    private static final String INDEX_FILENAME_2 = "card_deck.txt";
    private static final int BUFFER_SIZE = 2048;

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        importData(getIntent().getData());
    }

    private void importData(Uri source) {
        ZipInputStream zip = getZip(source);
        if (zip == null) {
            alertUserOfFailure();
            return;
        }
        String filename = source.getLastPathSegment();
        File targetDir = getTargetDirectory(filename);
        String indexFilename = readFiles(zip, targetDir);
        if (indexFilename == null) {
            alertUserOfFailure();
            return;
        }
        List<ImportItem> index = getIndex(targetDir, indexFilename);
        if (index == null) {
            alertUserOfFailure();
            return;
        }
        confirmAndLoadData(targetDir, index);
    }

    private ZipInputStream getZip(Uri source) {
        InputStream inputStream = null;
        try {
            inputStream = getContentResolver().openInputStream(source);
        } catch (FileNotFoundException e) {
            return null;
        }
        return new ZipInputStream(inputStream);
    }

    private File getTargetDirectory(String filename) {
        File recordingsDir = new File(getFilesDir(), "recordings");
        File targetDir = new File(recordingsDir, String.format("%s-%d",
                filename, (new Random()).nextInt()));
        targetDir.mkdirs();
        return targetDir;
    }

    private String readFiles(ZipInputStream zip, File targetDir) {
        String indexFilename = null;
        FileOutputStream outputStream = null;
        try {
            ZipEntry zipEntry = null;
            while ((zipEntry = zip.getNextEntry()) != null) {
                String name = zipEntry.getName();
                if (INDEX_FILENAME_1.equals(name) || INDEX_FILENAME_2.equals(name)) {
                    indexFilename = name;
                }
                outputStream = new FileOutputStream(new File(targetDir, name));
                byte[] buffer = new byte[BUFFER_SIZE];
                int read;
                while ((read = zip.read(buffer)) != -1) {
                    outputStream.write(buffer, 0, read);
                }
                outputStream.flush();
                outputStream.close();
                outputStream = null;
            }
        } catch (IOException e) {
        } finally {
            try {
                zip.close();
            } catch (IOException e) {
                return null;
            }
            if (outputStream != null) {
                try {
                    outputStream.close();
                } catch (IOException e) {
                    return null;
                }
            }
        }
        if (indexFilename == null) {
            targetDir.delete();
            Log.d(TAG, "Failed to find index.");
        }
        return indexFilename;
    }

    private List<ImportItem> getIndex(File dir, String indexFilename) {
        List<ImportItem> results = new ArrayList<>();
        Scanner s;
        try {
            s = new Scanner(new File(dir, indexFilename));
        } catch (FileNotFoundException e) {
            return null;
        }
        while (s.hasNextLine()) {
            String line = s.nextLine();
{'fraction_non_alphanumeric': 0.09731543624161074, 'fraction_numerical': 0.0021572387344199425, 'mean_word_length': 1.9665837184500534, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '144226', 'n_tokens_mistral': 2188, 'n_tokens_neox': 2088, 'n_words': 520}