<commit_before>package com.pseudosudostudios.jdoc;

import android.app.ProgressDialog;
import android.content.Context;
import android.content.Intent;
import android.os.AsyncTask;
import android.util.Log;

import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import data.FileInfoFactory;

/**
 * Created by Ben on 11/17/2014.
 * Copyright (c) 2014 Pseudo Sudo Studios
 */
public class DataLoader extends AsyncTask<Void, Integer, Void> {
    private static final String TAG = DataLoader.class.getSimpleName();
    private Context context;
    private ProgressDialog dialog;
    boolean publishProgress;
    private long start;

    final int THREAD_COUNT = 7;

    public DataLoader(Context context, ProgressDialog dialog, int style) {
        this.context = context;
        this.dialog = dialog;
        publishProgress = style == ProgressDialog.STYLE_HORIZONTAL;
    }

    @Override
    protected Void doInBackground(Void... voids) {

        start = System.currentTimeMillis();
        final String parent = "data-files";
        loadData(parent, "Activity.json");

        if (1 == 2)
            try {
                final String[] files = context.getAssets().list(parent);
                final int fileCount = 100; //files.length;

                ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT);
                for (int i = 0; i < fileCount; i++) {
                    final int a = i;
                    executorService.submit(new Runnable() {
                        @Override
                        public void run() {
                            if (Thread.interrupted())
                                return;
                            String file = files[a];
                            loadData(parent, file);
                            if (publishProgress)
                                publishProgress(a, fileCount);
                        }
                    });
                }
                executorService.shutdown(); //No more can be scheduled
                if (executorService.awaitTermination(90, TimeUnit.SECONDS)) {
                    Log.i(TAG, "Executor finished");
                } else {
                    Log.w(TAG, "Executor forced stop");
                    executorService.shutdownNow();
                }
                //TODO uncomment loadData(parent, "Activity.json");

            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }
        FileInfoFactory.sortPackages(); //sort when done
        Log.i("DataLoader", "Parse time: " +
                ((System.currentTimeMillis() - start) / 1000D));
        return null;
    }

    private void loadData(String parent, String currentFile) {
        try {
            InputStream stream =
                    context.getAssets().open(parent +
                            File.separator + currentFile);

            BufferedReader reader = new BufferedReader(
                    new InputStreamReader(stream));
            StringBuilder buffer = new StringBuilder();
            while (reader.ready())
                buffer.append(reader.readLine());
            String jString = buffer.toString();
            JSONObject jObj = new JSONObject(jString);
            FileInfoFactory.parseFile(jObj);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (JSONException e) {
            e.printStackTrace();
        }
    }

    @Override<commit_msg>#1 Using small set and Activity.json for testing<commit_after>package com.pseudosudostudios.jdoc;

import android.app.ProgressDialog;
import android.content.Context;
import android.content.Intent;
import android.os.AsyncTask;
import android.util.Log;

import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import data.FileInfoFactory;

/**
 * Created by Ben on 11/17/2014.
 * Copyright (c) 2014 Pseudo Sudo Studios
 */
public class DataLoader extends AsyncTask<Void, Integer, Void> {
    private static final String TAG = DataLoader.class.getSimpleName();
    private Context context;
    private ProgressDialog dialog;
    boolean publishProgress;
    private long start;

    int THREAD_COUNT = 30;

    public DataLoader(Context context, ProgressDialog dialog, int style) {
        this.context = context;
        this.dialog = dialog;
        publishProgress = style == ProgressDialog.STYLE_HORIZONTAL;
    }

    @Override
    protected Void doInBackground(Void... voids) {

        start = System.currentTimeMillis();
        final String parent = "data-files";

        try {
            final String[] files = context.getAssets().list(parent);
            final int fileCount = 5; //files.length / 10;
            for (int i = 0; i < fileCount; i++) {
                loadData(parent, files[i]);
                publishProgress(i, fileCount + 1);
            }
            loadData(parent, "Activity.json");
/*
            ExecutorService executorService = Executors.newCachedThreadPool();

            executorService.execute(new Runnable() {
                @Override
                public void run() {
                    loadData(parent, "Activity.json");
                }
            });

            for (int i = 0; i < fileCount; i++) {
                final int a = i;
                executorService.execute(new Runnable() {
                    @Override
                    public void run() {
                        if (Thread.interrupted())
                            return;
                        String file = files[a];
                        if (file.equals("Activity.json"))
                            return;
                        if (Thread.interrupted())
                            return;
                        loadData(parent, file);
                        if (publishProgress)
                            publishProgress(a, fileCount);
                    }
                });
            }
            executorService.shutdown(); //No more can be scheduled
            if (executorService.awaitTermination(1, TimeUnit.MINUTES)) {
                Log.i(TAG, "Executor finished");
            } else {
                Log.w(TAG, "Executor forced stop");
                executorService.shutdownNow();
            }
            //TODO uncomment loadData(parent, "Activity.json");

        } catch (InterruptedException e) {
            e.printStackTrace();*/
        } catch (IOException e) {
            e.printStackTrace();
        }
        FileInfoFactory.sortPackages(); //sort when done
        Log.i("DataLoader", "Parse time: " +
                ((System.currentTimeMillis() - start) / 1000D));
        return null;
    }

    private double loadData(String parent, String currentFile) {
        try {
            long start = System.currentTimeMillis();
            InputStream stream =
                    context.getAssets().open(parent +
                            File.separator + currentFile);
            BufferedReader reader = new BufferedReader(
                    new InputStreamReader(stream));
            StringBuilder buffer = new StringBuilder();
            while (reader.ready() && !Thread.interrupted())
                buffer.append(reader.readLine());
            reader.close();
            String jString = buffer.toString();
            JSONObject jObj = new JSONObject(jString);
            FileInfoFactory.parseFile(jObj);
            double d = (System.currentTimeMillis() - start) / 1000D;
            Log.i(TAG, String.format("%s parse time: %.03f", currentFile, d));
            return d;
        } catch (IOException e) {
            e.printStackTrace();
        } catch (JSONException e) {
            e.printStackTrace();
        }
        return -1;
    }

    @Override
{'fraction_non_alphanumeric': 0.09772145729255513, 'fraction_numerical': 0.007067137809187279, 'mean_word_length': 1.7151835924578234, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1216130', 'n_tokens_mistral': 2111, 'n_tokens_neox': 1985, 'n_words': 502}