<commit_before>package mil.nga.giat.mage.map.preference;

import android.Manifest;
import android.app.Activity;
import android.content.Context;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.os.Environment;
import android.preference.PreferenceManager;
import android.support.annotation.Nullable;
import android.support.annotation.UiThread;
import android.support.v4.app.ActivityCompat;
import android.support.v4.app.ListFragment;
import android.support.v4.content.ContextCompat;
import android.support.v7.app.AlertDialog;
import android.support.v7.app.AppCompatActivity;
import android.text.format.Formatter;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.Menu;
import android.view.MenuInflater;
import android.view.MenuItem;
import android.view.View;
import android.view.ViewGroup;
import android.widget.AdapterView;
import android.widget.BaseExpandableListAdapter;
import android.widget.CheckBox;
import android.widget.ExpandableListView;
import android.widget.ImageView;
import android.widget.ListView;
import android.widget.ProgressBar;
import android.widget.TextView;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Semaphore;

import mil.nga.geopackage.GeoPackageManager;
import mil.nga.geopackage.factory.GeoPackageFactory;
import mil.nga.giat.mage.R;
import mil.nga.giat.mage.cache.CacheUtils;
import mil.nga.giat.mage.map.cache.CacheOverlay;
import mil.nga.giat.mage.map.cache.CacheOverlayFilter;
import mil.nga.giat.mage.map.cache.CacheProvider;
import mil.nga.giat.mage.map.cache.CacheProvider.OnCacheOverlayListener;
import mil.nga.giat.mage.map.cache.GeoPackageCacheOverlay;
import mil.nga.giat.mage.map.cache.StaticFeatureCacheOverlay;
import mil.nga.giat.mage.map.cache.XYZDirectoryCacheOverlay;
import mil.nga.giat.mage.map.download.GeoPackageDownloadManager;
import mil.nga.giat.mage.sdk.datastore.layer.Layer;
import mil.nga.giat.mage.sdk.datastore.layer.LayerHelper;
import mil.nga.giat.mage.sdk.datastore.user.Event;
import mil.nga.giat.mage.sdk.datastore.user.EventHelper;
import mil.nga.giat.mage.sdk.exceptions.LayerException;
import mil.nga.giat.mage.sdk.fetch.StaticFeatureServerFetch;
import mil.nga.giat.mage.sdk.gson.deserializer.LayerDeserializer;
import mil.nga.giat.mage.sdk.http.HttpClientManager;
import mil.nga.giat.mage.sdk.http.resource.LayerResource;
import mil.nga.giat.mage.sdk.utils.StorageUtility;
import mil.nga.giat.mage.utils.ByteUtils;
import retrofit2.Call;
import retrofit2.Callback;
import retrofit2.Response;
import retrofit2.Retrofit;
import retrofit2.converter.gson.GsonConverterFactory;

/**
 * This activity is the downloadable layers section and deals with geopackages and static features
 */
public class TileOverlayPreferenceActivity extends AppCompatActivity {

    private static final String LOG_NAME = TileOverlayPreferenceActivity.class.getName();

    private static final int PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE = 100;

    private OverlayListFragment overlayFragment;

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.cache_overlay);

        overlayFragment = (OverlayListFragment) getSupportFragmentManager().findFragmentById(R.id.overlay_fragment);
    }

    @Override
    public void onBackPressed() {
        Intent intent = new Intent();
        intent.putStringArrayListExtra(MapPreferencesActivity.OVERLAY_EXTENDED_DATA_KEY, overlayFragment.getSelectedOverlays());
        setResult(Activity.RESULT_OK, intent);


        synchronized (overlayFragment.timerLock) {
            if (this.overlayFragment.downloadRefreshTimer != null) {
                this.overlayFragment.downloadRefreshTimer.cancel();
            }
        }

        finish();
    }

    @Override
    public boolean onOptionsItemSelected(MenuItem item) {
        switch (item.getItemId()) {
            case android.R.id.home:
                onBackPressed();
                return true;
            default:
                return super.onOptionsItemSelected(item);
        }
    }

    public static class OverlayListFragment extends ListFragment implements OnCacheOverlayListener {
        private OverlayAdapter overlayAdapter;
        private final Object overlayAdapterLock = new Object();
        private ExpandableListView listView;
        private View progress;
        private MenuItem refreshButton;
        private GeoPackageDownloadManager downloadManager;
        private Timer downloadRefreshTimer;
        private final Object timerLock = new Object();

        @Override
        public void onCreate(@Nullable Bundle savedInstanceState) {
            super.onCreate(savedInstanceState);

            setHasOptionsMenu(true);

            downloadManager = new GeoPackageDownloadManager(getActivity().getApplicationContext(), new GeoPackageDownloadManager.GeoPackageDownloadListener() {
                @Override
                public void onGeoPackageDownloaded(final Layer layer, final CacheOverlay overlay) {

                    getActivity().runOnUiThread(new Runnable() {
                        @Override
                        public void run() {
                            synchronized (overlayAdapterLock) {
                                overlayAdapter.addOverlay(overlay, layer);
                                overlayAdapter.notifyDataSetChanged();
                            }
                        }
                    });
                }
            });

            overlayAdapter = new OverlayAdapter(getActivity(), downloadManager);
        }

        @Override
        public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
            View view = inflater.inflate(R.layout.fragment_cache_overlay, container, false);
            listView = view.findViewById(android.R.id.list);
            listView.setEnabled(true);
            listView.setAdapter(overlayAdapter);

            listView.setOnItemLongClickListener(new AdapterView.OnItemLongClickListener() {
                @Override
                public boolean onItemLongClick(AdapterView<?> parent, View view, int position, long id) {
                    int itemType = ExpandableListView.getPackedPositionType(id);
                    if (itemType == ExpandableListView.PACKED_POSITION_TYPE_CHILD) {
                        int childPosition = ExpandableListView.getPackedPositionChild(id);
                        int groupPosition = ExpandableListView.getPackedPositionGroup(id);
                        // Handle child row long clicks here
                        return true;
                    } else if (itemType == ExpandableListView.PACKED_POSITION_TYPE_GROUP) {
                        int groupPosition = ExpandableListView.getPackedPositionGroup(id);

                        synchronized (overlayAdapterLock) {
                            Object group = overlayAdapter.getGroup(groupPosition);
                            if (group instanceof CacheOverlay) {
                                CacheOverlay cacheOverlay = (CacheOverlay) overlayAdapter.getGroup(groupPosition);
                                deleteCacheOverlayConfirm(cacheOverlay);
                                return true;
                            }
                        }

                        return false;
                    }

                    return false;
                }
            });

            progress = view.findViewById(R.id.progress);
            progress.setVisibility(View.VISIBLE);

            if (ContextCompat.checkSelfPermission(getActivity(), Manifest.permission.READ_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) {
                if (ActivityCompat.shouldShowRequestPermissionRationale(getActivity(), Manifest.permission.READ_EXTERNAL_STORAGE)) {
                    new AlertDialog.Builder(getActivity(), R.style.AppCompatAlertDialogStyle)
                            .setTitle(R.string.overlay_access_title)
                            .setMessage(R.string.overlay_access_message)
                            .setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() {
                                @Override
                                public void onClick(DialogInterface dialog, int which) {
                                    requestPermissions(new String[]{Manifest.permission.READ_EXTERNAL_STORAGE}, PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE);
                                }
                            })
                            .create()
                            .show();

                } else {
                    requestPermissions(new String[]{Manifest.permission.READ_EXTERNAL_STORAGE}, PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE);
                }
            }

            return view;
        }

        @Override
        public void onViewCreated(View view, Bundle savedInstanceState) {
            super.onViewCreated(view, savedInstanceState);

            ListView listView = getListView();
            listView.setChoiceMode(ListView.CHOICE_MODE_MULTIPLE);
        }

        @Override
        public void onResume() {
            super.onResume();

            downloadManager.onResume();
        }

        @Override
        public void onPause() {
            super.onPause();
            
            downloadManager.onPause();

            synchronized (timerLock) {
                if (downloadRefreshTimer != null) {
                    downloadRefreshTimer.cancel();
                }
            }
        }

        @Override
        public void onCreateOptionsMenu(Menu menu, MenuInflater inflater) {
            inflater.inflate(R.menu.tile_overlay_menu, menu);

            refreshButton = menu.findItem(R.id.tile_overlay_refresh);
            refreshButton.setEnabled(false);

            // This really should be done in the onResume, but I need to have the refreshButton
            // before I register as the callback will set it to enabled.
            // The problem is that onResume gets called before this so my menu is
            // not yet setup and I will not have a handle on this button
            CacheProvider.getInstance(getActivity()).registerCacheOverlayListener(this);

            manualRefresh(refreshButton);
        }

        @Override
        public boolean onOptionsItemSelected(MenuItem item) {
            switch (item.getItemId()) {
                case R.id.tile_overlay_refresh:
                    manualRefresh(item);
                    return true;
                default:
                    return super.onOptionsItemSelected(item);
            }
        }

        /**
         * This is called when the user click the refresh button
         *
         * @param item
         */
        @UiThread
        private void manualRefresh(MenuItem item) {
            item.setEnabled(false);
            progress.setVisibility(View.VISIBLE);
            listView.setEnabled(false);
            overlayAdapter.getLayers().clear();
            overlayAdapter.notifyDataSetChanged();

            Runnable fetcher = new Runnable() {
                @Override
                public void run() {
                    final Semaphore lock = new Semaphore(1);
                    lock.drainPermits();
                    fetchGeopackageLayers(new Callback<Collection<Layer>>() {
                        @Override
                        public void onResponse(Call<Collection<Layer>> call, Response<Collection<Layer>> response) {
                            try {
                                if (response.isSuccessful()) {
                                    saveGeopackageLayers(response.body());
                                }
                            } finally {
                                lock.release();
                            }
                        }

                        @Override
                        public void onFailure(Call<Collection<Layer>> call, Throwable t) {
                            Log.e(LOG_NAME, "Error fetching event geopackage layers", t);
                            lock.release();
                        }
                    });
                    try {
                        lock.acquire();
                    }catch(InterruptedException e) {
                    }
                    fetchStaticLayers();

                    final Event event = EventHelper.getInstance(getActivity().getApplicationContext()).getCurrentEvent();

                    List<Layer> layers = new ArrayList<>();
                    try {
                        for (Layer layer : LayerHelper.getInstance(getActivity().getApplicationContext()).readByEvent(event, "GeoPackage")) {
                            if (!layer.isLoaded()) {
                                layers.add(layer);
                            }
                        }
                        for (Layer layer : LayerHelper.getInstance(getActivity().getApplicationContext()).readByEvent(event, "Feature")) {
                            if (!layer.isLoaded()) {
                                layers.add(layer);
                            }
                        }
                    } catch (LayerException e) {
                    }

                    OverlayListFragment.this.onManualRefreshComplete(layers);
                }
            };
            new Thread(fetcher).start();
        }

        private void onManualRefreshComplete(final List<Layer> layers){
            getActivity().runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    synchronized (overlayAdapterLock){
                        overlayAdapter.getLayers().removeAll(layers);
                        overlayAdapter.getLayers().addAll(layers);
                        overlayAdapter.notifyDataSetChanged();
                    }
                    refreshButton.setEnabled(true);
                    progress.setVisibility(View.GONE);
                    listView.setEnabled(true);
                }
            });
        }

        /**
         * This reads the remote layers from the server but does not download them
         *
         * @return
         */
        private List<Layer> fetchStaticLayers(){
            StaticFeatureServerFetch staticFeatureServerFetch = new StaticFeatureServerFetch(getContext());
            return staticFeatureServerFetch.fetch(false, null);
        }

        private void fetchGeopackageLayers(Callback<Collection<Layer>> callback) {
            Context context = getContext();
            Event event = EventHelper.getInstance(context).getCurrentEvent();
            String baseUrl = PreferenceManager.getDefaultSharedPreferences(context).getString(context.getString(mil.nga.giat.mage.sdk.R.string.serverURLKey), context.getString(mil.nga.giat.mage.sdk.R.string.serverURLDefaultValue));
            Retrofit retrofit = new Retrofit.Builder()
                    .baseUrl(baseUrl)
                    .addConverterFactory(GsonConverterFactory.create(LayerDeserializer.getGsonBuilder(event)))
                    .client(HttpClientManager.getInstance().httpClient())
                    .build();

            LayerResource.LayerService service = retrofit.create(LayerResource.LayerService.class);

            service.getLayers(event.getRemoteId(), "GeoPackage").enqueue(callback);
        }

        private void saveGeopackageLayers(Collection<Layer> layers) {
            Context context = getActivity().getApplicationContext();
            LayerHelper layerHelper = LayerHelper.getInstance(context);
            try {
                layerHelper.deleteAll("GeoPackage");

                GeoPackageManager manager = GeoPackageFactory.getManager(context);
                for (Layer layer : layers) {
                    // Check if its loaded
                    File file = new File(context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS), String.format("MAGE/geopackages/%s/%s", layer.getRemoteId(), layer.getFileName()));
                    if (file.exists() && manager.existsAtExternalFile(file)) {
                        layer.setLoaded(true);
                    }
                    layerHelper.create(layer);
                }
            } catch (LayerException e) {
                Log.e(LOG_NAME, "Error saving geopackage layers", e);
            }
        }

        @Override
        public void onDestroy() {
            super.onDestroy();

            CacheProvider.getInstance(getActivity()).unregisterCacheOverlayListener(this);
        }

        @Override
        public void onCacheOverlay(final List<CacheOverlay> cacheOverlays) {
            final Event event = EventHelper.getInstance(getActivity().getApplicationContext()).getCurrentEvent();

            synchronized (overlayAdapterLock) {
                overlayAdapter.getOverlays().removeAll(cacheOverlays);

                for (CacheOverlay overlay : cacheOverlays) {
                    if(overlay instanceof StaticFeatureCacheOverlay) {
                        overlayAdapter.getOverlays().add(overlay);
                    }
                }
            }

            overlayAdapter.notifyDataSetChanged();

            List<Layer> geopackages = Collections.EMPTY_LIST;
            try {
                geopackages = LayerHelper.getInstance(getActivity().getApplicationContext()).readByEvent(event, "GeoPackage");
            } catch (LayerException e) {
            }

            downloadManager.reconcileDownloads(geopackages, new GeoPackageDownloadManager.GeoPackageLoadListener() {
                @Override
                public void onReady(List<Layer> layers) {

                    synchronized (overlayAdapterLock){
                        overlayAdapter.getLayers().removeAll(layers);
                        overlayAdapter.getLayers().addAll(layers);
                        List<CacheOverlay> filtered = new CacheOverlayFilter(getContext(), event).filter(cacheOverlays);
                        for(CacheOverlay overlay : filtered) {
                            if(overlay instanceof GeoPackageCacheOverlay) {
                                overlayAdapter.getOverlays().add(overlay);
                            }
                        }
                        overlayAdapter.notifyDataSetChanged();
                    }

                    synchronized (timerLock) {
                        downloadRefreshTimer = new Timer();
                        downloadRefreshTimer.schedule(new TimerTask() {
                            private boolean canceled = false;

                            @Override
                            public void run() {
                                synchronized (timerLock) {
                                    if (!canceled) {
                                        updateGeopackageDownloadProgress();
                                    }
                                }
                            }

                            @Override
                            public boolean cancel() {
                                synchronized (timerLock) {
                                    canceled = true;
                                }
                                return super.cancel();
                            }
                        }, 0, 2000);
                    }
                }
            });
        }

        @Override
        public void onRequestPermissionsResult(int requestCode, String permissions[], int[] grantResults) {
            super.onRequestPermissionsResult(requestCode, permissions, grantResults);

            switch (requestCode) {
                case PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE: {
                    if(grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
                        CacheProvider.getInstance(getActivity()).refreshTileOverlays();
                    }

                    break;
                }
            }
        }

        /**
         * Get the selected cache overlays and child cache overlays
         *
         * @return
         */
        @UiThread
        public ArrayList<String> getSelectedOverlays() {
            ArrayList<String> overlays = new ArrayList<>();
            for (CacheOverlay cacheOverlay : CacheProvider.getInstance(getContext()).getCacheOverlays()) {
                if(cacheOverlay instanceof GeoPackageCacheOverlay) {
                    boolean childAdded = false;
                    for (CacheOverlay childCache : cacheOverlay.getChildren()) {
                        if (childCache.isEnabled()) {
                            overlays.add(childCache.getCacheName());
                            childAdded = true;
                        }
                    }

                    if (!childAdded && cacheOverlay.isEnabled()) {
                        overlays.add(cacheOverlay.getCacheName());
                    }
                }
            }

            return overlays;
        }

        private void updateGeopackageDownloadProgress() {
            /*
            TODO this method was difficult to put on the UI thread, so as it sits, it is not robust
            */
            if(getActivity() == null) {
                return;
            }
            getActivity().runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    synchronized (overlayAdapterLock) {
                        try {
                            List<Layer> layers = overlayAdapter.getLayers();
                            for (int i = 0; i < layers.size(); i++) {
                                final Layer layer = layers.get(i);

                                if(!layer.getType().equalsIgnoreCase("geopackage")){
                                    continue;
                                }

                                if (layer.getDownloadId() != null && !layer.isLoaded()) {
                                    // layer is currently downloading, get progress and refresh view
                                    final View view = listView.getChildAt(i - listView.getFirstVisiblePosition());
                                    if (view == null) {
                                        continue;
                                    }

                                    Activity activity = getActivity();
                                    if (activity == null) {
                                        continue;
                                    }

                                    overlayAdapter.updateDownloadProgress(view, downloadManager.getProgress(layer), layer.getFileSize());
                                }
                            }
                        }catch(Exception e) {
                            //This is sort of a hack to manage the craziness in the lifecycle
                            //Typically this is an NPE
                        }
                    }
                }
            });
        }

        /**
         * Delete the cache overlay
         * @param cacheOverlay
         */
        private void deleteCacheOverlayConfirm(final CacheOverlay cacheOverlay) {
            getActivity().runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    AlertDialog deleteDialog = new AlertDialog.Builder(getActivity())
                            .setTitle("Delete Cache")
                            .setMessage("Delete " + cacheOverlay.getName() + " Cache?")
                            .setPositiveButton("Delete",
                                    new DialogInterface.OnClickListener() {
                                        @Override
                                        public void onClick(DialogInterface dialog,
                                                            int which) {
                                            deleteCacheOverlay(cacheOverlay);
                                        }
                                    })

                            .setNegativeButton(getString(R.string.cancel),
                                    new DialogInterface.OnClickListener() {
                                        @Override
                                        public void onClick(DialogInterface dialog,
                                                            int which) {
                                            dialog.dismiss();
                                        }<commit_msg>Changing wording on delete of downloaded layer from using the term cache to using the term layer<commit_after>package mil.nga.giat.mage.map.preference;

import android.Manifest;
import android.app.Activity;
import android.content.Context;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.os.Environment;
import android.preference.PreferenceManager;
import android.support.annotation.Nullable;
import android.support.annotation.UiThread;
import android.support.v4.app.ActivityCompat;
import android.support.v4.app.ListFragment;
import android.support.v4.content.ContextCompat;
import android.support.v7.app.AlertDialog;
import android.support.v7.app.AppCompatActivity;
import android.text.format.Formatter;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.Menu;
import android.view.MenuInflater;
import android.view.MenuItem;
import android.view.View;
import android.view.ViewGroup;
import android.widget.AdapterView;
import android.widget.BaseExpandableListAdapter;
import android.widget.CheckBox;
import android.widget.ExpandableListView;
import android.widget.ImageView;
import android.widget.ListView;
import android.widget.ProgressBar;
import android.widget.TextView;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Semaphore;

import mil.nga.geopackage.GeoPackageManager;
import mil.nga.geopackage.factory.GeoPackageFactory;
import mil.nga.giat.mage.R;
import mil.nga.giat.mage.cache.CacheUtils;
import mil.nga.giat.mage.map.cache.CacheOverlay;
import mil.nga.giat.mage.map.cache.CacheOverlayFilter;
import mil.nga.giat.mage.map.cache.CacheProvider;
import mil.nga.giat.mage.map.cache.CacheProvider.OnCacheOverlayListener;
import mil.nga.giat.mage.map.cache.GeoPackageCacheOverlay;
import mil.nga.giat.mage.map.cache.StaticFeatureCacheOverlay;
import mil.nga.giat.mage.map.cache.XYZDirectoryCacheOverlay;
import mil.nga.giat.mage.map.download.GeoPackageDownloadManager;
import mil.nga.giat.mage.sdk.datastore.layer.Layer;
import mil.nga.giat.mage.sdk.datastore.layer.LayerHelper;
import mil.nga.giat.mage.sdk.datastore.user.Event;
import mil.nga.giat.mage.sdk.datastore.user.EventHelper;
import mil.nga.giat.mage.sdk.exceptions.LayerException;
import mil.nga.giat.mage.sdk.fetch.StaticFeatureServerFetch;
import mil.nga.giat.mage.sdk.gson.deserializer.LayerDeserializer;
import mil.nga.giat.mage.sdk.http.HttpClientManager;
import mil.nga.giat.mage.sdk.http.resource.LayerResource;
import mil.nga.giat.mage.sdk.utils.StorageUtility;
import mil.nga.giat.mage.utils.ByteUtils;
import retrofit2.Call;
import retrofit2.Callback;
import retrofit2.Response;
import retrofit2.Retrofit;
import retrofit2.converter.gson.GsonConverterFactory;

/**
 * This activity is the downloadable layers section and deals with geopackages and static features
 */
public class TileOverlayPreferenceActivity extends AppCompatActivity {

    private static final String LOG_NAME = TileOverlayPreferenceActivity.class.getName();

    private static final int PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE = 100;

    private OverlayListFragment overlayFragment;

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.cache_overlay);

        overlayFragment = (OverlayListFragment) getSupportFragmentManager().findFragmentById(R.id.overlay_fragment);
    }

    @Override
    public void onBackPressed() {
        Intent intent = new Intent();
        intent.putStringArrayListExtra(MapPreferencesActivity.OVERLAY_EXTENDED_DATA_KEY, overlayFragment.getSelectedOverlays());
        setResult(Activity.RESULT_OK, intent);


        synchronized (overlayFragment.timerLock) {
            if (this.overlayFragment.downloadRefreshTimer != null) {
                this.overlayFragment.downloadRefreshTimer.cancel();
            }
        }

        finish();
    }

    @Override
    public boolean onOptionsItemSelected(MenuItem item) {
        switch (item.getItemId()) {
            case android.R.id.home:
                onBackPressed();
                return true;
            default:
                return super.onOptionsItemSelected(item);
        }
    }

    public static class OverlayListFragment extends ListFragment implements OnCacheOverlayListener {
        private OverlayAdapter overlayAdapter;
        private final Object overlayAdapterLock = new Object();
        private ExpandableListView listView;
        private View progress;
        private MenuItem refreshButton;
        private GeoPackageDownloadManager downloadManager;
        private Timer downloadRefreshTimer;
        private final Object timerLock = new Object();

        @Override
        public void onCreate(@Nullable Bundle savedInstanceState) {
            super.onCreate(savedInstanceState);

            setHasOptionsMenu(true);

            downloadManager = new GeoPackageDownloadManager(getActivity().getApplicationContext(), new GeoPackageDownloadManager.GeoPackageDownloadListener() {
                @Override
                public void onGeoPackageDownloaded(final Layer layer, final CacheOverlay overlay) {

                    getActivity().runOnUiThread(new Runnable() {
                        @Override
                        public void run() {
                            synchronized (overlayAdapterLock) {
                                overlayAdapter.addOverlay(overlay, layer);
                                overlayAdapter.notifyDataSetChanged();
                            }
                        }
                    });
                }
            });

            overlayAdapter = new OverlayAdapter(getActivity(), downloadManager);
        }

        @Override
        public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
            View view = inflater.inflate(R.layout.fragment_cache_overlay, container, false);
            listView = view.findViewById(android.R.id.list);
            listView.setEnabled(true);
            listView.setAdapter(overlayAdapter);

            listView.setOnItemLongClickListener(new AdapterView.OnItemLongClickListener() {
                @Override
                public boolean onItemLongClick(AdapterView<?> parent, View view, int position, long id) {
                    int itemType = ExpandableListView.getPackedPositionType(id);
                    if (itemType == ExpandableListView.PACKED_POSITION_TYPE_CHILD) {
                        int childPosition = ExpandableListView.getPackedPositionChild(id);
                        int groupPosition = ExpandableListView.getPackedPositionGroup(id);
                        // Handle child row long clicks here
                        return true;
                    } else if (itemType == ExpandableListView.PACKED_POSITION_TYPE_GROUP) {
                        int groupPosition = ExpandableListView.getPackedPositionGroup(id);

                        synchronized (overlayAdapterLock) {
                            Object group = overlayAdapter.getGroup(groupPosition);
                            if (group instanceof CacheOverlay) {
                                CacheOverlay cacheOverlay = (CacheOverlay) overlayAdapter.getGroup(groupPosition);
                                deleteCacheOverlayConfirm(cacheOverlay);
                                return true;
                            }
                        }

                        return false;
                    }

                    return false;
                }
            });

            progress = view.findViewById(R.id.progress);
            progress.setVisibility(View.VISIBLE);

            if (ContextCompat.checkSelfPermission(getActivity(), Manifest.permission.READ_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) {
                if (ActivityCompat.shouldShowRequestPermissionRationale(getActivity(), Manifest.permission.READ_EXTERNAL_STORAGE)) {
                    new AlertDialog.Builder(getActivity(), R.style.AppCompatAlertDialogStyle)
                            .setTitle(R.string.overlay_access_title)
                            .setMessage(R.string.overlay_access_message)
                            .setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() {
                                @Override
                                public void onClick(DialogInterface dialog, int which) {
                                    requestPermissions(new String[]{Manifest.permission.READ_EXTERNAL_STORAGE}, PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE);
                                }
                            })
                            .create()
                            .show();

                } else {
                    requestPermissions(new String[]{Manifest.permission.READ_EXTERNAL_STORAGE}, PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE);
                }
            }

            return view;
        }

        @Override
        public void onViewCreated(View view, Bundle savedInstanceState) {
            super.onViewCreated(view, savedInstanceState);

            ListView listView = getListView();
            listView.setChoiceMode(ListView.CHOICE_MODE_MULTIPLE);
        }

        @Override
        public void onResume() {
            super.onResume();

            downloadManager.onResume();
        }

        @Override
        public void onPause() {
            super.onPause();
            
            downloadManager.onPause();

            synchronized (timerLock) {
                if (downloadRefreshTimer != null) {
                    downloadRefreshTimer.cancel();
                }
            }
        }

        @Override
        public void onCreateOptionsMenu(Menu menu, MenuInflater inflater) {
            inflater.inflate(R.menu.tile_overlay_menu, menu);

            refreshButton = menu.findItem(R.id.tile_overlay_refresh);
            refreshButton.setEnabled(false);

            // This really should be done in the onResume, but I need to have the refreshButton
            // before I register as the callback will set it to enabled.
            // The problem is that onResume gets called before this so my menu is
            // not yet setup and I will not have a handle on this button
            CacheProvider.getInstance(getActivity()).registerCacheOverlayListener(this);

            manualRefresh(refreshButton);
        }

        @Override
        public boolean onOptionsItemSelected(MenuItem item) {
            switch (item.getItemId()) {
                case R.id.tile_overlay_refresh:
                    manualRefresh(item);
                    return true;
                default:
                    return super.onOptionsItemSelected(item);
            }
        }

        /**
         * This is called when the user click the refresh button
         *
         * @param item
         */
        @UiThread
        private void manualRefresh(MenuItem item) {
            item.setEnabled(false);
            progress.setVisibility(View.VISIBLE);
            listView.setEnabled(false);
            overlayAdapter.getLayers().clear();
            overlayAdapter.notifyDataSetChanged();

            Runnable fetcher = new Runnable() {
                @Override
                public void run() {
                    final Semaphore lock = new Semaphore(1);
                    lock.drainPermits();
                    fetchGeopackageLayers(new Callback<Collection<Layer>>() {
                        @Override
                        public void onResponse(Call<Collection<Layer>> call, Response<Collection<Layer>> response) {
                            try {
                                if (response.isSuccessful()) {
                                    saveGeopackageLayers(response.body());
                                }
                            } finally {
                                lock.release();
                            }
                        }

                        @Override
                        public void onFailure(Call<Collection<Layer>> call, Throwable t) {
                            Log.e(LOG_NAME, "Error fetching event geopackage layers", t);
                            lock.release();
                        }
                    });
                    try {
                        lock.acquire();
                    }catch(InterruptedException e) {
                    }
                    fetchStaticLayers();

                    final Event event = EventHelper.getInstance(getActivity().getApplicationContext()).getCurrentEvent();

                    List<Layer> layers = new ArrayList<>();
                    try {
                        for (Layer layer : LayerHelper.getInstance(getActivity().getApplicationContext()).readByEvent(event, "GeoPackage")) {
                            if (!layer.isLoaded()) {
                                layers.add(layer);
                            }
                        }
                        for (Layer layer : LayerHelper.getInstance(getActivity().getApplicationContext()).readByEvent(event, "Feature")) {
                            if (!layer.isLoaded()) {
                                layers.add(layer);
                            }
                        }
                    } catch (LayerException e) {
                    }

                    OverlayListFragment.this.onManualRefreshComplete(layers);
                }
            };
            new Thread(fetcher).start();
        }

        private void onManualRefreshComplete(final List<Layer> layers){
            getActivity().runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    synchronized (overlayAdapterLock){
                        overlayAdapter.getLayers().removeAll(layers);
                        overlayAdapter.getLayers().addAll(layers);
                        overlayAdapter.notifyDataSetChanged();
                    }
                    refreshButton.setEnabled(true);
                    progress.setVisibility(View.GONE);
                    listView.setEnabled(true);
                }
            });
        }

        /**
         * This reads the remote layers from the server but does not download them
         *
         * @return
         */
        private List<Layer> fetchStaticLayers(){
            StaticFeatureServerFetch staticFeatureServerFetch = new StaticFeatureServerFetch(getContext());
            return staticFeatureServerFetch.fetch(false, null);
        }

        private void fetchGeopackageLayers(Callback<Collection<Layer>> callback) {
            Context context = getContext();
            Event event = EventHelper.getInstance(context).getCurrentEvent();
            String baseUrl = PreferenceManager.getDefaultSharedPreferences(context).getString(context.getString(mil.nga.giat.mage.sdk.R.string.serverURLKey), context.getString(mil.nga.giat.mage.sdk.R.string.serverURLDefaultValue));
            Retrofit retrofit = new Retrofit.Builder()
                    .baseUrl(baseUrl)
                    .addConverterFactory(GsonConverterFactory.create(LayerDeserializer.getGsonBuilder(event)))
                    .client(HttpClientManager.getInstance().httpClient())
                    .build();

            LayerResource.LayerService service = retrofit.create(LayerResource.LayerService.class);

            service.getLayers(event.getRemoteId(), "GeoPackage").enqueue(callback);
        }

        private void saveGeopackageLayers(Collection<Layer> layers) {
            Context context = getActivity().getApplicationContext();
            LayerHelper layerHelper = LayerHelper.getInstance(context);
            try {
                layerHelper.deleteAll("GeoPackage");

                GeoPackageManager manager = GeoPackageFactory.getManager(context);
                for (Layer layer : layers) {
                    // Check if its loaded
                    File file = new File(context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS), String.format("MAGE/geopackages/%s/%s", layer.getRemoteId(), layer.getFileName()));
                    if (file.exists() && manager.existsAtExternalFile(file)) {
                        layer.setLoaded(true);
                    }
                    layerHelper.create(layer);
                }
            } catch (LayerException e) {
                Log.e(LOG_NAME, "Error saving geopackage layers", e);
            }
        }

        @Override
        public void onDestroy() {
            super.onDestroy();

            CacheProvider.getInstance(getActivity()).unregisterCacheOverlayListener(this);
        }

        @Override
        public void onCacheOverlay(final List<CacheOverlay> cacheOverlays) {
            final Event event = EventHelper.getInstance(getActivity().getApplicationContext()).getCurrentEvent();

            synchronized (overlayAdapterLock) {
                overlayAdapter.getOverlays().removeAll(cacheOverlays);

                for (CacheOverlay overlay : cacheOverlays) {
                    if(overlay instanceof StaticFeatureCacheOverlay) {
                        overlayAdapter.getOverlays().add(overlay);
                    }
                }
            }

            overlayAdapter.notifyDataSetChanged();

            List<Layer> geopackages = Collections.EMPTY_LIST;
            try {
                geopackages = LayerHelper.getInstance(getActivity().getApplicationContext()).readByEvent(event, "GeoPackage");
            } catch (LayerException e) {
            }

            downloadManager.reconcileDownloads(geopackages, new GeoPackageDownloadManager.GeoPackageLoadListener() {
                @Override
                public void onReady(List<Layer> layers) {

                    synchronized (overlayAdapterLock){
                        overlayAdapter.getLayers().removeAll(layers);
                        overlayAdapter.getLayers().addAll(layers);
                        List<CacheOverlay> filtered = new CacheOverlayFilter(getContext(), event).filter(cacheOverlays);
                        for(CacheOverlay overlay : filtered) {
                            if(overlay instanceof GeoPackageCacheOverlay) {
                                overlayAdapter.getOverlays().add(overlay);
                            }
                        }
                        overlayAdapter.notifyDataSetChanged();
                    }

                    synchronized (timerLock) {
                        downloadRefreshTimer = new Timer();
                        downloadRefreshTimer.schedule(new TimerTask() {
                            private boolean canceled = false;

                            @Override
                            public void run() {
                                synchronized (timerLock) {
                                    if (!canceled) {
                                        updateGeopackageDownloadProgress();
                                    }
                                }
                            }

                            @Override
                            public boolean cancel() {
                                synchronized (timerLock) {
                                    canceled = true;
                                }
                                return super.cancel();
                            }
                        }, 0, 2000);
                    }
                }
            });
        }

        @Override
        public void onRequestPermissionsResult(int requestCode, String permissions[], int[] grantResults) {
            super.onRequestPermissionsResult(requestCode, permissions, grantResults);

            switch (requestCode) {
                case PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE: {
                    if(grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
                        CacheProvider.getInstance(getActivity()).refreshTileOverlays();
                    }

                    break;
                }
            }
        }

        /**
         * Get the selected cache overlays and child cache overlays
         *
         * @return
         */
        @UiThread
        public ArrayList<String> getSelectedOverlays() {
            ArrayList<String> overlays = new ArrayList<>();
            for (CacheOverlay cacheOverlay : CacheProvider.getInstance(getContext()).getCacheOverlays()) {
                if(cacheOverlay instanceof GeoPackageCacheOverlay) {
                    boolean childAdded = false;
                    for (CacheOverlay childCache : cacheOverlay.getChildren()) {
                        if (childCache.isEnabled()) {
                            overlays.add(childCache.getCacheName());
                            childAdded = true;
                        }
                    }

                    if (!childAdded && cacheOverlay.isEnabled()) {
                        overlays.add(cacheOverlay.getCacheName());
                    }
                }
            }

            return overlays;
        }

        private void updateGeopackageDownloadProgress() {
            /*
            TODO this method was difficult to put on the UI thread, so as it sits, it is not robust
            */
            if(getActivity() == null) {
                return;
            }
            getActivity().runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    synchronized (overlayAdapterLock) {
                        try {
                            List<Layer> layers = overlayAdapter.getLayers();
                            for (int i = 0; i < layers.size(); i++) {
                                final Layer layer = layers.get(i);

                                if(!layer.getType().equalsIgnoreCase("geopackage")){
                                    continue;
                                }

                                if (layer.getDownloadId() != null && !layer.isLoaded()) {
                                    // layer is currently downloading, get progress and refresh view
                                    final View view = listView.getChildAt(i - listView.getFirstVisiblePosition());
                                    if (view == null) {
                                        continue;
                                    }

                                    Activity activity = getActivity();
                                    if (activity == null) {
                                        continue;
                                    }

                                    overlayAdapter.updateDownloadProgress(view, downloadManager.getProgress(layer), layer.getFileSize());
                                }
                            }
                        }catch(Exception e) {
                            //This is sort of a hack to manage the craziness in the lifecycle
                            //Typically this is an NPE
                        }
                    }
                }
            });
        }

        /**
         * Delete the cache overlay
         * @param cacheOverlay
         */
        private void deleteCacheOverlayConfirm(final CacheOverlay cacheOverlay) {
            getActivity().runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    AlertDialog deleteDialog = new AlertDialog.Builder(getActivity())
                            .setTitle("Delete Layer")
                            .setMessage("Delete " + cacheOverlay.getName() + " Layer?")
                            .setPositiveButton("Delete",
                                    new DialogInterface.OnClickListener() {
                                        @Override
                                        public void onClick(DialogInterface dialog,
                                                            int which) {
                                            deleteCacheOverlay(cacheOverlay);
                                        }
                                    })

                            .setNegativeButton(getString(R.string.cancel),
                                    new DialogInterface.OnClickListener() {
                                        @Override
                                        public void onClick(DialogInterface dialog,
                                                            int which) {
                                            dialog.dismiss();
                                        }
{'fraction_non_alphanumeric': 0.07888855330715795, 'fraction_numerical': 0.0008859357696566999, 'mean_word_length': 1.531783657032166, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 133, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3407252', 'n_tokens_mistral': 11550, 'n_tokens_neox': 10889, 'n_words': 2110}