<commit_before>package com.kaltura.playersdk.helpers;

import android.annotation.TargetApi;
import android.content.Context;
import android.net.Uri;
import android.os.Build;
import android.text.TextUtils;
import android.util.Log;
import android.webkit.WebResourceRequest;
import android.webkit.WebResourceResponse;

import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Iterator;

/**
 * Created by nissimpardo on 25/10/15.
 */
public class CacheManager {
    private static final String TAG = "CacheManager";
    public static final String CACHED_STRINGS_JSON = "CachedStrings.json";
    private static CacheManager ourInstance = new CacheManager();
    private JSONObject mCacheConditions;
    private Context mContext;
    private CacheSQLHelper mSQLHelper;
    private String mHost;
    private float mCacheSize = 0;
    private String mCachePath;

    public static CacheManager getInstance() {
        return ourInstance;
    }

    private CacheManager() {
    }

    public void setContext(Context context) {
        mContext = context;
        if (mSQLHelper == null) {
            mSQLHelper = new CacheSQLHelper(context);
        }
    }

    public void setHost(String host) {
        mHost = host;
    }

    public void setCacheSize(float cacheSize) {
        mCacheSize = cacheSize;
    }

    private String getCachePath() {
        if (mCachePath == null) {
            mCachePath = mContext.getFilesDir() + "/kaltura/";
            File cacheDir = new File(mCachePath);
            if (!cacheDir.exists()) {
                cacheDir.mkdirs();
            }
        }
        return mCachePath;
    }

    private static String readAssetAsString(Context context, String asset) {
        BufferedInputStream is = null;
        String string = null;
        try {
            is = new BufferedInputStream(context.getAssets().open(asset));
            ByteArrayOutputStream bytes = new ByteArrayOutputStream(1024);
            int oneByte;
            while ((oneByte = is.read()) != -1) {
                bytes.write(oneByte);
            }
            string = bytes.toString();
        } catch (IOException e) {
            Log.e(TAG, "Can't read " + asset, e);
        } finally {
            if (is != null) {
                try {
                    is.close();
                } catch (IOException e) {
                    Log.e(TAG, "Can't close " + asset, e);
                }
            }
        }
        return string;
    }
    
    private JSONObject getCacheConditions() {
        
        if (mCacheConditions == null) {
            String string = readAssetAsString(mContext, CACHED_STRINGS_JSON);
            if (string != null) {
                try {
                    mCacheConditions = new JSONObject(string);
                } catch (JSONException e) {
                    Log.e(TAG, "Invalid json", e);
                }
            }
        }

        return mCacheConditions;
    }

    public boolean shouldStore(Uri uri) throws URISyntaxException {
        String uriString = uri.toString();
        JSONObject conditions = getCacheConditions();

        String key = mHost.equals(uri.getHost()) ? "withDomain" : "substring";

        try {
            JSONObject object = conditions.getJSONObject(key);
            for (Iterator<String> it = object.keys(); it.hasNext(); ) {
                String str = it.next();
                if (uriString.contains(str)) {
                    return true;
                }
            }

        } catch (JSONException e) {
            Log.w(TAG, "Can't find required configuration data in " + CACHED_STRINGS_JSON, e);
        }

        return false;
    }

    private void deleteLessUsedFiles(long newCacheSize) {
        long freeBytesInternal = new File(mContext.getFilesDir().getAbsoluteFile().toString()).getFreeSpace();
//        long freeBytesExternal = new File(getExternalFilesDir(null).toString()).getFreeSpace();
        long cahceSize = (long)(mCacheSize * 1024 * 1024);
        long actualCacheSize = Math.min(cahceSize, freeBytesInternal);
        Log.d("KalturaCacheSize", String.valueOf(mSQLHelper.cacheSize()));
        boolean shouldDeleteLessUsedFiles = mSQLHelper.cacheSize() + newCacheSize > actualCacheSize;
        if (shouldDeleteLessUsedFiles) {
            mSQLHelper.deleteLessUsedFiles(mSQLHelper.cacheSize() + newCacheSize - actualCacheSize, new CacheSQLHelper.KSQLHelperDeleteListener() {
                @Override
                public void fileDeleted(String fileId) {
                    File lessUsedFile = new File(getCachePath() + fileId);
                    if (lessUsedFile.exists()) {
                        lessUsedFile.delete();
                    }
                }
            });
        }
    }

    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
    public WebResourceResponse getResponse(WebResourceRequest request) throws IOException, URISyntaxException {
        Uri requestUrl = request.getUrl();
        if (!shouldStore(requestUrl)) {
            return null;
        }
        InputStream inputStream = null;
        final String fileName = KStringUtilities.md5(requestUrl.toString());
        String filePath = getCachePath() + fileName;
        String contentType = null;
        String encoding = null;
        HashMap<String, Object> fileParams = mSQLHelper.fetchParamsForFile(fileName);
        if (mSQLHelper.sizeForId(fileName) > 0 && fileParams != null) {
            FileInputStream fileInputStream = new FileInputStream(filePath);
            inputStream = new BufferedInputStream(fileInputStream);
            contentType = (String)fileParams.get(CacheSQLHelper.MimeType);
            encoding = (String)fileParams.get(CacheSQLHelper.Encoding);
            mSQLHelper.updateDate(fileName);
        } else {
            URL url = null;
            HttpURLConnection connection = null;
            try {
                url = new URL(requestUrl.toString());
                connection = (HttpURLConnection) url.openConnection();
                connection.setRequestMethod(request.getMethod());
                for (String key : request.getRequestHeaders().keySet()) {
                    connection.setRequestProperty(key, request.getRequestHeaders().get(key));
                }
                connection.connect();
                contentType = connection.getContentType();
                if (contentType == null) {
                    contentType = "";
                }
                String[] contentTypeParts = TextUtils.split(contentType, ";");
                if (contentTypeParts.length >= 2) {
                    contentType = contentTypeParts[0].trim();
                    encoding = contentTypeParts[1].trim();
                }
                mSQLHelper.addFile(fileName, contentType, encoding);
                inputStream = new CachingInputStream(filePath, url.openStream(), new CachingInputStream.KInputStreamListener() {
                    @Override
                    public void streamClosed(long fileSize, String filePath) {
                        int trimLength = (int)getCachePath().length();
                        String fileId = filePath.substring(trimLength);
                        mSQLHelper.updateFileSize(fileId, fileSize);
                        deleteLessUsedFiles(fileSize);
                    }
                });

            } catch (Exception e) {
                Log.e(TAG, "Error loading URL " + requestUrl, e);
                return null;
            }
        }
        Log.d("Stored Responses", contentType + " " + encoding + " " + requestUrl.toString());
        return new WebResourceResponse(contentType, encoding, inputStream);
    }
}
<commit_msg>CacheManager: read json config using new utility.<commit_after>package com.kaltura.playersdk.helpers;

import android.annotation.TargetApi;
import android.content.Context;
import android.net.Uri;
import android.os.Build;
import android.text.TextUtils;
import android.util.Log;
import android.webkit.WebResourceRequest;
import android.webkit.WebResourceResponse;

import com.kaltura.playersdk.Utilities;

import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Iterator;

/**
 * Created by nissimpardo on 25/10/15.
 */
public class CacheManager {
    private static final String TAG = "CacheManager";
    public static final String CACHED_STRINGS_JSON = "CachedStrings.json";
    private static CacheManager ourInstance = new CacheManager();
    private JSONObject mCacheConditions;
    private Context mContext;
    private CacheSQLHelper mSQLHelper;
    private String mHost;
    private float mCacheSize = 0;
    private String mCachePath;

    public static CacheManager getInstance() {
        return ourInstance;
    }

    private CacheManager() {
    }

    public void setContext(Context context) {
        mContext = context;
        if (mSQLHelper == null) {
            mSQLHelper = new CacheSQLHelper(context);
        }
    }

    public void setHost(String host) {
        mHost = host;
    }

    public void setCacheSize(float cacheSize) {
        mCacheSize = cacheSize;
    }

    private String getCachePath() {
        if (mCachePath == null) {
            mCachePath = mContext.getFilesDir() + "/kaltura/";
            File cacheDir = new File(mCachePath);
            if (!cacheDir.exists()) {
                cacheDir.mkdirs();
            }
        }
        return mCachePath;
    }
    
    private JSONObject getCacheConditions() {
        
        if (mCacheConditions == null) {
            String string = Utilities.readAssetToString(mContext, CACHED_STRINGS_JSON);
            if (string != null) {
                try {
                    mCacheConditions = new JSONObject(string);
                } catch (JSONException e) {
                    Log.e(TAG, "Invalid json", e);
                }
            }
        }

        return mCacheConditions;
    }

    public boolean shouldStore(Uri uri) throws URISyntaxException {
        String uriString = uri.toString();
        JSONObject conditions = getCacheConditions();

        String key = mHost.equals(uri.getHost()) ? "withDomain" : "substring";

        try {
            JSONObject object = conditions.getJSONObject(key);
            for (Iterator<String> it = object.keys(); it.hasNext(); ) {
                String str = it.next();
                if (uriString.contains(str)) {
                    return true;
                }
            }

        } catch (JSONException e) {
            Log.w(TAG, "Can't find required configuration data in " + CACHED_STRINGS_JSON, e);
        }

        return false;
    }

    private void deleteLessUsedFiles(long newCacheSize) {
        long freeBytesInternal = new File(mContext.getFilesDir().getAbsoluteFile().toString()).getFreeSpace();
//        long freeBytesExternal = new File(getExternalFilesDir(null).toString()).getFreeSpace();
        long cahceSize = (long)(mCacheSize * 1024 * 1024);
        long actualCacheSize = Math.min(cahceSize, freeBytesInternal);
        Log.d("KalturaCacheSize", String.valueOf(mSQLHelper.cacheSize()));
        boolean shouldDeleteLessUsedFiles = mSQLHelper.cacheSize() + newCacheSize > actualCacheSize;
        if (shouldDeleteLessUsedFiles) {
            mSQLHelper.deleteLessUsedFiles(mSQLHelper.cacheSize() + newCacheSize - actualCacheSize, new CacheSQLHelper.KSQLHelperDeleteListener() {
                @Override
                public void fileDeleted(String fileId) {
                    File lessUsedFile = new File(getCachePath() + fileId);
                    if (lessUsedFile.exists()) {
                        lessUsedFile.delete();
                    }
                }
            });
        }
    }

    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
    public WebResourceResponse getResponse(WebResourceRequest request) throws IOException, URISyntaxException {
        Uri requestUrl = request.getUrl();
        if (!shouldStore(requestUrl)) {
            return null;
        }
        InputStream inputStream = null;
        final String fileName = KStringUtilities.md5(requestUrl.toString());
        String filePath = getCachePath() + fileName;
        String contentType = null;
        String encoding = null;
        HashMap<String, Object> fileParams = mSQLHelper.fetchParamsForFile(fileName);
        if (mSQLHelper.sizeForId(fileName) > 0 && fileParams != null) {
            FileInputStream fileInputStream = new FileInputStream(filePath);
            inputStream = new BufferedInputStream(fileInputStream);
            contentType = (String)fileParams.get(CacheSQLHelper.MimeType);
            encoding = (String)fileParams.get(CacheSQLHelper.Encoding);
            mSQLHelper.updateDate(fileName);
        } else {
            URL url = null;
            HttpURLConnection connection = null;
            try {
                url = new URL(requestUrl.toString());
                connection = (HttpURLConnection) url.openConnection();
                connection.setRequestMethod(request.getMethod());
                for (String key : request.getRequestHeaders().keySet()) {
                    connection.setRequestProperty(key, request.getRequestHeaders().get(key));
                }
                connection.connect();
                contentType = connection.getContentType();
                if (contentType == null) {
                    contentType = "";
                }
                String[] contentTypeParts = TextUtils.split(contentType, ";");
                if (contentTypeParts.length >= 2) {
                    contentType = contentTypeParts[0].trim();
                    encoding = contentTypeParts[1].trim();
                }
                mSQLHelper.addFile(fileName, contentType, encoding);
                inputStream = new CachingInputStream(filePath, url.openStream(), new CachingInputStream.KInputStreamListener() {
                    @Override
                    public void streamClosed(long fileSize, String filePath) {
                        int trimLength = (int)getCachePath().length();
                        String fileId = filePath.substring(trimLength);
                        mSQLHelper.updateFileSize(fileId, fileSize);
                        deleteLessUsedFiles(fileSize);
                    }
                });

            } catch (Exception e) {
                Log.e(TAG, "Error loading URL " + requestUrl, e);
                return null;
            }
        }
        Log.d("Stored Responses", contentType + " " + encoding + " " + requestUrl.toString());
        return new WebResourceResponse(contentType, encoding, inputStream);
    }
}

{'fraction_non_alphanumeric': 0.09569283456269757, 'fraction_numerical': 0.0029636459430979977, 'mean_word_length': 2.085754927860191, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3836664', 'n_tokens_mistral': 3963, 'n_tokens_neox': 3803, 'n_words': 854}