<commit_before>package com.tavultesoft.kmea.cloud;

import android.content.Context;
import android.os.Build;
import android.util.Log;

import com.tavultesoft.kmea.JSONParser;
import com.tavultesoft.kmea.KMKeyboardDownloaderActivity;
import com.tavultesoft.kmea.KMManager;
import com.tavultesoft.kmea.R;
import com.tavultesoft.kmea.cloud.CloudApiTypes;
import com.tavultesoft.kmea.data.Keyboard;
import com.tavultesoft.kmea.data.LexicalModel;
import com.tavultesoft.kmea.util.FileUtils;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class CloudDataJsonUtil {

  private static final String TAG = "CloudDataJsonUtil";
  private CloudDataJsonUtil()
  {
    //no instances
  }

  public static HashMap<String,String> createKeyboardInfoMap(String aPackageId,String aLanguageId, String aLanguageName, String aKeyboardId,
                                                   String aKeyboardName, String aKeyboardVersion, String anIsCustomKeyboard,
                                                   String aFont, String aOskFont)
  {
    HashMap<String, String> keyboardInfo = new HashMap<String, String>();
    if(aPackageId!=null)
      keyboardInfo.put(KMManager.KMKey_PackageID, aPackageId);
    keyboardInfo.put(KMManager.KMKey_KeyboardID, aKeyboardId);
    keyboardInfo.put(KMManager.KMKey_LanguageID, aLanguageId);
    keyboardInfo.put(KMManager.KMKey_KeyboardName, aKeyboardName);
    keyboardInfo.put(KMManager.KMKey_LanguageName, aLanguageName);
    keyboardInfo.put(KMManager.KMKey_KeyboardVersion, aKeyboardVersion);
    keyboardInfo.put(KMManager.KMKey_CustomKeyboard, anIsCustomKeyboard);
    keyboardInfo.put(KMManager.KMKey_Font, aFont);
    if (aOskFont != null)
      keyboardInfo.put(KMManager.KMKey_OskFont, aOskFont);

    return keyboardInfo;
  }

  public static List<Keyboard> processKeyboardJSON(JSONObject query, boolean fromKMP) {
    List<Keyboard> keyboardsList = new ArrayList<>();
    //keyboardModifiedDates = new HashMap<String, String>();

    String isCustom = fromKMP ? "Y" : "N";

    try {
      // Thank you, Cloud API format.
      JSONArray languages = query.getJSONObject(KMKeyboardDownloaderActivity.KMKey_Languages).getJSONArray(KMKeyboardDownloaderActivity.KMKey_Languages);
      for (int i = 0; i < languages.length(); i++) {
        JSONObject language = languages.getJSONObject(i);

        String langID = language.getString(KMManager.KMKey_ID);
        String langName = language.getString(KMManager.KMKey_Name);

        JSONArray langKeyboards = language.getJSONArray(KMKeyboardDownloaderActivity.KMKey_LanguageKeyboards);

        int kbLength = langKeyboards.length();
        for (int j = 0; j < kbLength; j++) {
          JSONObject keyboardJSON = langKeyboards.getJSONObject(j);
          String kbID = keyboardJSON.getString(KMManager.KMKey_ID);
          String kbName = keyboardJSON.getString(KMManager.KMKey_Name);
          String kbVersion = keyboardJSON.optString(KMManager.KMKey_KeyboardVersion, "1.0");
          String kbFont = keyboardJSON.optString(KMManager.KMKey_Font, "");

          //String kbKey = String.format("%s_%s", langID, kbID);
          HashMap<String, String> hashMap = createKeyboardInfoMap(null,langID,langName,kbID,kbName,kbVersion,isCustom,kbFont,null);


//          if (keyboardModifiedDates.get(kbID) == null) {
//            keyboardModifiedDates.put(kbID, keyboardJSON.getString(KMManager.KMKey_KeyboardModified));
//          }

          keyboardsList.add(new Keyboard(hashMap));
        }
      }
    } catch (JSONException | NullPointerException e) {
      Log.e(TAG, "JSONParse Error: " + e);
      return new ArrayList<>();  // Is this ideal?
    }

    return keyboardsList;
  }

  public static List<LexicalModel> processLexicalModelJSON(JSONArray models) {
    List<LexicalModel> modelList = new ArrayList<>(models.length());

    try {
      // Parse each model JSON Object.
      int modelsLength = models.length();
      for (int i = 0; i < modelsLength; i++) {
        JSONObject model = models.getJSONObject(i);
        String packageID = "", modelURL = "";
        if (model.has(KMManager.KMKey_PackageID)) {
          packageID = model.getString(KMManager.KMKey_PackageID);
        } else {
          // Determine package ID from packageFilename
          modelURL = model.optString("packageFilename", "");
          packageID = FileUtils.getFilename(modelURL);
          packageID = packageID.replace(FileUtils.MODELPACKAGE, "");
        }

        // api.keyman.com query returns an array of language IDs Strings while
        // kmp.json "languages" is an array of JSONObject
        String languageID = "", langName = "";
        Object obj = model.getJSONArray("languages");
        if (((JSONArray) obj).get(0) instanceof String) {
          // language name not provided, so re-use language ID
          languageID = model.getJSONArray("languages").getString(0);
          langName = languageID;
        } else if (((JSONArray) obj).get(0) instanceof JSONObject) {
          JSONObject languageObj = model.getJSONArray("languages").getJSONObject(0);
          languageID = languageObj.getString("id");
          langName = languageObj.getString("name");
        }

        String modelID = model.getString(KMManager.KMKey_ID);
        String modelName = model.getString(KMManager.KMKey_Name);
        String modelVersion = model.optString(KMManager.KMKey_Version,"1.0");

        String isCustom = model.optString(KMManager.KMKey_CustomModel, "N");
        
        String icon = "0";

        HashMap<String, String> hashMap = new HashMap<String, String>();
        hashMap.put(KMManager.KMKey_PackageID, packageID);
        hashMap.put(KMManager.KMKey_LanguageID, languageID);
        hashMap.put(KMManager.KMKey_LexicalModelID, modelID);
        hashMap.put(KMManager.KMKey_LexicalModelName, modelName);
        hashMap.put(KMManager.KMKey_LanguageName, langName);
        hashMap.put(KMManager.KMKey_LexicalModelVersion, modelVersion);
        hashMap.put(KMManager.KMKey_CustomModel, isCustom);
        hashMap.put(KMManager.KMKey_LexicalModelPackageFilename, modelURL);
        hashMap.put("isEnabled", "true");
        hashMap.put(KMManager.KMKey_Icon, String.valueOf(R.drawable.ic_arrow_forward));

        modelList.add(new LexicalModel(hashMap));
      }
    } catch (JSONException | NullPointerException e) {
      Log.e(TAG, "JSONParse Error: " + e);
      return new ArrayList<>();  // Is this ideal?
    }

    return modelList;
  }
<commit_msg>fix(android): Fallback to keyboard(package) version<commit_after>package com.tavultesoft.kmea.cloud;

import android.content.Context;
import android.os.Build;
import android.util.Log;

import com.tavultesoft.kmea.JSONParser;
import com.tavultesoft.kmea.KMKeyboardDownloaderActivity;
import com.tavultesoft.kmea.KMManager;
import com.tavultesoft.kmea.R;
import com.tavultesoft.kmea.cloud.CloudApiTypes;
import com.tavultesoft.kmea.data.Keyboard;
import com.tavultesoft.kmea.data.LexicalModel;
import com.tavultesoft.kmea.util.FileUtils;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class CloudDataJsonUtil {

  private static final String TAG = "CloudDataJsonUtil";
  private CloudDataJsonUtil()
  {
    //no instances
  }

  public static HashMap<String,String> createKeyboardInfoMap(String aPackageId,String aLanguageId, String aLanguageName, String aKeyboardId,
                                                   String aKeyboardName, String aKeyboardVersion, String anIsCustomKeyboard,
                                                   String aFont, String aOskFont)
  {
    HashMap<String, String> keyboardInfo = new HashMap<String, String>();
    if(aPackageId!=null)
      keyboardInfo.put(KMManager.KMKey_PackageID, aPackageId);
    keyboardInfo.put(KMManager.KMKey_KeyboardID, aKeyboardId);
    keyboardInfo.put(KMManager.KMKey_LanguageID, aLanguageId);
    keyboardInfo.put(KMManager.KMKey_KeyboardName, aKeyboardName);
    keyboardInfo.put(KMManager.KMKey_LanguageName, aLanguageName);
    keyboardInfo.put(KMManager.KMKey_KeyboardVersion, aKeyboardVersion);
    keyboardInfo.put(KMManager.KMKey_CustomKeyboard, anIsCustomKeyboard);
    keyboardInfo.put(KMManager.KMKey_Font, aFont);
    if (aOskFont != null)
      keyboardInfo.put(KMManager.KMKey_OskFont, aOskFont);

    return keyboardInfo;
  }

  public static List<Keyboard> processKeyboardJSON(JSONObject query, boolean fromKMP) {
    List<Keyboard> keyboardsList = new ArrayList<>();
    //keyboardModifiedDates = new HashMap<String, String>();

    String isCustom = fromKMP ? "Y" : "N";

    try {
      // Thank you, Cloud API format.
      JSONArray languages = query.getJSONObject(KMKeyboardDownloaderActivity.KMKey_Languages).getJSONArray(KMKeyboardDownloaderActivity.KMKey_Languages);
      for (int i = 0; i < languages.length(); i++) {
        JSONObject language = languages.getJSONObject(i);

        String langID = language.getString(KMManager.KMKey_ID);
        String langName = language.getString(KMManager.KMKey_Name);

        JSONArray langKeyboards = language.getJSONArray(KMKeyboardDownloaderActivity.KMKey_LanguageKeyboards);

        int kbLength = langKeyboards.length();
        for (int j = 0; j < kbLength; j++) {
          JSONObject keyboardJSON = langKeyboards.getJSONObject(j);
          String kbID = keyboardJSON.getString(KMManager.KMKey_ID);
          String kbName = keyboardJSON.getString(KMManager.KMKey_Name);
          String kbVersion = keyboardJSON.optString(KMManager.KMKey_KeyboardVersion, "1.0");
          String kbFont = keyboardJSON.optString(KMManager.KMKey_Font, "");

          //String kbKey = String.format("%s_%s", langID, kbID);
          HashMap<String, String> hashMap = createKeyboardInfoMap(null,langID,langName,kbID,kbName,kbVersion,isCustom,kbFont,null);


//          if (keyboardModifiedDates.get(kbID) == null) {
//            keyboardModifiedDates.put(kbID, keyboardJSON.getString(KMManager.KMKey_KeyboardModified));
//          }

          keyboardsList.add(new Keyboard(hashMap));
        }
      }
    } catch (JSONException | NullPointerException e) {
      Log.e(TAG, "JSONParse Error: " + e);
      return new ArrayList<>();  // Is this ideal?
    }

    return keyboardsList;
  }

  public static List<LexicalModel> processLexicalModelJSON(JSONArray models) {
    List<LexicalModel> modelList = new ArrayList<>(models.length());

    try {
      // Parse each model JSON Object.
      int modelsLength = models.length();
      for (int i = 0; i < modelsLength; i++) {
        JSONObject model = models.getJSONObject(i);
        String packageID = "", modelURL = "";
        if (model.has(KMManager.KMKey_PackageID)) {
          packageID = model.getString(KMManager.KMKey_PackageID);
        } else {
          // Determine package ID from packageFilename
          modelURL = model.optString("packageFilename", "");
          packageID = FileUtils.getFilename(modelURL);
          packageID = packageID.replace(FileUtils.MODELPACKAGE, "");
        }

        // api.keyman.com query returns an array of language IDs Strings while
        // kmp.json "languages" is an array of JSONObject
        String languageID = "", langName = "";
        Object obj = model.getJSONArray("languages");
        if (((JSONArray) obj).get(0) instanceof String) {
          // language name not provided, so re-use language ID
          languageID = model.getJSONArray("languages").getString(0);
          langName = languageID;
        } else if (((JSONArray) obj).get(0) instanceof JSONObject) {
          JSONObject languageObj = model.getJSONArray("languages").getJSONObject(0);
          languageID = languageObj.getString("id");
          langName = languageObj.getString("name");
        }

        String modelID = model.getString(KMManager.KMKey_ID);
        String modelName = model.getString(KMManager.KMKey_Name);

        // Cloud data may not contain lexical model version, so fallback to keyboard (package) version
        String keyboardVersion = model.optString(KMManager.KMKey_KeyboardVersion, "1.0");
        String modelVersion = model.getString(KMManager.KMKey_LexicalModelVersion);

        String isCustom = model.optString(KMManager.KMKey_CustomModel, "N");
        
        String icon = "0";

        HashMap<String, String> hashMap = new HashMap<String, String>();
        hashMap.put(KMManager.KMKey_PackageID, packageID);
        hashMap.put(KMManager.KMKey_LanguageID, languageID);
        hashMap.put(KMManager.KMKey_LexicalModelID, modelID);
        hashMap.put(KMManager.KMKey_LexicalModelName, modelName);
        hashMap.put(KMManager.KMKey_LanguageName, langName);
        hashMap.put(KMManager.KMKey_LexicalModelVersion, modelVersion);
        hashMap.put(KMManager.KMKey_CustomModel, isCustom);
        hashMap.put(KMManager.KMKey_LexicalModelPackageFilename, modelURL);
        hashMap.put("isEnabled", "true");
        hashMap.put(KMManager.KMKey_Icon, String.valueOf(R.drawable.ic_arrow_forward));

        modelList.add(new LexicalModel(hashMap));
      }
    } catch (JSONException | NullPointerException e) {
      Log.e(TAG, "JSONParse Error: " + e);
      return new ArrayList<>();  // Is this ideal?
    }

    return modelList;
  }

{'fraction_non_alphanumeric': 0.10595738512827946, 'fraction_numerical': 0.001739382519205682, 'mean_word_length': 3.4426915647134577, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '682597', 'n_tokens_mistral': 4204, 'n_tokens_neox': 4051, 'n_words': 776}