<jupyter_start><jupyter_text>Import Dataset<jupyter_code>df_api = pd.read_csv("../db/cdv/cordova_API.csv")
df_plugin_declaration = pd.read_csv("../db/cdv/cordova_PLUGIN_DECLARATION.csv")
df_plugin_permission_declaration = pd.read_csv("../db/cdv/cordova_PLUGIN_PERMISSION_DECLARATION.csv")
# df_feature = pd.read_csv("../db/fcordova/eatures.csv")

df_api.columns
l_api = list(df_api.columns)
l_api
df_api
df_api = df_api.dropna()
df_api = df_api.reset_index(drop=True)
df_api
df_plugin_declaration
df_plugin_declaration = df_plugin_declaration.dropna()
df_plugin_declaration = df_plugin_declaration.reset_index(drop=True)
df_plugin_declaration
df_plugin_permission_declaration
df_plugin_permission_declaration = df_plugin_permission_declaration.dropna()
df_plugin_permission_declaration = df_plugin_permission_declaration.reset_index(drop=True)
df_plugin_permission_declaration<jupyter_output><empty_output><jupyter_text>Analyse API calls  The occurances  The occurances of funcitons detected for each plugin in each APK<jupyter_code>df_plugins_only = df_api.drop(columns=["apk_name"])
df_plugins_only<jupyter_output><empty_output><jupyter_text>The occurance of plugins for entire dataset<jupyter_code>total_apk = df_plugins_only.shape[0]
print(f"Total APKs: {total_apk}")
df_cnt = df_plugins_only.astype(bool).sum(axis=0).sort_values(ascending=True)
df_cnt
# percentage of apks using each plugin
df_pct = df_cnt.apply(lambda x: round(x/total_apk*100, 2))
df_pct
plt.figure(figsize=(14, 8))
sns.set(font_scale=1.5) # font size 2
sns_pct = sns.barplot(x=df_pct.values, y=df_pct.index)
# sns_pct.set_xticklabels(sns_pct.get_xticklabels(), rotation=45, horizontalalignment='right')
sns_pct.set_xticks(range(0, 101, 10))
plt.xlabel("Plugin Usage")
plt.ylabel("Plugin")
for p in sns_pct.patches:
#     print(p)
    sns_pct.annotate(
        "{:.1%}".format(p.get_width()/100), 
        (p.get_width(), p.get_y() + p.get_height()),
        fontsize=15, 
        color='black', 
        xytext=(2, 5),
        textcoords='offset points')

plt.show()<jupyter_output><empty_output><jupyter_text>Plugins declaration<jupyter_code>df_plugin_declaration_only = df_plugin_declaration.drop(columns=["apk_name", "config.xml", "plugins.xml"])
df_plugin_dc = df_plugin_declaration_only
total_apk = df_plugin_dc.shape[0]
print(f"Total APKs: {total_apk}")
df_plugin_dc_cnt = df_plugin_dc.astype(bool).sum(axis=0).sort_values(ascending=True)
df_plugin_dc_cnt
# percentage of apks using each plugin
df_plugin_dc_pct = df_plugin_dc_cnt.apply(lambda x: round(x/total_apk*100, 2))
df_plugin_dc_pct
plt.figure(figsize=(14, 8))
sns.set(font_scale=1.5) # font size 2
sns_pct = sns.barplot(x=df_plugin_dc_pct.values, y=df_plugin_dc_pct.index)
# sns_pct.set_xticklabels(sns_pct.get_xticklabels(), rotation=45, horizontalalignment='right')
sns_pct.set_xticks(range(0, 101, 10))
plt.xlabel("Plugin Declration")
plt.ylabel("Plugin")
for p in sns_pct.patches:
#     print(p)
    sns_pct.annotate(
        "{:.1%}".format(p.get_width()/100), 
        (p.get_width(), p.get_y() + p.get_height()),
        fontsize=15, 
        color='black', 
        xytext=(2, 5),
        textcoords='offset points')

plt.show()<jupyter_output><empty_output><jupyter_text>Plugin Uages VS. Plugin Dclaration<jupyter_code>df_usage_cnt = df_plugins_only.astype(bool).sum(axis=0)
df_usage_cnt = df_usage_cnt.apply(lambda x: round(x/total_apk*100, 2))
df_declaration_cnt = df_plugin_dc.astype(bool).sum(axis=0)
df_declaration_cnt = df_declaration_cnt.apply(lambda x: round(x/total_apk*100, 2))
d_plugin = {"plugin": list(df_usage_cnt.axes)}
d_usage = {"usage": list(df_usage_cnt.values)}
d_declaration = {"usage": list(df_declaration_cnt.values)}
d_plugin_vs = {
    "plugin": list(df_plugins_only.columns),
    "usage": list(df_usage_cnt.values),
    "declaration": list(df_declaration_cnt.values)
}
# print(d_plugin_vs)
df_plugin_vs = pd.DataFrame(data=d_plugin_vs)
df_plugin_vs
# order by value in declaration
df_plugin_vs_order = df_plugin_vs.sort_values(by=['declaration'])
df_plugin_vs_order
# ax = sns.barplot(x="Percentage", y="Plugin", hue="plugin", data=df_plugin_vs_order)
fig, ax1 = plt.subplots(figsize=(18, 12))
tidy = df_plugin_vs_order.melt(id_vars='plugin').rename(columns=str.title)
# print(tidy)
sns_pct = sns.barplot(x='Value', y='Plugin', hue='Variable', data=tidy, ax=ax1)
# sns.despine(fig)
for p in sns_pct.patches:
#     print(p)
    sns_pct.annotate(
        "{:.1%}".format(p.get_width()/100), 
        (p.get_width(), p.get_y() + p.get_height()),
        fontsize=15, 
        color='black', 
        xytext=(2, 5),
        textcoords='offset points')
plt.xlabel("")
plt.ylabel("")
plt.title(f'Plugin Usage VS. Plugin Declaration for {total_apk} APKs')
plt.show()<jupyter_output><empty_output><jupyter_text>XML analysis<jupyter_code># count APKs missing plugins or config
df_xml = df_plugin_declaration[["config.xml", "plugins.xml"]]
total_apk = df_xml.shape[0]
print(f"Total APKs: {total_apk}")
df_xml_cnt = df_xml.astype(bool).sum(axis=0).sort_values(ascending=True)
df_xml_cnt

df_xml["config+plugins"] = df_xml["config.xml"] + df_xml["plugins.xml"]
df_xml
xlm_values = Counter(df_xml['config+plugins'])
labels = ["Both files", "One file", "None"]
sizes = [xlm_values[2], xlm_values[1], xlm_values[0]]
print(sizes)
# print(sizes) # adds up to 1433, which is the total number of participants
fig1, ax1 = plt.subplots(figsize=(15, 10))
ax1.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=True)
# ax1.axis('equal')
plt.title(f"Cordova APK contains 'config.xml' and 'plugins.xml' for {df_xml.shape[0]} APKs")
plt.show()<jupyter_output>[55, 2518, 86]<jupyter_text>Delete APK has none<jupyter_code>def output_xml_csv(filename, df_output):
    df_output_new = deepcopy(df_output)
    df_output_new["config+plugins"] = df_xml["config+plugins"]
    df_output_new = df_output_new[df_output_new["config+plugins"]>0]
    df_output_new = df_output_new.drop(columns=["config+plugins"])
    df_output_new.to_csv(f"../db/cdv/{filename}.csv", index=False)

output_xml_csv("cordova_API_xml", df_api)
df_plugin_declaration = df_plugin_declaration.drop(columns=["config.xml", "plugins.xml"])
output_xml_csv("cordova_PLUGIN_DECLARATION_xml", df_plugin_declaration)
output_xml_csv("cordova_PLUGIN_PERMISSION_DECLARATION_xml", df_plugin_permission_declaration)<jupyter_output><empty_output><jupyter_text>Heatmap  Heatmap for Entire database<jupyter_code>df_plugins_only_T = df_plugins_only.T    # transpose
plt.figure(figsize=(20,10))
sns.set(font_scale=2) # font size 2
ax = sns.heatmap(df_plugins_only_T)<jupyter_output><empty_output><jupyter_text>Heatmap for a small set of dataset<jupyter_code># select a set of apks, originial
set_num = 20
plt.figure(figsize=(20,10))
sns.set(font_scale=2) # font size 2
ax = sns.heatmap(df_plugins_only_T.iloc[:, :set_num])

# select a set of apks, heatmap with annotation
plt.figure(figsize=(20,10))
sns.set(font_scale=2) # font size 2
ax = sns.heatmap(df_plugins_only_T.iloc[:, :set_num], annot=True)<jupyter_output><empty_output><jupyter_text>Heatmap without media and device<jupyter_code>df_plugins_media = df_api.drop(columns=["apk_name", "media"])
df_plugins_media_T = df_plugins_media.T
# select a set of apks, heatmap with annotation
plt.figure(figsize=(20,10))
sns.set(font_scale=2) # font size 2
ax = sns.heatmap(df_plugins_media_T.iloc[:, :set_num], annot=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14115738236884803, 'fraction_numerical': 0.018253109789075177, 'mean_word_length': 7.07532751091703, 'pattern_counts': {'":': 6, '<': 40, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 29}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '636588', 'license': 'Apache-2.0', 'n_tokens_mistral': 3042, 'n_tokens_neox': 2895, 'n_words': 429, 'path': 'apk_analysis/data_analysis/cdv_plugins_declaration.ipynb', 'repo_name': 'AllenSun7/Android-Security-Analysis', 'size': '7396'}