<commit_before>#!/usr/bin/env python

# Intent bifuz.
#
# Copyright (C) 2015 Intel Corporation
# Author: <NAME> <<EMAIL>>
# Author: <NAME> <<EMAIL>>
#
# Licensed under the MIT license, see COPYING.MIT for details

import os, sys
import re
import pprint
import random, string
import multiprocessing
from common import *


#list with domains used to generate random URIs
domains=[".com",".org",".net",".int",".gov",".mil"]
def generate_random_uri():
    return random.choice(["http","https"])+"://"+str(string_generator(random.randint(10,100)))+random.choice(domains)


def string_generator(size=8, chars=string.ascii_uppercase + string.digits):
      return ''.join(random.choice(chars) for _ in range(size))


def get_default_values():

    global categories
    global extra_keys
    global extra_types
    global activity_actions
    global flags

    with open(path_txt + "categories.txt") as f:
        categories = f.read().splitlines()

    with open(path_txt + "extra_keys.txt") as f:
        extra_keys = f.read().splitlines()

    with open(path_txt + "extra_types.txt") as f:
        extra_types = f.read().splitlines()

    with open(path_txt + "flags.txt") as f:
        flags = f.read().splitlines()
    for i in range(len(flags)):
        index_fl = flags[i].index(':')
        if index_fl > 0:
            flags[i] = flags[i][index_fl+1:]

    with open(path_txt + "activity_actions.txt") as f:
        activity_actions = f.read().splitlines()

    return True


def parse_logcat(ip, log_filename):
    root_index = log_filename.rfind('/')
    root_path = log_filename[:root_index]
    created_logfiles = {}
    with open(log_filename, 'r') as logcat:
        error = False
        package_name = ''
        new_name = ''
        for line in logcat:
            if line.startswith('F/BIFUZ'):
                m = re.search("F/BIFUZ_.* -n .*\.(.*)/.*\.(.*) -f", line)
                try:
                    package_name = m.group(1)
                    activity = m.group(2).strip()
                except:
                    pass
            elif "Caused" in line:
                error = True
                m = re.search(".* Caused by\: (.*)\:.*", line)
                try:
                    new_name = m.group(1)
                except:
                    pass

    if not error:
        os.remove(log_filename)
        return True
    if package_name and new_name:
        new_filename = re.sub('\W+','',new_filename)
        new_filename = root_path + "/e_" + activity + "." + new_name
        if new_filename in created_logfiles.keys():
            created_logfiles[new_filename] += 1
        else:
            created_logfiles[new_filename] = 1
        os.rename(log_filename, new_filename + "_" + str(created_logfiles[new_filename]) + ".txt")
    return True


def populate_activity():
    get_default_values()
    for k_pk in activity_map.keys():
        for actv in activity_map[k_pk].keys():
            if len(activity_map[k_pk].get(actv, {})) == 0:
                activity_map[k_pk][actv] = dict([(x, categories) for x in activity_actions])
            else:
                for intent in activity_map[k_pk][actv].keys():
                    if len(activity_map[k_pk][actv][intent]) == 0:
                        activity_map[k_pk][actv][intent] = categories
    return True


def get_info(data, line):
    actv_index = data.find("Activity Resolver Table")
    data_actv = data[actv_index:]

    data_actv = data_actv.split("\r\n")
    activity_end = 0

    for i in range(len(data_actv)-1):
        if data_actv[i] == '' and data_actv[i+1] == data_actv[i+1].lstrip():
            activity_end = i
            data_actv = data_actv[:i]
            break

    activity_map[line] = {}

    # get activites
    parse_dumpsys(data_actv, line)
    #pp.pprint(activity_map)

    return True


def parse_dumpsys(data, line):

    act_ctg = {}
    part_line = ''
    for i in range(1, len(data)):
        pk_re = re.search("\d+\w+\s("+line+"\S*)", data[i])
        try:
            part_line = pk_re.group(1).strip()

            part_line = part_line.replace('/.', '.')
            index_sl = part_line.find('/')
            if index_sl != -1:
                part_line = part_line[index_sl+1:]

            if part_line not in activity_map[line].keys():
                act_ctg = {}
        except:
            act_re = re.search("Action: \"(.*)\".*", data[i])
            try:
                act = act_re.group(1).strip()
                if not act in act_ctg.keys():
                    act_ctg[act] = []
            except:
                catg_re = re.search("Category: \"(.*)\".*", data[i])
                try:
                    catg = catg_re.group(1).strip()
                    if not catg in act_ctg[act]:
                        act_ctg[act].append(catg)
                except:
                    continue

        if part_line in activity_map[line].keys():
            activity_map[line][part_line] = act_ctg
        else:
            activity_map[line].update({part_line : act_ctg})
    populate_activity()
    return True


def collect_info(ip, log_dir, selected_packages):

    lines = get_package_list(ip, log_dir, selected_packages)
    if not linesu:
        log_in_logcat(ip, 'BIFUZ_INTENT no valid packages. STOP!')
        sys.exit(1)

    global activity_map
    activity_map ={}

    for line in lines:
        line = line.replace("package:", "")
        cmnd = "shell dumpsys package %s > %s/package_%s.txt"%(line, log_dir, line)
        print cmnd

        run_resp = run_inadb(ip, cmnd)<commit_msg>fixed small typo at line 174 linesu -> lines<commit_after>#!/usr/bin/env python

# Intent bifuz.
#
# Copyright (C) 2015 Intel Corporation
# Author: <NAME> <<EMAIL>>
# Author: <NAME> <<EMAIL>>
#
# Licensed under the MIT license, see COPYING.MIT for details

import os, sys
import re
import pprint
import random, string
import multiprocessing
from common import *


#list with domains used to generate random URIs
domains=[".com",".org",".net",".int",".gov",".mil"]
def generate_random_uri():
    return random.choice(["http","https"])+"://"+str(string_generator(random.randint(10,100)))+random.choice(domains)


def string_generator(size=8, chars=string.ascii_uppercase + string.digits):
      return ''.join(random.choice(chars) for _ in range(size))


def get_default_values():

    global categories
    global extra_keys
    global extra_types
    global activity_actions
    global flags

    with open(path_txt + "categories.txt") as f:
        categories = f.read().splitlines()

    with open(path_txt + "extra_keys.txt") as f:
        extra_keys = f.read().splitlines()

    with open(path_txt + "extra_types.txt") as f:
        extra_types = f.read().splitlines()

    with open(path_txt + "flags.txt") as f:
        flags = f.read().splitlines()
    for i in range(len(flags)):
        index_fl = flags[i].index(':')
        if index_fl > 0:
            flags[i] = flags[i][index_fl+1:]

    with open(path_txt + "activity_actions.txt") as f:
        activity_actions = f.read().splitlines()

    return True


def parse_logcat(ip, log_filename):
    root_index = log_filename.rfind('/')
    root_path = log_filename[:root_index]
    created_logfiles = {}
    with open(log_filename, 'r') as logcat:
        error = False
        package_name = ''
        new_name = ''
        for line in logcat:
            if line.startswith('F/BIFUZ'):
                m = re.search("F/BIFUZ_.* -n .*\.(.*)/.*\.(.*) -f", line)
                try:
                    package_name = m.group(1)
                    activity = m.group(2).strip()
                except:
                    pass
            elif "Caused" in line:
                error = True
                m = re.search(".* Caused by\: (.*)\:.*", line)
                try:
                    new_name = m.group(1)
                except:
                    pass

    if not error:
        os.remove(log_filename)
        return True
    if package_name and new_name:
        new_filename = re.sub('\W+','',new_filename)
        new_filename = root_path + "/e_" + activity + "." + new_name
        if new_filename in created_logfiles.keys():
            created_logfiles[new_filename] += 1
        else:
            created_logfiles[new_filename] = 1
        os.rename(log_filename, new_filename + "_" + str(created_logfiles[new_filename]) + ".txt")
    return True


def populate_activity():
    get_default_values()
    for k_pk in activity_map.keys():
        for actv in activity_map[k_pk].keys():
            if len(activity_map[k_pk].get(actv, {})) == 0:
                activity_map[k_pk][actv] = dict([(x, categories) for x in activity_actions])
            else:
                for intent in activity_map[k_pk][actv].keys():
                    if len(activity_map[k_pk][actv][intent]) == 0:
                        activity_map[k_pk][actv][intent] = categories
    return True


def get_info(data, line):
    actv_index = data.find("Activity Resolver Table")
    data_actv = data[actv_index:]

    data_actv = data_actv.split("\r\n")
    activity_end = 0

    for i in range(len(data_actv)-1):
        if data_actv[i] == '' and data_actv[i+1] == data_actv[i+1].lstrip():
            activity_end = i
            data_actv = data_actv[:i]
            break

    activity_map[line] = {}

    # get activites
    parse_dumpsys(data_actv, line)
    #pp.pprint(activity_map)

    return True


def parse_dumpsys(data, line):

    act_ctg = {}
    part_line = ''
    for i in range(1, len(data)):
        pk_re = re.search("\d+\w+\s("+line+"\S*)", data[i])
        try:
            part_line = pk_re.group(1).strip()

            part_line = part_line.replace('/.', '.')
            index_sl = part_line.find('/')
            if index_sl != -1:
                part_line = part_line[index_sl+1:]

            if part_line not in activity_map[line].keys():
                act_ctg = {}
        except:
            act_re = re.search("Action: \"(.*)\".*", data[i])
            try:
                act = act_re.group(1).strip()
                if not act in act_ctg.keys():
                    act_ctg[act] = []
            except:
                catg_re = re.search("Category: \"(.*)\".*", data[i])
                try:
                    catg = catg_re.group(1).strip()
                    if not catg in act_ctg[act]:
                        act_ctg[act].append(catg)
                except:
                    continue

        if part_line in activity_map[line].keys():
            activity_map[line][part_line] = act_ctg
        else:
            activity_map[line].update({part_line : act_ctg})
    populate_activity()
    return True


def collect_info(ip, log_dir, selected_packages):

    lines = get_package_list(ip, log_dir, selected_packages)
    if not lines:
        log_in_logcat(ip, 'BIFUZ_INTENT no valid packages. STOP!')
        sys.exit(1)

    global activity_map
    activity_map ={}

    for line in lines:
        line = line.replace("package:", "")
        cmnd = "shell dumpsys package %s > %s/package_%s.txt"%(line, log_dir, line)
        print cmnd

        run_resp = run_inadb(ip, cmnd)
{'fraction_non_alphanumeric': 0.13381165919282512, 'fraction_numerical': 0.005650224215246637, 'mean_word_length': 2.0685195376995047, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3254801', 'n_tokens_mistral': 3701, 'n_tokens_neox': 3520, 'n_words': 772}