<jupyter_start><jupyter_text>Analytics  Date: 2020/02 SUMMARY:- This notebook represents the project quality analysis of the date exposed right above.   TEAM: Lend.it Semester: 2020/02 Professor: Hilmer Neri Members:| Nome              | Papel           | E-mail                          ||-------------------|-----------------|---------------------------------|| <NAME>    | Tech Lead       |     <EMAIL>   || <NAME>      | Product Manager |    <EMAIL>       || <NAME>       | DevOps          |    <EMAIL>           || <NAME>   | Arquiteto       | <EMAIL> || <NAME>       | Desenvolvedor   |     <EMAIL>  || <NAME>    | Desenvolvedor   |    <EMAIL>   || <NAME>  | Desenvolvedor   | <EMAIL>    || <NAME>    | Desenvolvedor   | <EMAIL>        || <NAME>   | Desenvolvedor   | <EMAIL>     || <NAME>| Desenvolvedor   | <EMAIL> |  LIBRARIES<jupyter_code># Deal with data
import pandas as pd
import numpy as np
import json

# Deal with API request
import urllib3
from urllib3 import request

# Deal with visualization
import seaborn as sns
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>GRAPH SETTINGS<jupyter_code>%config InlineBackend.figure_format ='retina'
sns.set(font_scale=1.5)
sns.set_style('darkgrid',
              {'xtick.bottom' : True,
               'ytick.left': True,
               'grid.linestyle':'--',
               'font.monospace': ['Computer Modern Typewriter'],
               'axes.edgecolor' : 'white'})<jupyter_output><empty_output><jupyter_text>DATAFRAME SETTINGS<jupyter_code>pd.set_option("display.max_rows", None, "display.max_columns", None)<jupyter_output><empty_output><jupyter_text>SonarCloud  KEYS<jupyter_code>request_service = "Lend-it_Request"
rating_service = "Lend-it_Rating"
user_service = "Lend-it_User"
gateway_service = "Lend-it_Gateway"<jupyter_output><empty_output><jupyter_text>METRICS<jupyter_code>metric_list = ['files',
               'functions',
               'complexity',
               'coverage',
               'comment_lines_density',
               'duplicated_lines_density',
               'security_rating',
               'tests',
               'test_success_density',
               'test_execution_time',
               'reliability_rating',
                'ncloc']

len(metric_list)
def generate_metric_string(metric_list):
    metric_str = ''
    
    for metric in metric_list:
        metric_str += metric + ','
        
    return metric_str
metric_str = generate_metric_string(metric_list)<jupyter_output><empty_output><jupyter_text>URLS<jupyter_code>request_url = f"https://sonarcloud.io/api/measures/component_tree?component={request_service}&metricKeys={metric_str}&ps=500"
rating_url = f"https://sonarcloud.io/api/measures/component_tree?component={rating_service}&metricKeys={metric_str}&ps=500"
user_url = f"https://sonarcloud.io/api/measures/component_tree?component={user_service}&metricKeys={metric_str}&ps=500"
gateway_url = f"https://sonarcloud.io/api/measures/component_tree?component={gateway_service}&metricKeys={metric_str}&ps=500"<jupyter_output><empty_output><jupyter_text>JSON options  API REQUEST<jupyter_code>http = urllib3.PoolManager()
request_request = http.request('GET', request_url)
rating_request = http.request('GET', rating_url)
user_request = http.request('GET', user_url)
gateway_request = http.request('GET', gateway_url)<jupyter_output><empty_output><jupyter_text>JSON DECODING<jupyter_code>request_json = json.loads(request_request.data.decode('utf-8'))
rating_json = json.loads(rating_request.data.decode('utf-8'))
user_json = json.loads(user_request.data.decode('utf-8'))
gateway_json = json.loads(gateway_request.data.decode('utf-8'))<jupyter_output><empty_output><jupyter_text>Local files<jupyter_code>release = "Rubik"
date = "25-04-2021"
request_json_path = f"releases/{release}/fga-eps-mds-2020_2-Lend.it-{release}-request-{date}.json"
rating_json_path = f"releases/{release}/fga-eps-mds-2020_2-Lend.it-{release}-rating-{date}.json"
user_json_path = f"releases/{release}/fga-eps-mds-2020_2-Lend.it-{release}-user-{date}.json"
gateway_json_path = f"releases/{release}/fga-eps-mds-2020_2-Lend.it-{release}-gateway-{date}.json"
with open(request_json_path) as f:
    request_local_json = json.load(f)
with open(rating_json_path) as f:
    rating_local_json = json.load(f)
with open(user_json_path) as f:
    user_local_json = json.load(f)
with open(gateway_json_path) as f:
    gateway_local_json = json.load(f)<jupyter_output><empty_output><jupyter_text>DATA  PROJECT<jupyter_code>project_request_json = request_local_json['baseComponent']['measures']
project_rating_json = rating_local_json['baseComponent']['measures']
project_user_json = user_local_json['baseComponent']['measures']
project_gateway_json = gateway_local_json['baseComponent']['measures']
project_request_data = pd.DataFrame(project_request_json)
project_rating_data = pd.DataFrame(project_rating_json)
project_user_data = pd.DataFrame(project_user_json)
project_gateway_data = pd.DataFrame(project_gateway_json)<jupyter_output><empty_output><jupyter_text>Request<jupyter_code>project_request_data<jupyter_output><empty_output><jupyter_text>Rating<jupyter_code>project_rating_data<jupyter_output><empty_output><jupyter_text>User<jupyter_code>project_user_data<jupyter_output><empty_output><jupyter_text>Gateway<jupyter_code>project_gateway_data<jupyter_output><empty_output><jupyter_text>FILES<jupyter_code>def metric_per_file(json):
    file_json = []
    
    for component in json['components']:
        if component['qualifier'] == 'FIL':
            file_json.append(component)
            
    return file_json
request_file_json = metric_per_file(request_local_json)
rating_file_json = metric_per_file(rating_local_json)
user_file_json = metric_per_file(user_local_json)
gateway_file_json = metric_per_file(gateway_local_json)
def generate_file_dataframe(metric_list, json, language_extension):
    df_columns = metric_list
    df = pd.DataFrame(columns = df_columns)
    
    for file in json:
        try:
            if file['language'] == language_extension:
                for measure in file['measures']:
                    df.at[file['path'], measure['metric']] = measure['value']
        except:
            pass
        
    df.reset_index(inplace = True)
    df = df.rename({'index': 'path'}, axis=1).drop(['files'], axis=1)

    return df
request_files_data = generate_file_dataframe(metric_list, request_file_json, language_extension = 'py')
rating_files_data = generate_file_dataframe(metric_list, rating_file_json, language_extension = 'py')
user_files_data = generate_file_dataframe(metric_list, user_file_json, language_extension = 'js')
gateway_files_data = generate_file_dataframe(metric_list, gateway_file_json, language_extension = 'js')<jupyter_output><empty_output><jupyter_text>Request<jupyter_code>request_files_data<jupyter_output><empty_output><jupyter_text>Rating<jupyter_code>rating_files_data<jupyter_output><empty_output><jupyter_text>User<jupyter_code>user_files_data<jupyter_output><empty_output><jupyter_text>Gateway<jupyter_code>gateway_files_data<jupyter_output><empty_output><jupyter_text>ANALYSIS  MAINTAINABILITY  CODE QUALITY  COMPLEXITY<jupyter_code>def m1(df):
    
    density_non_complex_files = round((len(df[(df['complexity'].astype(float)/df['functions'].astype(float)) < 10])/len(df))*100, 2)
    
    return density_non_complex_files<jupyter_output><empty_output><jupyter_text>COMMENTS<jupyter_code>def m2(df):
    
    density_comment_files = round((len(df[(df['comment_lines_density'].astype(float) > 10) & (df['comment_lines_density'].astype(float) < 30)])/len(df))*100, 2)
    
    return density_comment_files<jupyter_output><empty_output><jupyter_text>DUPLICATIONS<jupyter_code>def m3(df):
    
    duplication = round((len(df[(df['duplicated_lines_density'].astype(float) < 5)])/len(df))*100, 2)
    
    return duplication<jupyter_output><empty_output><jupyter_text>BLOCKING CODE  NON-BLOCKING FILES<jupyter_code>def m4(df):
    
    non_blocking_files = round((len(df[(df['security_rating'].astype(float) >= 4)])/len(df))*100,2)
    
    return non_blocking_files<jupyter_output><empty_output><jupyter_text>RELIABILITY  TEST SUCCESS<jupyter_code>def m5(df):
    
    test_success_file = df[['path', 'test_success_density']]
    test_success_repository = df['test_success_density'].astype(float).mean()
    
    print("Project test unit density: ", test_success_repository)
    
    return test_success_file<jupyter_output><empty_output><jupyter_text>FAST TESTS<jupyter_code>def m6(df):
    
    fast_test_df = df[(df['test_execution_time'].astype(float) < 300)]
    fast_test_df['fast_test'] = fast_test_df['test_execution_time']/fast_test_df['tests']
    
    fast_test_file = fast_test_df[['path', 'fast_test']]
    
    fast_test_repository = fast_test_df['fast_test'].astype(float).mean()
    
    print("Project test unit density: ", fast_test_repository)
    
    return fast_test_file<jupyter_output><empty_output><jupyter_text>PRODUCTIVITY  TIMEFRAME: PUT_YOUR_TIMEFRAME_HERE<jupyter_code>NUMBER_OF_ISSUES_RESOLVED = 109
NUMBER_OF_ISSUES = 130

TAGS = {
    "HOTFIX": 3,
    "DOCS": 56,
    "FEATURE": 9,
    "ARQ": 3,
    "DEVOPS": 10,
    "ANALYTICS": 8,
    "US": 28,
    "EASY": 8,
    "MEDIUM": 40,
    "HARD": 6,
    "EPS": 79,
    "MDS": 65
}<jupyter_output><empty_output><jupyter_text>RESOLVED ISSUES' THROUGHPUT<jupyter_code>def m7(number_of_issues_resolved, number_of_issues):
    
    resolved_issues_throughput = round((number_of_issues_resolved / number_of_issues) * 100, 2)
    
    return resolved_issues_throughput<jupyter_output><empty_output><jupyter_text>ISSUE TYPE IN A TIMEFRAME<jupyter_code>def density(issue, number_of_issues):
    issue_density = round((issue / number_of_issues) * 100, 2)
    return issue_density
def m8(tag_dict, number_of_issues):
    
    issue_densities = {
        "hotfix": [density(tag_dict["HOTFIX"], number_of_issues)],
        "docs": [density(tag_dict["DOCS"], number_of_issues)],
        "feature": [density(tag_dict["FEATURE"], number_of_issues)],
        "arq": [density(tag_dict["ARQ"], number_of_issues)],
        "devops": [density(tag_dict["DEVOPS"], number_of_issues)],
        "analytics": [density(tag_dict["ANALYTICS"], number_of_issues)],
        "us": [density(tag_dict["US"], number_of_issues)],
        "easy": [density(tag_dict["EASY"], number_of_issues)],
        "medium": [density(tag_dict["MEDIUM"], number_of_issues)],
        "hard": [density(tag_dict["HARD"], number_of_issues)],
        "eps": [density(tag_dict["EPS"], number_of_issues)],
        "mds": [density(tag_dict["MDS"], number_of_issues)]
    }

    issue_densities = pd.DataFrame.from_dict(issue_densities).T.reset_index()
    
    issue_densities.columns = ['density' ,'percentage']
    
    return issue_densities<jupyter_output><empty_output><jupyter_text>BUGS RATIO<jupyter_code>def m9(tag_dict, number_of_issues):
    
    bugs_ratio = round(((tag_dict["DOCS"] + tag_dict["FEATURE"] + tag_dict["ARQ"] + tag_dict["DEVOPS"] + tag_dict["ANALYTICS"]) / number_of_issues) * 100, 2)
    
    return bugs_ratio<jupyter_output><empty_output><jupyter_text>METRIC RESULTS  Request<jupyter_code>m1(request_files_data)
m2(request_files_data)
m3(request_files_data)
m4(request_files_data)
m5(request_files_data)
m6(request_files_data)<jupyter_output>Project test unit density:  nan<jupyter_text>Rating<jupyter_code>m1(rating_files_data)
m2(rating_files_data)
m3(rating_files_data)
m4(rating_files_data)
m5(rating_files_data)
m6(rating_files_data)<jupyter_output>Project test unit density:  nan<jupyter_text>User<jupyter_code>m1(user_files_data)
m2(user_files_data)
m3(user_files_data)
m4(user_files_data)
m5(user_files_data)
m6(user_files_data)<jupyter_output>Project test unit density:  nan<jupyter_text>Gateway<jupyter_code>m1(gateway_files_data)
m2(gateway_files_data)
m3(gateway_files_data)
m4(gateway_files_data)
m5(gateway_files_data)
m6(gateway_files_data)<jupyter_output>Project test unit density:  nan<jupyter_text>Issues related Metrics<jupyter_code>m7(NUMBER_OF_ISSUES_RESOLVED, NUMBER_OF_ISSUES)
m8(TAGS, NUMBER_OF_ISSUES)
m9(TAGS, NUMBER_OF_ISSUES)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1318032786885246, 'fraction_numerical': 0.013278688524590163, 'mean_word_length': 5.3513794898490366, 'pattern_counts': {'":': 24, '<': 157, '<?xml version=': 0, '>': 155, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '34', 'dir': 'jupyter-structured-clean-dedup', 'id': '630632', 'license': 'MIT', 'n_tokens_mistral': 4545, 'n_tokens_neox': 4344, 'n_words': 538, 'path': 'analytics.ipynb', 'repo_name': 'parlamentaqui/Analytics', 'size': '12468'}