<jupyter_start><jupyter_text>Import issues<jupyter_code>issues = pd.read_csv(f"{csv_folder}/issues.csv", index_col=0)
for date_field in ["created", "duedate", "lastViewed", "resolutiondate", "updated"]:
    issues[date_field] = pd.to_datetime(issues[date_field], errors="coerce")
issues = issues[issues.created > '2015-03-17 15:04:32+0000']
issues.head()<jupyter_output>/Library/Python/3.8/site-packages/IPython/core/interactiveshell.py:3145: DtypeWarning: Columns (4,6,9,12,13,17,19,22,41,42,43,60,61,82,84,89) have mixed types.Specify dtype option on import or set low_memory=False.
  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,<jupyter_text>Import fixversions<jupyter_code>fixversions = pd.read_csv(f"{csv_folder}/issues_fixversions.csv", index_col=0)
fixversions.head(5)<jupyter_output><empty_output><jupyter_text>Import versions<jupyter_code>versions = pd.read_csv(f"{csv_folder}/issues_versions.csv", index_col=0)
versions.head(5)<jupyter_output><empty_output><jupyter_text>Import tags<jupyter_code>tags = pd.read_csv(f"{csv_folder}/tags.csv", index_col=0)
tags.Date = pd.to_datetime(tags.Date)
tags["month"] = tags.Date.dt.month
tags["year"] = tags.Date.dt.year
tags<jupyter_output><empty_output><jupyter_text>Szz Unleashed - Bug issues with Affected Version* python3 fetch_jira_bugs/fetch.py --issue-code SONAR --jira-project jira.sonarsource.com* python3 fetch_jira_bugs/git_log_to_array.py --repo-path ../sonarqube --from-commit b326bfd875b0b41* python3 fetch_jira_bugs/find_bug_fixes.py --gitlog gitlog.json --issue-list issue/ --gitlog-pattern "SONAR-{nbr}"* java -jar build/libs/szz_find_bug_introducers-0.1.jar -i ../issue_list.json -r ../../sonarqube/<jupyter_code>working_folder = "compare_parameters_unleashed/smaller-subset-test"
szz_files = glob.glob(f"{szz_folder}/{working_folder}/issues/*.json")
szz_files<jupyter_output><empty_output><jupyter_text>Bug fixing commits<jupyter_code>data = {}
for szz_file in szz_files:
    with open(szz_file, "r") as f:
        data.update(json.load(f))
fields =[ 'creationdate',
 'resolutiondate',
 'commitdate',
 'hash']
tuples = []
for key in data.keys():
    inner_tuple = []
    inner_tuple.append(key)
    for field in fields:
        inner_tuple.append(data[key][field])
    tuples.append(tuple(inner_tuple))
issues_fixing_commit = pd.DataFrame(tuples, columns=["issue_name"]+fields)
issues_fixing_commit<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>def load_fix_inducers(folder):
    szz_inducing_folders = list(filter(lambda path: True if os.path.isdir(path) else False, glob.glob(f"{szz_folder}/{working_folder}/{folder}/results/*")))
    szz_inducing_files = list(filter(lambda path: True if (os.path.isfile(path) and ("fix_and_introducers_pairs" in path)) else False, glob.glob(f"{szz_folder}/{working_folder}/{folder}/results/*")))
    fix_and_introducers_pairs = []

    for file in szz_inducing_files:
        with open(file, "r") as f:
            fix_and_introducers_pairs.append(json.load(f))

    for folder in szz_inducing_folders:
        with open(f"{folder}/fix_and_introducers_pairs.json", "r") as f:
            fix_and_introducers_pairs.append(json.load(f))
            
    fix_and_introducers_pairs_tuples = []
    for pair_list in fix_and_introducers_pairs:
        for pair in pair_list:
            fix_and_introducers_pairs_tuples.append((pair[0], pair[1]))
    fix_and_introducers = pd.DataFrame(fix_and_introducers_pairs_tuples, columns=["fixing_commit", "inducing_commit"])

    return fix_and_introducers<jupyter_output><empty_output><jupyter_text>Load simple depth 3<jupyter_code>depth_3 = load_fix_inducers("simple_depth_3")
depth_3 = depth_3.drop_duplicates(subset=["fixing_commit", "inducing_commit"], keep="first")<jupyter_output><empty_output><jupyter_text>Load simple depth 5<jupyter_code>depth_5 = load_fix_inducers("simple_depth_5")
depth_5 = depth_5.drop_duplicates(subset=["fixing_commit", "inducing_commit"], keep="first")<jupyter_output><empty_output><jupyter_text>Retrieve tags for commits<jupyter_code>def find_first_tag_contains_commit(commit):
    tags = git.tag("--contains", commit).split("\n")
    if len(tags) > 0:
        return tags[0]
    return None

def all_tag_contains_commit(commit):
    tags = git.tag("--contains", commit).split("\n")
    return tags

def find_all_tags_for_df(df, newColumn, func, field):
    cache = {}
    df[newColumn] = np.nan
    for i, row in df.iterrows():
        if(not row[field] in cache):
            cache[row[field]] = func(row[field])
        else:
            row[newColumn] = cache[row[field]]
        if(i%100 == 0):
            print(i)
            print(cache)
from pandarallel import pandarallel
pandarallel.initialize()

dataframes = [depth_3, depth_5]

#for df in dataframes:
    #df["all_affected_tags_fixing"] = df.fixing_commit.parallel_apply(lambda sha: all_tag_contains_commit(sha))
    #df["all_affected_tags_inducing"] = df.inducing_commit.apply(lambda sha: all_tag_contains_commit(sha))<jupyter_output>INFO: Pandarallel will run on 8 workers.
INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.<jupyter_text>depth_3.to_csv(f"{csv_folder}/fix_and_introducers_simple_depth_3.csv")depth_5.to_csv(f"{csv_folder}/fix_and_introducers_simple_depth_5.csv")  Reloading from csv<jupyter_code>def load_df(name):
    df = pd.read_csv(f"{csv_folder}/{name}.csv", index_col=0)
    df['all_affected_tags_fixing'] = df['all_affected_tags_fixing'].apply(lambda x: literal_eval(x))
    df['all_affected_tags_inducing'] = df['all_affected_tags_inducing'].apply(lambda x: literal_eval(x))
    df["tags_affected_only_inducing"] = df.apply(\
                                                                               lambda row: list(set(row.all_affected_tags_inducing)\
                                                                               .difference(set(row.all_affected_tags_fixing))),axis=1)
    return df
depth_3 = load_df("fix_and_introducers_simple_depth_3")                        
depth_5 = load_df("fix_and_introducers_simple_depth_5")<jupyter_output><empty_output><jupyter_text>Analysis  Merging all datasets<jupyter_code>versions = versions.merge(issues[["issue_key", "issue_id"]])
versions
versions_list = versions.groupby(["issue_id", "issue_key"]).version_name.apply(list).reset_index()
versions_list = versions_list.rename(columns={"version_name":"versions"})
versions_list
def merge_with_versions(df):
    merge = versions_list.merge(issues_fixing_commit, left_on="issue_key", right_on="issue_name")
    inducing_and_versions = df[df.index.isin(df[["fixing_commit", "inducing_commit"]].drop_duplicates().index)]
    merge = merge.merge(inducing_and_versions[["fixing_commit", "inducing_commit", "tags_affected_only_inducing", "all_affected_tags_inducing"]], left_on="hash", right_on="fixing_commit")
    return merge

depth_3_merged = merge_with_versions(depth_3)
depth_5_merged = merge_with_versions(depth_5)<jupyter_output><empty_output><jupyter_text>**Uniform versions from Jira to Github**<jupyter_code>map_versions = {"8.5.0.37579": "8.5",
               "8.4.0.35506": "8.4",
               "8.3.0.34182": "8.3",
               "8.2.0.32929": "8.2",
               "8.1.0.31237": "8.1",
               "8.4.2.36762": "8.4.2",
               "8.4.1.35646": "8.4.1"}
def replace_versions(tags):
    for i, tag in enumerate(tags):
       if(tag in map_versions):
        tags[i] = map_versions[tag]
    return tags
for df in [depth_3_merged, depth_5_merged]:
    df.all_affected_tags_inducing = df.all_affected_tags_inducing.apply(replace_versions)
    df.tags_affected_only_inducing = df.tags_affected_only_inducing.apply(replace_versions)
def set_intersection(series):
    ret_val = set()
    for i, l in enumerate(series):
        if i == 0:
            ret_val = set(l)
        else:
            ret_val = ret_val.intersection(set(l))
    return list(ret_val)

def set_union(series):
    ret_val = set()
    for i, l in enumerate(series):
        if i == 0:
            ret_val = set(l)
        else:
            ret_val = ret_val.union(set(l))
    return list(ret_val)

def merge_tags(df):
    intersection = df.groupby('issue_id').tags_affected_only_inducing.apply(set_intersection).reset_index()
    union = df.groupby('issue_id').tags_affected_only_inducing.apply(set_union).reset_index()
    x = intersection.merge(union, on="issue_id", suffixes =["_intersection","_union"])
    x = x.rename(columns={"tags_affected_only_inducing_intersection":"tags_intersection",
                      "tags_affected_only_inducing_union": "tags_union"})
    return x

def merge_with_versions(df):
    merge = versions_list.merge(df, on="issue_id")
    return merge
    
depth_3_final = merge_with_versions(merge_tags(depth_3_merged))
depth_5_final = merge_with_versions(merge_tags(depth_5_merged))<jupyter_output><empty_output><jupyter_text>**Number of issues for which SZZ was able to find an inducing commit**<jupyter_code>for df in [depth_3_merged, depth_5_merged]:
    print(df.issue_id.nunique())<jupyter_output>31
31<jupyter_text>**Number of inducing commit per issue**<jupyter_code>for df in [depth_3_merged, depth_5_merged]:
    print(df.groupby("issue_id").inducing_commit.count().value_counts().sort_index())<jupyter_output>1     8
2     4
3     3
4     1
5     2
6     2
7     1
8     3
9     1
11    1
14    2
19    1
24    1
53    1
Name: inducing_commit, dtype: int64
1     9
2     4
3     3
4     1
5     5
6     1
8     4
10    2
11    1
13    1
Name: inducing_commit, dtype: int64<jupyter_text>**How many issues have their version among the predicted ones by SZZ?**<jupyter_code>for df in [depth_3_final, depth_5_final]:
    df["intersection_versions_tags_intersection"] = df.apply(lambda row: len(set(row.versions).intersection(row.tags_intersection)), axis=1)
    df["intersection_versions_tags_union"] = df.apply(lambda row: len(set(row.versions).intersection(row.tags_union)), axis=1)
for df in [depth_3_final, depth_5_final]:
    print(df.intersection_versions_tags_intersection.value_counts(normalize=True).sort_index())
    print(df.intersection_versions_tags_union.value_counts(normalize=True).sort_index())<jupyter_output>0    0.225806
1    0.677419
2    0.064516
3    0.032258
Name: intersection_versions_tags_intersection, dtype: float64
0    0.064516
1    0.774194
2    0.129032
3    0.032258
Name: intersection_versions_tags_union, dtype: float64
0    0.193548
1    0.709677
2    0.064516
3    0.032258
Name: intersection_versions_tags_intersection, dtype: float64
0    0.064516
1    0.774194
2    0.129032
3    0.032258
Name: intersection_versions_tags_union, dtype: float64<jupyter_text>**How many issue have all the affected Versions in the SZZ versions?**<jupyter_code>for df in [depth_3_final, depth_5_final]:
    df["full_intersection_intersection"] = df.apply(lambda row: row.intersection_versions_tags_intersection == len(row.versions), axis=1)
    df["full_intersection_union"] = df.apply(lambda row: row.intersection_versions_tags_union == len(row.versions), axis=1)
for df in [depth_3_final, depth_5_final]:
    print(df.full_intersection_intersection.value_counts(normalize=True).sort_index())
    print(df.full_intersection_union.value_counts(normalize=True).sort_index())<jupyter_output>False    0.258065
True     0.741935
Name: full_intersection_intersection, dtype: float64
False    0.064516
True     0.935484
Name: full_intersection_union, dtype: float64
False    0.225806
True     0.774194
Name: full_intersection_intersection, dtype: float64
False    0.064516
True     0.935484
Name: full_intersection_union, dtype: float64<jupyter_text>**What is the ratio between the intersection and the number of reported versions on Jira?**<jupyter_code>for df in [depth_3_final, depth_5_final]:
    df["intersection_ratio_intersection"] = df.apply(lambda row: row.intersection_versions_tags_intersection / len(row.versions), axis=1)
    df["intersection_ratio_union"] = df.apply(lambda row: row.intersection_versions_tags_union / len(row.versions), axis=1)
for df in [depth_3_final, depth_5_final]:
    print(df.intersection_ratio_intersection.value_counts(normalize=True).sort_index())
    print(df.intersection_ratio_union.value_counts(normalize=True).sort_index())<jupyter_output>0.0    0.225806
0.5    0.032258
1.0    0.741935
Name: intersection_ratio_intersection, dtype: float64
0.0    0.064516
1.0    0.935484
Name: intersection_ratio_union, dtype: float64
0.0    0.193548
0.5    0.032258
1.0    0.774194
Name: intersection_ratio_intersection, dtype: float64
0.0    0.064516
1.0    0.935484
Name: intersection_ratio_union, dtype: float64<jupyter_text>**Number of SZZ tags compared to Jira ones**<jupyter_code>for df in [depth_3_final, depth_5_final]:
    df["szz_size_jira_ratio_intersection"] = df.apply(lambda row: len(row.tags_intersection) / len(row.versions), axis=1)
    df["szz_size_jira_ratio_union"] = df.apply(lambda row: len(row.tags_union) / len(row.versions), axis=1)
for df in [depth_3_final, depth_5_final]:
    print(f"Intersection median: {df.szz_size_jira_ratio_intersection.median()}")
    print(f"Intersection mean: {df.szz_size_jira_ratio_intersection.mean()}")
    print(f"Union median: {df.szz_size_jira_ratio_union.median()}")
    print(f"Union mean: {df.szz_size_jira_ratio_union.mean()}")
fig, axes =plt.subplots(4,1, sharex=True, figsize=(20, 10))
i = 0
for df in [depth_3_final, depth_5_final]:
    sns.boxplot(x=df.szz_size_jira_ratio_intersection,ax=axes[i])
    i+=1
    sns.boxplot(x=df.szz_size_jira_ratio_union, ax=axes[i])
    i+=1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12219451371571072, 'fraction_numerical': 0.04320082147572246, 'mean_word_length': 4.844406343763395, 'pattern_counts': {'":': 10, '<': 70, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 7, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '606187', 'license': 'MIT', 'n_tokens_mistral': 5231, 'n_tokens_neox': 4889, 'n_words': 821, 'path': 'notebooks/analysis/Szz_analysis_inducing_commits_comparison.ipynb', 'repo_name': 'IncioMan/ci_data_issue_prediction', 'size': '13634'}