<jupyter_start><jupyter_text>Import df<jupyter_code>jobsDf = pd.read_csv(f"{csv_folder}/allJobs.csv", index_col=0)
tagsDf = pd.read_csv(f"{csv_folder}/tags_updated.csv", index_col=0)
tagsDf.Date = pd.to_datetime(tagsDf.Date)
tagsDf<jupyter_output><empty_output><jupyter_text>**Use release date as pairing method**<jupyter_code>tagsDf.sort_values(by="Date")<jupyter_output><empty_output><jupyter_text>**Use release name as pairing method**<jupyter_code>tagsDf.sort_values(by="Tag")
tagsDf_travis = tagsDf[tagsDf.Date > pd.to_datetime(datetime.date(2015,3,10))]<jupyter_output><empty_output><jupyter_text>Import comparisons (no RC and Milestones)<jupyter_code>comp_comb_tags_df = pd.read_csv(f"{csv_folder}/comparisons_no_rc_and_milestones.csv", index_col=0)
comp_comb_tags_df
comp_comb_tags_df = tagsDf[["Tag", "Date"]].merge(comp_comb_tags_df, left_on="Tag", right_on="to_tag").drop(columns=["Tag"]).rename(columns={"Date": "to_date"})
compareTagsDf = tagsDf[["Tag", "Date"]].merge(comp_comb_tags_df, left_on="Tag", right_on="from_tag").drop(columns=["Tag"]).rename(columns={"Date": "from_date"})
#compareTagsDf.to_csv(f"{csv_folder}/compare_tags_sha.csv")
compareTagsDf.from_date = pd.to_datetime(compareTagsDf.from_date)
compareTagsDf.to_date = pd.to_datetime(compareTagsDf.to_date)
compareTagsDf
grp = compareTagsDf[(compareTagsDf.behind_by == 0)].groupby("to_tag").idxmin()
tags_final = compareTagsDf.loc[grp.ahead_by].sort_values(by="from_tag")
tags_final<jupyter_output><empty_output><jupyter_text>**Any version missing?**<jupyter_code>set(tags_final.to_tag.unique()).difference(set(compareTagsDf.to_tag))
x = compareTagsDf[(compareTagsDf.behind_by < 3) & (compareTagsDf.ahead_by > 0)]
x["ahead_behind_sum"] = x.apply(lambda row: row.ahead_by + row.behind_by,axis=1)
tags_final = x.loc[x.groupby("to_tag").ahead_behind_sum.idxmin()].sort_values(by="from_tag")
tags_final<jupyter_output><ipython-input-48-daec223e8f84>:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  x["ahead_behind_sum"] = x.apply(lambda row: row.ahead_by + row.behind_by,axis=1)<jupyter_text>The improved mappings<jupyter_code>tags_final[tags_final.behind_by > 0]
set(tagsDf_travis.Tag.unique()) - set(tags_final.to_tag.unique())
print(tags_final[["from_tag", "to_tag"]].to_latex(index=False))<jupyter_output>\begin{tabular}{ll}
\toprule
    from\_tag &       to\_tag \\
\midrule
       4.5.6 &        4.5.7 \\
         5.1 &        5.1.1 \\
       5.1.1 &        5.1.2 \\
       5.1.2 &          5.2 \\
         5.2 &          5.3 \\
         5.3 &          5.4 \\
         5.4 &          5.5 \\
         5.5 &          5.6 \\
         5.6 &        5.6.1 \\
       5.6.1 &        5.6.2 \\
       5.6.1 &          6.0 \\
       5.6.2 &        5.6.3 \\
       5.6.3 &        5.6.4 \\
       5.6.4 &        5.6.5 \\
       5.6.5 &        5.6.6 \\
       5.6.6 &        5.6.7 \\
         6.0 &          6.1 \\
         6.1 &        6.1.1 \\
       6.1.1 &          6.2 \\
         6.2 &          6.3 \\
         6.3 &        6.3.1 \\
       6.3.1 &        6.3.2 \\
       6.3.2 &          6.4 \\
         6.4 &          6.5 \\
         6.5 &          6.6 \\
         6.6 &          6.7 \\
         6.7 &        6.7.1 \\
       6.7.1 &          7.0 \\
       6.7.1 &        6.7.2 \\
       6.7.2 &        6.7.3 \\[...]<jupyter_text>Analysis **Ahead by distribution**<jupyter_code>sns.boxplot(x=tags_final.ahead_by)
sns.distplot(tags_final.ahead_by)
tags_final.sort_values(by="ahead_by", ascending=False).head(5)
tags_final.sort_values(by="ahead_by").head(5)<jupyter_output><empty_output><jupyter_text>**Difference in days**<jupyter_code>tags_final.to_date = pd.to_datetime(tags_final.to_date)
tags_final.from_date = pd.to_datetime(tags_final.from_date)
tags_final["diff_dates"] = tags_final.to_date - tags_final.from_date
tags_final["diff_days"] = tags_final.diff_dates / np.timedelta64(1, "D")
sns.distplot(tags_final.diff_days)
tags_final.sort_values(by = "diff_days", ascending=False)
tags_final.sort_values(by = "diff_days", ascending=False).tail()
len(tags_final)
import git
git = git.Git("../../../sonarqube/") 
def retrieve_date_for_sha(sha):
    lines = git.show("--pretty=fuller", sha, "--date=format:'%Y-%m-%d %H:%M:%S'").split("\n")
    for line in lines:
        if "CommitDate" in line:
            commit_date = line
            st = commit_date.split(": ")[1]
            return st[st.find("'")+1:st.find("''")]
    return None
tags_final["from_date_new"] = tags_final.from_sha.apply(retrieve_date_for_sha)
tags_final["to_date_new"] = tags_final.to_sha.apply(retrieve_date_for_sha)
tags_final.from_date_new = pd.to_datetime(tags_final.from_date_new)
tags_final.to_date_new = pd.to_datetime(tags_final.to_date_new)
tags_final = tags_final.rename(columns={
    "from_date":"from_author_date",
    "to_date":"to_author_date",
    "from_date_new":"from_commit_date",
    "to_date_new":"to_commit_date",
    "diff_dates":"diff_author_dates",
    "diff_days":"diff_author_days",
})
tags_final["diff_commit_dates"] = tags_final.to_commit_date - tags_final.from_commit_date
tags_final["diff_commit_days"] = tags_final.diff_commit_dates / np.timedelta64(1, "D")<jupyter_output><empty_output><jupyter_text>**Merge with jira version**<jupyter_code>tags_final.columns
tags = tagsDf
#Replace git tag with jira versions so that it matches the format of the other datasets
tags_final = tags_final.merge(tags[["Tag","jira_version"]], left_on="to_tag", right_on="Tag")\
.drop(columns=["to_tag", "Tag"]).rename(columns={"jira_version":"to_tag"})
tags_final = tags_final.merge(tags[["Tag","jira_version"]], left_on="from_tag", right_on="Tag")\
.drop(columns=["from_tag", "Tag"]).rename(columns={"jira_version":"from_tag"})
tags_final<jupyter_output><empty_output><jupyter_text>Export to csv<jupyter_code>tags_final.to_csv(f"{csv_folder}/tags_comparison_final_updated_no_rc_and_milestones.csv")
tags_final = pd.read_csv(f"{csv_folder}/tags_comparison_final_updated_no_rc_and_milestones.csv", index_col=0)
tags_final<jupyter_output><empty_output><jupyter_text>Commits difference between tags<jupyter_code>tags_comparison_shas = {}
for i, row in tags_final.iterrows():
    shas = []
    for commit in git.log("--pretty=oneline", "--right-only", f"{row.from_sha}...{row.to_sha}").split("\n"):
        sha = commit.split(" ")[0]
        shas.append(sha)
    if(len(shas) != (row.ahead_by)):
        print(f"{row.from_tag}...{row.to_tag} Mismatch in commits retrieved and ahead_by! {len(shas)} and {row.ahead_by}")
        print(shas)
        break
    tags_comparison_shas[f"{row.from_tag}...{row.to_tag}"] = shas
data = []
for key in tags_comparison_shas:
    from_tag = key.split("...")[0]
    to_tag = key.split("...")[1]
    for commit in tags_comparison_shas[key]:
        data.append((from_tag, to_tag, commit))
tags_comparison_commits = pd.DataFrame(data, columns=["from_tag", "to_tag", "ahead_sha"])
tags_comparison_commits.to_csv(f"{csv_folder}/tags_comparison_commits_no_rc_and_milestones.csv")
tags_comparison_commits<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16299559471365638, 'fraction_numerical': 0.026294052863436123, 'mean_word_length': 3.8823924731182795, 'pattern_counts': {'":': 11, '<': 45, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '208971', 'license': 'MIT', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2832, 'n_words': 352, 'path': 'notebooks/parser_and_validation/Tag_comparison_parser_no_rc_and_milestones.ipynb', 'repo_name': 'IncioMan/ci_data_issue_prediction', 'size': '7264'}