<jupyter_start><jupyter_text>Select the columns<jupyter_code>relevant_columns = ["work_item", "work_type", "work_priority", "domain", "platform", "components", "timestamp", "process_phase"]
duration_type = ["duration_in_days"]<jupyter_output><empty_output><jupyter_text>Compute times for every process phase<jupyter_code>filename = Path("../data/gdsc2_public.csv")
data = load_table(filename)
data.sort_values(by='timestamp', inplace=True)
data = add_phase(data, col_name="process_phase", cols=["from_phase", "to_phase"], as_str=False)
#data = add_phase(data, col_name="resource_phase", cols=["from_resource", "to_resource"], as_str=False)
times = time_for_phase(data, relevant_columns=relevant_columns, end_date=None)<jupyter_output><empty_output><jupyter_text>Rename and order columns<jupyter_code>times = add_phase(times, col_name="work_type_change", cols=["work_type_x", "work_type_y"])
times = add_phase(times, col_name="work_priority_change", cols=["work_priority_x", "work_priority_y"])
times = add_phase(times, col_name="domain_change", cols=["domain_x", "domain_y"])
times = add_phase(times, col_name="components_change", cols=["components_x", "components_y"])
new_names = {
            "to_phase": "current_phase",
            "process_phase_x": "process_change",
            "timestamp_x": "from_timestamp",
            "timestamp_y": "to_timestamp",
            "process_phase_y": "next_process"
            }
times.rename(columns=new_names, inplace=True)
relevant_cols = ["work_item", "process_index", "process_change", "work_type_change", "work_priority_change",
                 "domain_change", "components_change", "current_phase", "next_process", *duration_type]
times= times[relevant_cols]
times.head()<jupyter_output><empty_output><jupyter_text>Only look at the closed processes<jupyter_code>is_open = pd.isnull(times["duration_in_days"])
times_open = times[is_open]
times_closed = times[~is_open]
# end = times[times["next_process"].apply(lambda x: "End" in str(x))]
# times_closed = times[times["work_item"].isin(end.work_item.values)]
print(times_closed[times_closed["process_index"]==1]["process_change"].value_counts(dropna=False))
print(times_closed[times_closed["process_change"].apply(lambda x: x[0] == "Start")]["current_phase"].value_counts(dropna=False))<jupyter_output>(Start, Analyze)    11048
Name: process_change, dtype: int64
Analyze    11048
Name: current_phase, dtype: int64<jupyter_text>Interesting: All work items start with the label "start" and go to the label "analyze". Analyze should be main indicator on how long the ongoing process takes. We should check:1. Is there a correlation between other features and the change from x -> analyze -> y2. Is there a correlation between the time it takes to analyze and the change from analyze -> y3. When and how often does a process come back to the state analyzeFirst we should check if there is a change between the other features<jupyter_code>times_closed["work_type_change"].value_counts()
times_closed["work_priority_change"].value_counts()
times_closed["domain_change"].value_counts()
times_closed[times_closed["components_change"].apply(lambda x: str(x[0]) != str(x[1]))]<jupyter_output><empty_output><jupyter_text>There is no change in work_type, work_priority, domain and components, so these features are static and do not affect the change of process2. Let's check if there is a correlation between the time it takes to analyze and the change from analyze -> y<jupyter_code>analyze = times_closed[times_closed["current_phase"] == "Analyze"]
analyze["next_process"].value_counts()
analyze["next_process"] = analyze["next_process"].astype("category")
analyze["next_process_cat"] = analyze["next_process"].cat.codes
lab = dict([(code, category) for code, category in enumerate(analyze["next_process"].cat.categories)])
analyze.plot(kind="scatter", x="next_process_cat", y="duration_in_days", figsize=(15,8), s=5)
plt.xlabel(lab)
plt.title("Following processes after Analyze corresponding to durarion_in_days for state Analyze")
plt.show()<jupyter_output><empty_output><jupyter_text>Analyze -> Design and Analyze -> End are most frequent paths but it looks like Analyze is individual and work ticket based, maybe some other feature affects the time of the state analyze, we should add a third dimension using a color<jupyter_code>wt_1 = analyze[analyze["work_type_change"].apply(lambda x: "WT_00001" in str(x))]
wt_2 = analyze[analyze["work_type_change"].apply(lambda x: "WT_00002" in str(x))]
wt_3 = analyze[analyze["work_type_change"].apply(lambda x: "WT_00003" in str(x))]
plt.figure(figsize=(15,8))
plt.scatter(x=wt_1["next_process_cat"], y=wt_1["duration_in_days"], s=8, marker='<', color='blue', label="work_type_1")
plt.scatter(x=wt_2["next_process_cat"], y=wt_2["duration_in_days"], s=6, marker='2', color='red', label="work_type_2")
plt.scatter(x=wt_3["next_process_cat"], y=wt_3["duration_in_days"], s=5, marker='>', color='limegreen', label="work_type_3")
plt.legend()
plt.xlabel(lab)
plt.title("Following processes after Analyze corresponding to durarion_in_days for state Analyze")
#plt.savefig("C:/Users/marcelfe/Desktop/scatter.png", dpi=800)
plt.show()
wp_1 = analyze[analyze["work_priority_change"].apply(lambda x: "WP_00001" in str(x))]
wp_2 = analyze[analyze["work_priority_change"].apply(lambda x: "WP_00002" in str(x))]
wp_3 = analyze[analyze["work_priority_change"].apply(lambda x: "WP_00003" in str(x))]
wp_4 = analyze[analyze["work_priority_change"].apply(lambda x: "WP_00004" in str(x))]
wp_5 = analyze[analyze["work_priority_change"].apply(lambda x: "WP_00005" in str(x))]
plt.figure(figsize=(15,8))
plt.scatter(x=wp_1["next_process_cat"], y=wp_1["duration_in_days"], s=6, marker='<', color='blue', label="work_prio_1")
#plt.scatter(x=wp_2["next_process_cat"], y=wp_2["duration_in_days"], s=markersize, marker='2', color='red', label="work_prio_2")
#plt.scatter(x=wp_3["next_process_cat"], y=wp_3["duration_in_days"], s=markersize, marker='3', color='green', label="work_prio_3")
#plt.scatter(x=wp_4["next_process_cat"], y=wp_4["duration_in_days"], s=markersize, marker='4', color='black', label="work_prio_4")
plt.scatter(x=wp_5["next_process_cat"], y=wp_5["duration_in_days"], s=4, marker='>', color='limegreen', label="work_prio_5")
plt.legend()
plt.xlabel(lab)
plt.title("Following processes after Analyze corresponding to durarion_in_days for state Analyze")
#plt.savefig("C:/Users/marcelfe/Desktop/scatter.png", dpi=800)
plt.show()<jupyter_output><empty_output><jupyter_text>Seems like work_type doesn't have a big impact on the analyze step, while work_priority could be an indicator for the duration, lower work priorities are analyzed faster than higher ones. Could be because low priority problems are simpler than the high prios. Let's have a look at the third question:3. When and how often does a process come back to the state analyzeFirst of all we want to take a look on the phases and times in general again<jupyter_code>tclosed = times_closed
tclosed.loc[:, "work_type"] = tclosed["work_type_change"].apply(lambda x: x[0])
tclosed.loc[:, "work_priority"] = tclosed["work_priority_change"].apply(lambda x: x[0])
tclosed.loc[:, "domain"] = tclosed["domain_change"].apply(lambda x: x[0])
tclosed.loc[:, "components"] = tclosed["components_change"].apply(lambda x: x[0])
tclosed = tclosed[["work_item", "process_index", "process_change", "current_phase", "next_process", "work_type",
                  "work_priority", "domain", "components", "duration_in_days"]]
tclosed["work_item"] = tclosed["work_item"].astype("category")
tclosed.loc[:, "duration_days_cumsum"] = tclosed.groupby("work_item")["duration_in_days"].apply(lambda x: x.cumsum())
analyze = tclosed[tclosed["current_phase"]=="Analyze"]
design = tclosed[tclosed["current_phase"]=="Design"]
build = tclosed[tclosed["current_phase"]=="Build"]
test = tclosed[tclosed["current_phase"]=="Test"]
package = tclosed[tclosed["current_phase"]=="Package"]
accept = tclosed[tclosed["current_phase"]=="Accept"]
deploy = tclosed[tclosed["current_phase"]=="Deploy"]
plt.figure(figsize=(15,8))
plt.scatter(x=analyze["work_item"].cat.codes, y=analyze["duration_days_cumsum"], s=6, color='blue', label="Analyze")
plt.scatter(x=design["work_item"].cat.codes, y=design["duration_days_cumsum"], s=6, marker='<', color='red', label="Design")
plt.scatter(x=build["work_item"].cat.codes, y=build["duration_days_cumsum"], s=6, marker='3', color='green', label="Build")
plt.scatter(x=test["work_item"].cat.codes, y=test["duration_days_cumsum"], s=6, marker='4', color='cyan', label="Test")
plt.scatter(x=package["work_item"].cat.codes, y=package["duration_days_cumsum"], s=4, marker='2', color='grey', label="Package")
plt.scatter(x=accept["work_item"].cat.codes, y=accept["duration_days_cumsum"], s=4, marker='1', color='yellow', label="Accept")
plt.scatter(x=deploy["work_item"].cat.codes, y=deploy["duration_days_cumsum"], s=4, marker='>', color='magenta', label="Deploy")
plt.legend()
plt.xlabel("Work items")
plt.ylabel("Cumulated duration in Days")
plt.title("Current Phase based on work item")
plt.show()<jupyter_output><empty_output><jupyter_text>With these outlieres it looks like a exponential distribution with low lambda. Outliers decrease with increasing work items, the reason could be optimized processes over time, more support employees or automization. Or just because they don't have an end yet.Let's have a closer look on the crouded range on the bottom of the plot<jupyter_code>plt.figure(figsize=(15,8))
plt.scatter(x=analyze["work_item"].cat.codes, y=analyze["duration_days_cumsum"], s=20, color='blue', label="Analyze")
plt.scatter(x=design["work_item"].cat.codes, y=design["duration_days_cumsum"], s=6, marker='<', color='red', label="Design")
plt.scatter(x=build["work_item"].cat.codes, y=build["duration_days_cumsum"], s=6, marker='3', color='green', label="Build")
plt.scatter(x=test["work_item"].cat.codes, y=test["duration_days_cumsum"], s=6, marker='4', color='cyan', label="Test")
plt.scatter(x=package["work_item"].cat.codes, y=package["duration_days_cumsum"], s=4, marker='2', color='grey', label="Package")
plt.scatter(x=accept["work_item"].cat.codes, y=accept["duration_days_cumsum"], s=4, marker='1', color='yellow', label="Accept")
plt.scatter(x=deploy["work_item"].cat.codes, y=deploy["duration_days_cumsum"], s=4, marker='>', color='magenta', label="Deploy")
plt.legend()
plt.ylim((0,200))
plt.xlabel("Work items")
plt.ylabel("Cumulated duration in Days")
plt.title("Current Phase based on work item - ZOOMED IN")
plt.show()<jupyter_output><empty_output><jupyter_text>We see blue dots all over the range. But we can't identify if the inital Analyze phase lasts long or it's a reccuring process, so we should restrict on the work items which have a reccuring phase<jupyter_code>tclosed.loc[:, "dupl"] = tclosed.groupby("work_item")["current_phase"].apply(lambda x: x.duplicated()==True)
group_dupl = tclosed.groupby("work_item").apply(lambda x: True if (True in x["dupl"].values) else False)
dupl_df = pd.DataFrame(group_dupl).reset_index()
dupl_wi = dupl_df[dupl_df[0]==True]
dupl_phase = tclosed[tclosed["work_item"].apply(lambda x: x in dupl_wi["work_item"].values)]
analyze = dupl_phase[dupl_phase["current_phase"]=="Analyze"]
design = dupl_phase[dupl_phase["current_phase"]=="Design"]
build = dupl_phase[dupl_phase["current_phase"]=="Build"]
test = dupl_phase[dupl_phase["current_phase"]=="Test"]
package = dupl_phase[dupl_phase["current_phase"]=="Package"]
accept = dupl_phase[dupl_phase["current_phase"]=="Accept"]
deploy = dupl_phase[dupl_phase["current_phase"]=="Deploy"]
print("There are %i unique work items, that have a recurrent process flow" %dupl_phase.work_item.nunique())
plt.figure(figsize=(15,8))
plt.scatter(x=analyze["work_item"].cat.codes, y=analyze["process_index"], s=20, marker='v', color='blue', label="Analyze")
plt.scatter(x=design["work_item"].cat.codes, y=design["process_index"], s=6, marker='<', color='red', label="Design")
plt.scatter(x=build["work_item"].cat.codes, y=build["process_index"], s=6, marker='3', color='green', label="Build")
plt.scatter(x=test["work_item"].cat.codes, y=test["process_index"], s=6, marker='4', color='cyan', label="Test")
plt.scatter(x=package["work_item"].cat.codes, y=package["process_index"], s=4, marker='2', color='black', label="Package")
plt.scatter(x=accept["work_item"].cat.codes, y=accept["process_index"], s=4, marker='1', color='yellow', label="Accept")
plt.scatter(x=deploy["work_item"].cat.codes, y=deploy["process_index"], s=4, marker='>', color='magenta', label="Deploy")
plt.legend()
#plt.ylim((0,5))
plt.xlabel("Work items")
plt.ylabel("Process step")
plt.title("Current Phase flow based on work items who have duplicated phases")
plt.show()
analyze.process_change.value_counts()
tclosed[tclosed["process_change"].apply(lambda x: x[0]==x[1])]["process_change"].value_counts()<jupyter_output><empty_output><jupyter_text>We can now answer the third question: It seems like the state Analyze comes back only once and directly after the phase of Design in most of the tickets. That is not really a big suprise because many ideas and concepts are good in theory but not feasable in an appropriate amount of time.There are also some items where the phase value don't change, like Analyze->Analyze and Design->Design. There are columns from_resource -> to_resource in the dataset which might be the answer to this.Also interesting: Does work items which have unique phases as many outliers as the duplicates?<jupyter_code>no_dupl_df = dupl_df[dupl_df[0]==False]
no_dupl_phase = tclosed[tclosed["work_item"].apply(lambda x: x in no_dupl_df["work_item"].values)]
analyze = no_dupl_phase[no_dupl_phase["current_phase"]=="Analyze"]
design = no_dupl_phase[no_dupl_phase["current_phase"]=="Design"]
build = no_dupl_phase[no_dupl_phase["current_phase"]=="Build"]
test = no_dupl_phase[no_dupl_phase["current_phase"]=="Test"]
package = no_dupl_phase[no_dupl_phase["current_phase"]=="Package"]
accept = no_dupl_phase[no_dupl_phase["current_phase"]=="Accept"]
deploy = no_dupl_phase[no_dupl_phase["current_phase"]=="Deploy"]
print("There are %i unique work items, that have a unique process flow" %no_dupl_phase.work_item.nunique())
plt.figure(figsize=(15,8))
plt.scatter(x=analyze["work_item"].cat.codes, y=analyze["duration_days_cumsum"], s=8, marker='v', color='blue', label="Analyze")
plt.scatter(x=design["work_item"].cat.codes, y=design["duration_days_cumsum"], s=6, marker='<', color='red', label="Design")
plt.scatter(x=build["work_item"].cat.codes, y=build["duration_days_cumsum"], s=6, marker='3', color='green', label="Build")
plt.scatter(x=test["work_item"].cat.codes, y=test["duration_days_cumsum"], s=6, marker='4', color='cyan', label="Test")
plt.scatter(x=package["work_item"].cat.codes, y=package["duration_days_cumsum"], s=4, marker='2', color='black', label="Package")
plt.scatter(x=accept["work_item"].cat.codes, y=accept["duration_days_cumsum"], s=4, marker='1', color='yellow', label="Accept")
plt.scatter(x=deploy["work_item"].cat.codes, y=deploy["duration_days_cumsum"], s=4, marker='>', color='magenta', label="Deploy")
plt.legend()
#plt.ylim((0,5))
plt.xlabel("Work items")
plt.ylabel("Cumulated duration in Days")
plt.title("Current Phase flow based on work items which have a unique process flow")
#plt.savefig("C:/Users/marcelfe/Desktop/scatter.png", dpi=800)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16528174500129836, 'fraction_numerical': 0.012853804206699558, 'mean_word_length': 8.996755353666451, 'pattern_counts': {'":': 5, '<': 50, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '468787', 'license': 'MIT', 'n_tokens_mistral': 5424, 'n_tokens_neox': 5212, 'n_words': 1101, 'path': 'notebooks/03_phase_analyze_eda.ipynb', 'repo_name': 'Ijusttyped/it_support_tickets', 'size': '15404'}