<jupyter_start><jupyter_text>Startup codes  Imports<jupyter_code>%load_ext autotime
import warnings
warnings.filterwarnings('always')
warnings.simplefilter('always')
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.simplefilter(action='ignore', category=DeprecationWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
import glob, os, sys
# import matplotlib.pylab as plt
# from matplotlib import pyplot
import numpy as np
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
print(tf.__version__)
import pandas as pd
pd.options.mode.chained_assignment = None
import csv
import xml.etree.ElementTree as ET
import pickle
import math
import matplotlib.pyplot as plt
%config Completer.use_jedi = False
from importlib import reload<jupyter_output><empty_output><jupyter_text>Utility methods<jupyter_code>def save_obj(obj, name ):
    with open('pickle/'+ name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
    print("Saved object to a file: %s" % (str(f)))

def load_obj(name ):
    with open('pickle/' + name + '.pkl', 'rb') as f:
        return pickle.load(f)

def save_df(df, file_name):
    df.to_csv(file_name, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)

import re
def get_tags(tags_str):
    return re.findall(r'<(.+?)>', tags_str)



def make_link(id, type):
    '''
    id = postid
    type : 'q' for question
           'a' for answer
    '''
    url = f'https://stackoverflow.com/{type}/{id}'
    return f'=HYPERLINK("{url}", "{id}")'


def linkToId(x):
    '''
    Takes a excel styled link and retures the QuestionID
    '''
    return int(x.split('"')[::-1][1])

COLS = ["Id", "PostTypeId", "AcceptedAnswerId", "ParentId", "CreationDate", "DeletionDate", "Score", "ViewCount", "Body",
        "OwnerUserId", "OwnerDisplayName", "LastEditorUserId", "LastEditorDisplayName", "LastEditDate", "LastActivityDate",
        "Title", "Tags", "AnswerCount", "CommentCount", "FavoriteCount", "ClosedDate", "CommunityOwnedDate", "ContentLicense"]

ROOT_DIR = '/'<jupyter_output><empty_output><jupyter_text>Dataset filtering  Bootstrap<jupyter_code>TAG_list_initial_file = os.path.join( ROOT_DIR, "Data", "tag_list_initial.csv")
TAG_list_final_file = os.path.join( ROOT_DIR, "Data", "tag_list_final.csv")
QUESTIONS_initial_tag_file = os.path.join(ROOT_DIR, "Data", "questions_initial_tag.csv")
QUESTIONS_final_tag_file = os.path.join(ROOT_DIR, "Data", "questions_final_tag.csv")
QUESTIONS_nss_final_tag_file = os.path.join(ROOT_DIR, "Data", "questions_nss_final_tag.csv")
ANSWERS_nss_final_tag_file = os.path.join(ROOT_DIR, "Data", "answers_nss_final_tag.csv")
TAG_manual_relevance_file = os.path.join( ROOT_DIR, "Data", "tag_manual_relevance.csv")
Tag_list_initial = None
df_questions_initial_tag = None
Tag_stat_Q_initial = {}
Tag_stat_Q_all = {}

Significance_tag = {}
Relevance_tag = {}

Tag_stat_manual_relevance = []
DF_tag_stat_summary = None
Tag_list_relevent = []
Tag_list_final = []
DF_question_final_tag = None
DF_answer_final_tag = None
DF_tag_stat_summary = None
DF_question_nss_final_tag = None
DF_answer_nss_final_tag = None<jupyter_output><empty_output><jupyter_text>Load all variables<jupyter_code>Tag_list_initial = load_obj("Tag_list_initial")
df_questions_initial_tag = load_obj("df_questions_initial_tag")
Tag_stat_Q_initial = load_obj("Tag_stat_Q_initial")
Tag_stat_Q_all = load_obj("Tag_stat_Q_all")
Significance_tag = load_obj("Significance_tag")
Relevance_tag = load_obj("Relevance_tag")
Tag_stat_manual_relevance = load_obj("Tag_stat_manual_relevance")
Tag_list_final = load_obj("Tag_list_final")
DF_question_final_tag = load_obj("DF_question_final_tag")
DF_answer_final_tag = load_obj("DF_answer_final_tag")
DF_question_nss_final_tag = load_obj("DF_question_nss_final_tag")
DF_answer_nss_final_tag = load_obj("DF_answer_nss_final_tag")
Map_tag_qids = load_obj("Map_tag_qids")
# DF_answer_final_tag = load_obj("DF_answer_final_tag")
# DF_question_nss_final_tag = load_obj("DF_question_nss_final_tag")
# DF_answer_nss_final_tag = load_obj("DF_answer_nss_final_tag")


print(len(Tag_list_initial))
print(len(df_questions_initial_tag))
print(len(Tag_stat_Q_initial))
print(len(Tag_stat_Q_all))
print(len(Significance_tag))
print(len(Relevance_tag))
print(len(Tag_stat_manual_relevance))
print(len(Tag_list_final))
print(len(DF_question_final_tag))
print(len(DF_answer_final_tag))
print(len(DF_question_nss_final_tag))
print(len(DF_answer_nss_final_tag))
# print(len(DF_answer_final_tag))
# print(len(DF_question_nss_final_tag))
# print(len(DF_answer_nss_final_tag))<jupyter_output><empty_output><jupyter_text>Helper methods<jupyter_code>def get_initial_taglist(input_file):
    df_initial_tag_list = pd.read_csv(input_file)
    df_initial_tag_list = df_initial_tag_list['Tag'].dropna()
    initial_tag_list = df_initial_tag_list.tolist()
    return initial_tag_list

import lxml.etree  as ET
def get_questions_from_tags(POSTS_file, TAGS):
    context = ET.iterparse(POSTS_file, events=("end",))
    print("Going to extract questions")
    df_posts = pd.DataFrame(columns = COLS)
    total_questions = 0
    count = 0
    _, root = next(context)
    for event, elem in context:
        # if (count > 5):
        #     break
        if elem.tag == "row":
            tags = elem.attrib.get('Tags', '')
            tags_list = get_tags(tags) # list of tags
            for tag in TAGS:
                if tag in tags_list:
                    dic = {}
                    for col in COLS:
                        dic[col] = elem.attrib.get(col, '')
                        # data.append(elem.attrib.get(col, ''))
                    df_posts = df_posts.append(pd.Series(dic), ignore_index = True)    
                    continue
            # progress
            if total_questions % 1000000 == 0:
                print('done', elem.attrib['Id'])
            elem.clear()
            root.clear()
            total_questions += 1
            # count += 1
    df_posts.drop_duplicates('Id',inplace=True)
    return df_posts<jupyter_output><empty_output><jupyter_text>Extract Initial analysis  Get Initial taglist<jupyter_code>Tag_list_initial = get_initial_taglist(TAG_list_initial_file)
print(len(Tag_list_initial))
print(Tag_list_initial)
save_obj(Tag_list_initial, "Tag_list_initial")<jupyter_output><empty_output><jupyter_text>Extract Initial Qestions<jupyter_code>df_questions_initial_tag = get_questions_from_tags(POSTS_file, Tag_list_initial)
print(len(df_questions_initial_tag))
df_questions_initial_tag.drop_duplicates('Id', inplace=True)
print(len(df_questions_initial_tag))
print(len(df_questions_initial_tag))
output_file = os.path.join(ROOT_dir, "output", "questions_initial_tag.csv")
df_questions_initial_tag.to_csv(output_file, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
print("Check the output file and then move it to dataset folder")
save_obj(df_questions_initial_tag, "df_questions_initial_tag")<jupyter_output><empty_output><jupyter_text>Significance and Relevance  Tag stat selected question<jupyter_code>def get_tags_stat(df_questions):
    tags_stat = {}
    for index, row in df_questions.iterrows():
        tags = get_tags(row['Tags'])
        for tag in tags:
            tags_stat[tag] = tags_stat.get(tag, 0) + 1
    return tags_stat
Tag_stat_Q_initial = get_tags_stat(df_questions_initial_tag)
print(len(Tag_stat_Q_initial))
save_obj(Tag_stat_Q_initial, "Tag_stat_Q_initial")
# Tag_stat_Q_initial = load_obj("Tag_stat_Q_initial")
print(len(Tag_stat_Q_initial))<jupyter_output><empty_output><jupyter_text>Tag stat all questions<jupyter_code>def tag_stat_all_questions(selected_tag_list):
    tags_stat = {}
    context = ET.iterparse(POSTS_file, events=("end",))
    print("Going to extract all tags stat from questions")
    # df_posts = pd.DataFrame(columns = COLS)
    total_questions = 0
    _, root = next(context)
    for event, elem in context:
        if elem.tag == "row":
            post_tags = elem.attrib.get('Tags', '')
            post_tags_list = get_tags(post_tags) # list of tags
            for tag in post_tags_list:
                if tag in selected_tag_list:
                    tags_stat[tag] = tags_stat.get(tag, 0) + 1

            # progress
            if total_questions % 1000000 == 0:
                print('done', elem.attrib['Id'])
            elem.clear()
            root.clear()
            total_questions += 1
            # count += 1
    # print(len(all_tags_in_Q_all))
    return tags_stat
Tag_stat_Q_all = tag_stat_all_questions(Tag_stat_Q_initial)
# save_obj(Tag_stat_Q_all, "Tag_stat_Q_all")
Tag_stat_Q_all = load_obj("Tag_stat_Q_all")
print(len(Tag_stat_Q_all))

# Tag_stat_Q_all = load_obj("all_tags_in_Q_all_stat")
# save_obj(all_tags_in_Q_all, "all_tags_in_Q_all_stat")
# # all_tags_in_Q_all = load_obj("all_tags_in_Q_all")<jupyter_output><empty_output><jupyter_text>Calculate Significance relevance<jupyter_code>for tag in Tag_stat_Q_initial:
    if tag in Tag_list_initial:
        continue
    sig = Tag_stat_Q_initial[tag] * 1.0 / Tag_stat_Q_all[tag]
    rel = Tag_stat_Q_initial[tag] * 1.0 / len(df_questions_initial_tag)
    Significance_tag[tag] = sig
    Relevance_tag[tag] = rel
for tag in Tag_stat_Q_initial:
    if tag in Tag_list_initial:
        continue
    print("Tag: %s sig: %f rel: %f" % (tag, Significance_tag[tag], Relevance_tag[tag] ))
save_obj(Significance_tag, "Significance_tag")
save_obj(Relevance_tag, "Relevance_tag")<jupyter_output><empty_output><jupyter_text>Extract based on sig rel<jupyter_code>def get_tag_with_higher_sig_and_rel(sig, rel, sig_map, rel_map):
    assert len(sig_map) == len(rel_map)
    results = []
    for tag in sig_map:
        if(sig_map[tag] >= sig and rel_map[tag] >= rel):
            results.append(tag)
    return results

def get_intersection_info(list_big, list_ref):
    list_ref = set(list_ref)
    count = 0
    for item in list_big:
        if item in list_ref:
            count += 1
    pct = round(count * 100.0 / len(list_big), 1)
    return count, pct
tags_with_lowest_sig_rel = get_tag_with_higher_sig_and_rel(.05, .001, Significance_tag, Relevance_tag)
print(len(tags_with_lowest_sig_rel))
tmp_sigs = []
tmp_rel = []
tmp_posts = []
for tag in tags_with_lowest_sig_rel:
    tmp_sigs.append(Significance_tag[tag])
    tmp_rel.append(Relevance_tag[tag])
    tmp_posts.append(Tag_stat_Q_all[tag])
df_tag_recommend = pd.DataFrame()
df_tag_recommend['sig'] = tmp_sigs
df_tag_recommend['tag'] = tags_with_lowest_sig_rel
df_tag_recommend['rel'] = tmp_rel
df_tag_recommend['posts'] = tmp_posts
output_file = os.path.join("tags", "tags_recommended_stat.csv")
df_tag_recommend.to_csv(output_file, index=False)
df_Tag_stat_manual_relevance = pd.read_csv(TAG_manual_relevance_file)
print(len(df_Tag_stat_manual_relevance))
df_Tag_stat_manual_relevance = df_Tag_stat_manual_relevance[df_Tag_stat_manual_relevance['isrelavent'] == True]
print(len(df_Tag_stat_manual_relevance))
Tag_stat_manual_relevance = df_Tag_stat_manual_relevance['tags'].tolist()
# print(Tag_stat_manual_relevance)
temp = []
for tag in Tag_stat_manual_relevance:
    tag = tag[1:-1]
    temp.append(tag)
Tag_stat_manual_relevance = temp
print(Tag_stat_manual_relevance)
save_obj(Tag_stat_manual_relevance, 'Tag_stat_manual_relevance')
# Uses global variable of Significance_tag and Relevance_tag map
def get_tag_stat_summary(significance_values, relavance_values, Tag_stat_manual_relevance):
    sig_res = []
    rel_res = []
    tags_recommended = []
    tags_recommended_count = []
    tags_relevant = []
    tags_relavant_pct = []
    for rel in relavance_values:
        for sig in significance_values:
            reco_tags = get_tag_with_higher_sig_and_rel(sig, rel, Significance_tag, Relevance_tag)
            rel_tags_count, rel_pct = get_intersection_info(reco_tags, Tag_stat_manual_relevance)
            # df = pd.DataFrame()
            # df['tags'] = selected_tags
            # file_name = "Sig_%.3f_Rel_%.3f.csv" % (sig, rel)
            # file_name = os.path.join("tags", file_name)
            # # print(file_name)
            # df.to_csv(file_name, index=False)

            sig_res.append(sig)
            rel_res.append(rel)
            tags_recommended.append(reco_tags)
            tags_recommended_count.append(len(reco_tags))
            tags_relevant.append(rel_tags_count)
            tags_relavant_pct.append(rel_pct)
            # print("Sig: %f Rel: %f len_tags: %d" % (sig, rel, len(res)))

    df = pd.DataFrame()
    df["significance"] = sig_res
    df['relevance'] = rel_res
    df['tags_recommended'] = tags_recommended
    df['tags_relevant'] = tags_relevant
    df['tags_relavant_pct'] = tags_relavant_pct
    df['tags_recommended_count'] = tags_recommended_count
    return df

significance_values = [0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35]
relavance_values    = [0.001, .005, .01, .015, .02, .025, .03]
DF_tag_stat_summary = get_tag_stat_summary(significance_values, relavance_values, Tag_stat_manual_relevance)
file_name = os.path.join("tags", "tags_stat_summary.csv")
DF_tag_stat_summary.to_csv(file_name, index=False)
save_obj(DF_tag_stat_summary, 'DF_tag_stat_summary')<jupyter_output><empty_output><jupyter_text>Extract final posts  Final taglist<jupyter_code>print(Tag_list_relevent)
print(len(Tag_list_relevent))
print(len(Tag_list_initial))

tag_unique = set(Tag_list_relevent + Tag_list_initial)
print(len(tag_unique))
Tag_list_relevent = util.load_obj(ROOT_dir, "Tag_stat_manual_relevance")
Tag_list_initial = util.load_obj(ROOT_dir, "Tag_list_initial")
Tag_list_final = Tag_list_relevent + Tag_list_initial
print(len(Tag_list_final))
# save_obj(Tag_list_final, "Tag_list_final")<jupyter_output><empty_output><jupyter_text>Final questions<jupyter_code>DF_question_final_tag = util.load_obj(ROOT_dir, "DF_question_final_tag")
print(len(DF_question_final_tag))

DF_question_final_tag = util.load_obj(ROOT_dir, "DF_question_final_tag")
DF_question_final_tag = get_questions_from_tags(POSTS_file, Tag_list_final)
print(len(DF_question_final_tag))
output_file = os.path.join("output", "questions_final_tag.csv")
DF_question_final_tag.to_csv(output_file, index=False)
save_obj(DF_question_final_tag, "DF_question_final_tag")
# print(len(DF_question_final_tag))
# DF_question_final_tag.drop_duplicates('Id', inplace=True)
# output_file = os.path.join("output", "questions_final_tag.csv")
# DF_question_final_tag.to_csv(output_file, index=False)
# save_obj(DF_question_final_tag, "DF_question_final_tag")
# print(len(DF_question_final_tag))
# print(len(DF_question_final_tag))
DF_question_nss_final_tag = DF_question_final_tag[DF_question_final_tag['Score'].astype(int) >= 0]
print(len(DF_question_nss_final_tag))
save_df(DF_question_nss_final_tag, QUESTIONS_nss_final_tag_file)
save_obj(DF_question_nss_final_tag, "DF_question_nss_final_tag")<jupyter_output><empty_output><jupyter_text>Final answers<jupyter_code>def get_answers_from_questions(question_ids):
    question_ids = set(question_ids)
    context = ET.iterparse(POSTS_file, events=("end",))
    print("Going to extract answers")
    df_answers = pd.DataFrame(columns = COLS)
    total_answers = 0
    count = 0
    _, root = next(context)
    for event, elem in context:
        if elem.tag == "row":
            post_type = int(elem.attrib.get('PostTypeId'))
            if(post_type == 2):
                parent_id = elem.attrib.get('ParentId')
                if(parent_id in question_ids):
                    dic = {}
                    for col in COLS:
                        dic[col] = elem.attrib.get(col, '')
                    df_answers = df_answers.append(pd.Series(dic), ignore_index = True)    
            # progress
            if (int(elem.attrib['Id']) % 1000000) == 0:
                print('done id: %s and len of res: %d' % (elem.attrib['Id'], len(df_answers)))
            total_answers += 1
            elem.clear()
            root.clear()            
    print("Total answers: %d" % (total_answers))
    return df_answers


question_ids = DF_question_final_tag['Id'].tolist()
print(len(question_ids))
DF_answer_final_tag = get_answers_from_questions(question_ids)

print(len(DF_answer_final_tag))
output_file = os.path.join("output", "answers_final_tag.csv")
DF_answer_final_tag.to_csv(output_file, index=False)
save_obj(DF_answer_final_tag, "DF_answer_final_tag")
print(len(DF_answer_final_tag))
DF_answer_final_tag.drop_duplicates('Id', inplace=True)
output_file = os.path.join("output", "answers_final_tag.csv")
DF_answer_final_tag.to_csv(output_file, index=False)
save_obj(DF_answer_final_tag, "DF_answer_final_tag")
print(len(DF_answer_final_tag))
q_ids = DF_question_nss_final_tag["Id"].tolist()
print(len(q_ids))
print(len(DF_answer_final_tag))
DF_answer_nss_final_tag = DF_answer_final_tag[DF_answer_final_tag['ParentId'].isin(q_ids)]
print(len(DF_answer_nss_final_tag))
save_df(DF_answer_nss_final_tag, ANSWERS_nss_final_tag_file)
save_obj(DF_answer_nss_final_tag, "DF_answer_nss_final_tag")<jupyter_output><empty_output><jupyter_text>Tag list to Questions<jupyter_code>Map_tag_qids = {}
for tag in Tag_list_final:
    Map_tag_qids[tag] = []

for index, row in All_Questions_df.iterrows():
        tags = get_tags(row['Tags'])
        for tag in tags:
            if(tag in Tag_list_final):
                Map_tag_qids[tag].append(int(row['Id']))
save_obj(Map_tag_qids, "Map_tag_qids")

total = 0
for tag in Map_tag_qids:
    total += len(Map_tag_qids[tag])
    print(tag, len(Map_tag_qids[tag]))
print(total)<jupyter_output><empty_output><jupyter_text>Generate final stats on posts.xml<jupyter_code>def generate_final_stats(POSTS_file):
    context = ET.iterparse(POSTS_file, events=("end",))
    print("Going to extract questions")
    total_posts = 0
    total_Q = 0
    total_A = 0
    total_Q_with_acc = 0
    all_post_ids = set()

    _, root = next(context)
    for event, elem in context:
        if elem.tag == "row":
            id = int(elem.attrib.get('Id'))
            if id in all_post_ids:
                continue
            all_post_ids.add(id)
            total_posts += 1
            post_type = int(elem.attrib.get('PostTypeId'))            
            if(post_type == 1):
                total_Q += 1
                acc_id = (elem.attrib.get('AcceptedAnswerId'))
                if(acc_id is not None and len(acc_id) > 4):
                    total_Q_with_acc += 1

                
            else:
                total_A += 1
            # progress
            if total_posts % 10000000 == 0:
                print('done', elem.attrib['Id'])
                # break
            elem.clear()
            root.clear()

    return total_posts, total_Q, total_A, total_Q_with_acc

total_posts, total_Q, total_A, total_Q_with_acc = generate_final_stats(POSTS_file)

print(total_posts, total_Q, total_A, total_Q_with_acc)<jupyter_output><empty_output><jupyter_text>Sanity testing<jupyter_code>df = DF_question_final_tag[DF_question_final_tag['Score'].astype(int) < 0]
print(len(df))
ids = df_questions_initial_tag['Id'].tolist()
print(len(ids))
print(len(set(ids)))
df = pd.read_csv("experiment/IoTPostInformation.csv")
print(len(df))
print(df.head())
ids = df['PostId'].tolist()
print(len(ids))
print(len(set(ids)))
# assert len(question_ids) == len(set(question_ids))
question_ids = DF_question_final_tag['Id'].tolist()
print(len(question_ids))
print(len(set(question_ids)))
total_answers = DF_question_final_tag['AnswerCount'].astype(int).sum()
accepted_answers = DF_question_final_tag[DF_question_final_tag['AcceptedAnswerId'].str.len() > 0]['AcceptedAnswerId'].tolist()
# print(accepted_answers)
print(len(accepted_answers))
print(total_answers)
print(len(DF_answer_final_tag))
print(type(question_ids[0]))<jupyter_output><empty_output><jupyter_text>Topic Modelling  Bootstrap<jupyter_code>import string
import gensim
import unicodedata
from gensim import corpora
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import *
from gensim.models.wrappers import LdaMallet
from gensim.models import CoherenceModel
import pyLDAvis.gensim
import pyLDAvis
import json
import html
import warnings
from tqdm import tqdm
import matplotlib.pyplot as plt
warnings.filterwarnings('ignore')
stemmer = PorterStemmer()
pyLDAvis.enable_notebook()
# os.environ.update({'MALLET_HOME':'./Mallet'}) 
# mallet_path = './Mallet/bin/mallet'
os.environ.update({'MALLET_HOME': r"C:\Mallet"})
mallet_path = r'C:\Mallet\bin\mallet'  # update this path
current_dir = os.getcwd()
coherence_scores = []

# ldamallet = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=5, id2word=dictionary, alpha=5)


def create_dir(parent_dir, dir_name):
    temp = os.path.join(parent_dir,dir_name)
    try:  
        os.mkdir(temp)  
    except OSError as error:  
        print(error)
    return temp<jupyter_output><empty_output><jupyter_text>Variable initializations<jupyter_code>DF_question_nss_final_tag = load_obj("DF_question_nss_final_tag")
DF_answer_nss_final_tag = load_obj("DF_answer_nss_final_tag")
DF_accepted_answer_nss_final_tag = None
TM_data_file = os.path.join("output", "topicModeling", "TM_data.csv")
TM_data_processed_file = os.path.join("output", "topicModeling", "TM_data_processed.csv")<jupyter_output><empty_output><jupyter_text>Preprocess the posts<jupyter_code>accepted_answers_ids = DF_question_nss_final_tag[DF_question_nss_final_tag['AcceptedAnswerId'].str.len() > 0]['AcceptedAnswerId'].tolist()
print(len(accepted_answers_ids))
DF_accepted_answer_nss_final_tag = DF_answer_nss_final_tag[DF_answer_nss_final_tag['Id'].isin(accepted_answers_ids)]
print(len(DF_accepted_answer_nss_final_tag))
answers_df = DF_accepted_answer_nss_final_tag.copy()
answers_df = answers_df[['Id', 'Body']]
answers_df = answers_df.dropna()
answers_df.columns = ['Id', 'raw_data']
answers_df.insert(1, 'qa', 'a')
answers_df.Id = answers_df.Id.apply(np.int64)
print(answers_df.head())
print(len(answers_df))
questions_df = DF_question_nss_final_tag.copy()
questions_df = questions_df[['Id', 'Body', 'Title']]
questions_df['titlePlusQuestion'] = questions_df[['Title', 'Body']].apply(lambda x: ' '.join(x), axis=1)
questions_df = questions_df.drop(['Title', 'Body'], axis=1)
questions_df.columns = ['Id', 'raw_data']
questions_df.insert(1, 'qa', 'q')
questions_df.Id = questions_df.Id.apply(np.int64)
print(questions_df.head())
print(len(questions_df))
df = pd.concat([questions_df,answers_df])
print(len(df))
# output_file = os.path.join("output", "topicModeling", "TM_data")
# df.to_csv(TM_data_file, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
save_df(df, TM_data_file)<jupyter_output><empty_output><jupyter_text>Generate temporary TM file  Process stop words<jupyter_code>import nltk
# nltk.download('stopwords')
nltk.download('wordnet')
# stop words set
STOP_WORDS_FILES = ['mallet_stop_words.txt']
stop_words_set = set()
stop_words = set(stopwords.words('english'))
for word in stop_words:
    if('\'' in word):
        stop_words_set.add(word.strip().replace('\'', ''))
    stop_words_set.add(word)
for swfile in STOP_WORDS_FILES:
    try:
        with open(swfile, 'r') as f:
            words = f.readlines()
            for word in words:
                if('\'' in word):
                    stop_words_set.add(word.strip().replace('\'', ''))
                stop_words_set.add(word.strip())
    except:
        pass
def preprocess_text(text):
    # remove non ascii
    text = unicodedata.normalize('NFKD', text).encode(
        'ascii', 'ignore').decode('utf-8', 'ignore')
    text = text.lower()
    # unescaping
    text = html.unescape(text)
    text = re.sub(r'<code>(.|\n)*?</code>','', text) # removing <code>...</code>
    text = re.sub(r'<a.*?</a>', '', text)  # removing whole anchor tags
    text = re.sub(r'(<.*?>)', '', text)  # removing html markup
    text = re.sub(r'[^\w\s]', '', text)  # removing punctuation
    text = re.sub(r'[\d]', '', text)  # removing digits
    # remove stopwords
    tokenized = []
    for word in text.split():
        if word in stop_words_set:
            continue
        tokenized.append(word)
    for i in range(len(tokenized)):
        word = tokenized[i]
        word = WordNetLemmatizer().lemmatize(word, pos='v')
        tokenized[i] = stemmer.stem(word)
        # tokenized[i] = word
    return tokenized
df_raw = pd.read_csv(TM_data_file)
df_raw['preprocessed'] = ""
for i in range(len(df_raw)):
    df_raw['preprocessed'].iloc[i] = preprocess_text(df_raw.raw_data.iloc[i])
save_df(df_raw, TM_data_processed_file)
# df.to_csv('preprocesseedData.csv')
df_raw.head()
save_obj(df_raw, "processed_df")
# save_obj(df_raw, "processed_df")
df_TM = load_obj("processed_df")
print(len(df_TM))<jupyter_output><empty_output><jupyter_text>Topic Modelling  Optimal number of topics<jupyter_code># df_TM = pd.read_csv(TM_data_processed_file)
# print(df.head)
topic_range = []
print(topic_range)
for i in range(5, 71, 5):
    if i not in topic_range:
        topic_range.append(i)
topic_range.sort()
print(topic_range)
# finding out optimum topic number
coherence_file = os.path.join("output", "topicModeling", "Coherence_Scores_5_70_nns.csv")
image_file = "scores_5_70_nns.png"
data = df_TM['preprocessed']
print(data)
dictionary = gensim.corpora.Dictionary(data)
corpus = [dictionary.doc2bow(doc) for doc in data]
coherence_scores = []
for num_topics in tqdm(topic_range):
    ldamallet = gensim.models.wrappers.LdaMallet(
        mallet_path, corpus=corpus, num_topics=num_topics, id2word=dictionary, alpha=50/num_topics,  iterations=1000)
    coherenceModel = CoherenceModel(model=ldamallet, texts=data, dictionary=dictionary, coherence='c_v')
    score = coherenceModel.get_coherence()
    coherence_scores.append([num_topics,score])
# save scores as csv
ch_df = pd.DataFrame(coherence_scores,columns=['Num Topic','Score'])

save_df(ch_df, coherence_file)
# ch_df.to_csv('Coherence_Scores.csv')
# plot
plt.xlabel('Number of Topics') 
plt.ylabel('Coherence Score') 
x = []
y = []
for score in coherence_scores:
    x.append(score[0])
    y.append(score[1])
plt.plot(x,y,c='r')
plt.gca().set_aspect('auto')
plt.grid()
plt.savefig(image_file, dpi=500)
plt.show()<jupyter_output><empty_output><jupyter_text>Generating topics<jupyter_code>res_dir = create_dir(current_dir, 'TM_Run2')
data = df_TM['preprocessed']
dictionary = gensim.corpora.Dictionary(data)
corpus = [dictionary.doc2bow(doc) for doc in data]
# create folder for topic number
NUM_TOPIS = [25, 45]
for num_topics in NUM_TOPIS:
    topic_dir = create_dir(res_dir, f'{num_topics}_Topics')
    if os.path.isfile(os.path.join(topic_dir, 'ldamallet.pkl')):
        ldamallet = pickle.load(
            open(os.path.join(topic_dir, 'ldamallet.pkl'), "rb"))
    else:
        ldamallet = gensim.models.wrappers.LdaMallet(
            mallet_path, corpus=corpus, num_topics=num_topics, id2word=dictionary, alpha=50/num_topics, iterations=1000)
        # save the model as pickle
        pickle.dump(ldamallet, open(os.path.join(
            topic_dir, 'ldamallet.pkl'), "wb"))
    topics = [[(word, word_prob) for word, word_prob in ldamallet.show_topic(
        n, topn=30)] for n in range(ldamallet.num_topics)]
    # term-topic matrix
    topics_df = pd.DataFrame([[f'{word} {round(word_prob,4)}' for word, word_prob in topic] for topic in topics], columns=[
        f'Term {i}' for i in range(1, 31)], index=[f'Topic {n}' for n in range(ldamallet.num_topics)]).T
    topics_df.to_csv(os.path.join(topic_dir, 'term x topic.csv'))
    # topic words
    topic_words_dir = create_dir(topic_dir, 'TopicWords')
    for n in range(num_topics):
        topic_words_df = pd.DataFrame(
            [[word_prob, word]for word, word_prob in topics[n]], columns=['Prob', 'Word'])
        topic_words_df.to_csv(os.path.join(topic_words_dir, f'{n}.csv'))
    # post to dominant topic
    corpus_topic_df = pd.DataFrame()
    corpus_topic_df['link'] = df.Id
    corpus_topic_df['qa'] = df.qa
    for i in range(len(corpus_topic_df)):
        corpus_topic_df.link.iloc[i] = make_link(df.Id.iloc[i],df.qa.iloc[i])
    topic_model_results = ldamallet[corpus]
    corpus_topics = [sorted(doc, key=lambda x: -x[1])[0]
                    for doc in topic_model_results]
    corpus_topic_df['Dominant Topic'] = [item[0] for item in corpus_topics]
    corpus_topic_df['Correlation'] = [item[1] for item in corpus_topics]
    corpus_topic_df.to_csv(os.path.join(topic_dir, 'postToTopic.csv'))
    topic_to_post_dir = create_dir(topic_dir, 'TopicToPost')
    for i in range(num_topics):
        temp = create_dir(topic_to_post_dir, str(i))
        temp_q_df = corpus_topic_df.loc[corpus_topic_df['Dominant Topic'] == i]
        temp_q_df = temp_q_df.loc[temp_q_df['qa'] == 'q']
        temp_a_df = corpus_topic_df.loc[corpus_topic_df['Dominant Topic'] == i]
        temp_a_df = temp_a_df.loc[temp_a_df['qa'] == 'a']
        temp_q_df.drop(columns=['Dominant Topic','qa']).to_csv(
            os.path.join(temp, 'Questions.csv'), index=False)
        # temp_q_df.drop(columns=['Dominant Topic','qa']).to_excel(
        #     os.path.join(temp, 'Questions.xlsx'), index=False)
        temp_a_df.drop(columns=['Dominant Topic','qa']).to_csv(
            os.path.join(temp, 'Answers.csv'), index=False)
        # temp_a_df.drop(columns=['Dominant Topic','qa']).to_excel(
        #     os.path.join(temp, 'Answers.xlsx'), index=False)
    # post count under any topic
    topic_post_cnt_df = corpus_topic_df.groupby('Dominant Topic').agg(
        Document_Count=('Dominant Topic', np.size),
        Percentage=('Dominant Topic', np.size)).reset_index()
    topic_post_cnt_df['Percentage'] = topic_post_cnt_df['Percentage'].apply(
        lambda x: round((x*100) / len(corpus), 2))
    topic_post_cnt_df.to_csv(os.path.join(topic_dir, 'postPerTopic.csv'))
    # pyLDAvis
    vis = pyLDAvis.gensim.prepare(
        gensim.models.wrappers.ldamallet.malletmodel2ldamodel(ldamallet), corpus, dictionary)
    pyLDAvis.save_html(vis, os.path.join(topic_dir, f'pyLDAvis.html'))<jupyter_output><empty_output><jupyter_text>Update topic outputs<jupyter_code>print(res_dir)
def get_post_stat(df, qids):
    res_score = []
    res_view = []
    for qid in qids:
        q = df[df['Id'] == str(qid)].iloc[0]
        res_score.append(q.Score)
        res_view.append(q.ViewCount)
    return res_score, res_view

def update_TM_reresult(file):
    df = pd.read_csv(file)
    links = df['link'].tolist()
    # print(links[0])
    # print(linkToId(links[0]))
    ids = [linkToId(i) for i in links]
    res_score, res_view = get_post_stat(DF_question_nss_final_tag, ids)
    df['score'] = res_score
    df['view_count'] = res_view
    # print(df.head)
    save_df(df, file)
    # df.to_csv("updated.csv")
    # return df

update_TM_reresult(file)<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
MAP_merged_tm_quid = load_obj("MAP_merged_tm_quid")
TMs = load_obj("TMs")
Map_cat_topic = load_obj("Map_cat_topic")
DF_question_nss_final_tag = load_obj("DF_question_nss_final_tag")
Map_topic_pop_diff = load_obj("Map_topic_pop_diff")
Map_cat_ques = load_obj("Map_cat_ques")
All_Questions_df = load_obj("DF_question_nss_final_tag")<jupyter_output><empty_output><jupyter_text>RQ Topic Modelling<jupyter_code>class TM:
    def __init__(self):
        self.name = ""
        self.ids = []
        self.low_cat = ""
        self.mid_cat = ""
        self.high_cat = ""
    def __str__(self):
        print("Name: %s, Ids: %s, Low: %s, Mid: %s, High: %s" % (self.name, self.ids, self.low_cat, self.mid_cat, self.high_cat))
# https://www.surveysystem.com/sscalc.htm
High_cats = ["Customization", "Data Storage", "Platform Adoption", "Platform Maintenance", "Third-Party Integration" ]
Sample_size = {"Customization": 95 , "Data Storage": 95, "Platform Adoption": 94, "Platform Maintenance": 94, "Third-Party Integration": 93}
File_Cat_sample = os.path.join(ROOT_dir, "output", "Category_sample.csv")
Total_Questions = 26763
Total_Ansers = 11010

MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
MAP_merged_tm_quid = load_obj("MAP_merged_tm_quid")
MAP_tm_id_ansid = load_obj("MAP_tm_id_ansid")
MAP_merged_tm_ansid = load_obj("MAP_merged_tm_ansid")
TMs = load_obj("TMs")
Map_sample = load_obj("Map_sample")
Map_cat_ques = load_obj("Map_cat_ques")
Map_cat_topic = load_obj("Map_cat_topic")
DF_sample = load_obj("DF_sample")


# MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
# MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
# MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
# MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
# MAP_tm_id_quid = load_obj("MAP_tm_id_quid")
# MAP_tm_id_quid = load_obj("MAP_tm_id_quid")

Map_cat_ques = {}<jupyter_output><empty_output><jupyter_text>TopicID vs QuestionID topics to question_ids<jupyter_code>df_tm_ques = df_tm[df_tm['qa'] == 'q']
print(len(df_tm_ques))
for topic_id in range(45):
    MAP_tm_id_quid[topic_id] = []
    ques_ids = df_tm_ques[df_tm_ques['Dominant Topic'] == topic_id]['link'].tolist()
    for i in range(len(ques_ids)):
        ques_ids[i] = linkToId(ques_ids[i])
    MAP_tm_id_quid[topic_id] = ques_ids

save_obj(MAP_tm_id_quid, "MAP_tm_id_quid")<jupyter_output><empty_output><jupyter_text>Topics to answers ids<jupyter_code>df_tm_ans = df_tm[df_tm['qa'] == 'a']
print(len(df_tm_ans))
MAP_tm_id_ansid = {}
for topic_id in range(45):
    MAP_tm_id_ansid[topic_id] = []
    ans_ids = df_tm_ans[df_tm_ans['Dominant Topic'] == topic_id]['link'].tolist()
    for i in range(len(ans_ids)):
        ans_ids[i] = linkToId(ans_ids[i])
    MAP_tm_id_ansid[topic_id] = ans_ids
save_obj(MAP_tm_id_ansid, "MAP_tm_id_ansid")<jupyter_output><empty_output><jupyter_text>Topic Categorization  Parse TM labeling file<jupyter_code>input_file = os.path.join(ROOT_dir, "dataset", "TM_label.csv")
df_tm_label = pd.read_csv(input_file)
print(df_tm_label.columns)
df_tm_label = pd.read_csv(input_file).dropna( subset=['Id', "Topic_name"])
print(len(df_tm_label))
# print(df_tm_label)
TMs = []
for index, row in df_tm_label.iterrows():
    id = int(row["Id"])
    topic_name = row["Topic_name"]
    merged_id = row["Merged Topic"]
    low_cat = row["Lower Cat"]
    mid_cat = row["Middle Cat"]
    high_cat = row["Higher Cat"]

    if(math.isnan(merged_id)):
        merged_id = -2
    merged_id = int(merged_id)
    # print(id)
    # print(merged_id)
    if(merged_id == -1):
        # print(math.isnan(merged_id))
        print("Id %d is merged" % (id))
        continue
    tm = TM()
    tm.name = topic_name
    tm.ids.append(id)
    if(merged_id != -2):
        tm.ids.append(merged_id)
    tm.low_cat = low_cat
    tm.mid_cat = mid_cat
    tm.high_cat = high_cat

    TMs.append(tm)

    print("name: %s and ids: %s" % (tm.name, tm.ids))

    # print(tm)
    # break
save_obj(TMs, "TMs")<jupyter_output><empty_output><jupyter_text>Map Questions to Categories and samples<jupyter_code>import random

print(len(TMs))
total = 0
Map_sample = {}
for high_cat in High_cats:
    # print(high_cat)
    ids = []
    topics = 0
    for tm in TMs:
        if tm.high_cat == high_cat:
            topics += 1
            ids += tm.ids
    print(ids)
    questions = []
    for id in ids:
        total_questions += len(MAP_tm_id_quid[id])
        questions += MAP_tm_id_quid[id]
    total += len(questions)
    sample = random.sample(questions, Sample_size[high_cat])
    assert len(sample) == len(set(sample))
    assert len(sample) == Sample_size[high_cat]
    Map_sample[high_cat] = sample
    Map_cat_ques[high_cat] = questions
    print("High Cat: %s Topics: %d and topics_ids: %d total_ques: %d" % (high_cat, topics, len(ids), len(questions)))
    print(sample)
print(total)           
save_obj(Map_sample, "Map_sample")    
save_obj(Map_cat_ques, "Map_cat_ques")     
for high_cat in High_cats:
    print("%s Categories has %d questions" % (high_cat, len(Map_cat_ques[high_cat])))<jupyter_output><empty_output><jupyter_text>Map cat to topic<jupyter_code>Map_cat_topic = {}
for high_cat in High_cats:
    Map_cat_topic[high_cat] = []
for tm in TMs:
    Map_cat_topic[tm.high_cat].append(tm)
for high_cat in High_cats:
    print("%s => #%d" % (high_cat, len(Map_cat_topic[high_cat])))

save_obj(Map_cat_topic, "Map_cat_topic")
#     print("%s Categories has %d questions" % (high_cat, len(Map_cat_ques[high_cat])))<jupyter_output><empty_output><jupyter_text>Create labeling sample<jupyter_code>input_file = os.path.join(ROOT_dir, "dataset", "MSR_SDLC.csv")
df_msr_sdlc = pd.read_csv(input_file)
print(len(df_msr_sdlc))
df_msr_sdlc = df_msr_sdlc[['Id', "SDLC"]]
print(df_msr_sdlc)
# df_merged = pd.merge(DF_sample, df_msr_sdlc, on="Id", how="left")
# print(len(df_merged))
# save_df(df_merged, File_Cat_sample)
DF_question_nss_final_tag = load_obj("DF_question_nss_final_tag")
print(len(DF_question_nss_final_tag))
sample_ques_ids = []
for high_cat in High_cats:
    print("High Cat: %s total_ques: %d" % (high_cat, len(Map_sample[high_cat])))
    sample_ques_ids += Map_sample[high_cat]
print(len(sample_ques_ids))
DF_question_nss_final_tag['Id'] = DF_question_nss_final_tag['Id'].astype(int)
DF_sample = DF_question_nss_final_tag[DF_question_nss_final_tag['Id'].isin(sample_ques_ids)]
print(len(DF_sample))
df_merged = pd.merge(DF_sample, df_msr_sdlc, on="Id", how="left")
save_obj(df_merged, "DF_sample")
save_df(df_merged, File_Cat_sample)
print(sample_ques_ids)
DF_sample = DF_question_nss_final_tag[DF_question_nss_final_tag['Id'] == "12556"]
print(DF_sample)
# print(type(sample_ques_ids[0]))
# print(type(DF_question_nss_final_tag.Id))
# DF_question_nss_final_tag.Id


print(len(DF_question_nss_final_tag))<jupyter_output><empty_output><jupyter_text>TMs statistics<jupyter_code># Total_Questions = 26763
# print(len(TMs))
# for tm in TMs:
#     print(tm.ids, tm.name)
a = [1, 2, 3]
b = a
a = []
b[0] = 100
print(b, a)<jupyter_output><empty_output><jupyter_text>Generate merged TM stat<jupyter_code>MAP_merged_tm_quid = {}
t = 0
for tm in TMs:
    qids = []
    for id in tm.ids:
        qids += MAP_tm_id_quid[id]
    t += len(qids)
    pct = round(len(qids) * 100.0 / Total_Questions, 1)
    print("%s %s %d %.1f" % (tm.ids, tm.name, len(qids), pct))
    MAP_merged_tm_quid[tm.name] = qids
assert t == Total_Questions
save_obj(MAP_merged_tm_quid, "MAP_merged_tm_quid")
    # print(tm.ids, tm.name)<jupyter_output><empty_output><jupyter_text>Merged TM stat with answers<jupyter_code>MAP_merged_tm_ansid = {}
t = 0
for tm in TMs:
    ans_ids = []
    for id in tm.ids:
        ans_ids += MAP_tm_id_ansid[id]
    t += len(ans_ids)
    pct = round(len(ans_ids) * 100.0 / Total_Ansers, 1)
    print("%s %s %d %.1f" % (tm.ids, tm.name, len(ans_ids), pct))
    MAP_merged_tm_ansid[tm.name] = ans_ids
assert t == Total_Ansers
save_obj(MAP_merged_tm_ansid, "MAP_merged_tm_ansid")
    # print(tm.ids, tm.name)<jupyter_output><empty_output><jupyter_text>Initialization code  RQ Topic Evolution<jupyter_code>Tag_list_final = load_obj("Tag_list_final")<jupyter_output><empty_output><jupyter_text>Absolute impact  5 Topic categories<jupyter_code>SAMPLING = '1MS'
DUMMY_DATE = pd.to_datetime('2008-8-1 18:18:37.777')
Map_cat_ques_df = {}
df_all_absolute = All_Questions_df[['Id', 'CreationDate']]
df_all_absolute.CreationDate = df_all_absolute.CreationDate.apply(pd.to_datetime)
df_all_absolute = df_all_absolute.resample(SAMPLING, on="CreationDate").count()[["Id"]]
df_all_absolute.columns = ['All']
# print(df_all.head())
for high_cat in High_cats:
    # print("%s Categories has %d questions" % (high_cat, len(Map_cat_ques[high_cat])))
    qids = Map_cat_ques[high_cat]
    df_cat = All_Questions_df[All_Questions_df['Id'].astype(int).isin(qids)]
    print("%s ==========> %d" % (high_cat, len(df_cat)))
    # df_cat = Map_cat_ques_df['Customization']
    df_cat.CreationDate = df_cat.CreationDate.apply(pd.to_datetime)
    df_cat = df_cat.append(pd.Series(data=[DUMMY_DATE], index = ['CreationDate']), ignore_index=True)
    df_cat = df_cat.resample(SAMPLING, on='CreationDate').count()[["Id"]]
    df_cat.columns = [high_cat]
    df_all_absolute = df_all_absolute.merge(df_cat, how='left', on='CreationDate', validate='one_to_one')
    Map_cat_ques_df[high_cat] = df_cat

print(df_all_absolute.head())

# for high_cat in High_cats:
#     print("%s Categories has %d questions" % (high_cat, len(Map_cat_ques_df[high_cat])))


file_name = os.path.join(ROOT_dir, "Output", "category_evolution_monthly.csv")
# save_df(df_all_absolute, file_name)
df_all_absolute.to_csv(file_name, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)<jupyter_output><empty_output><jupyter_text>Absolute topic impact  All LCSD posts evolution<jupyter_code>file_name = os.path.join(ROOT_dir, "Output", "All_posts_absolute_impact.png")

df = df_all_absolute[['All']]
df.columns = ['LCSD Posts']
ax = df.plot(figsize=(10,6))
ax.xaxis.label.set_visible(False)
# plt.xlabel('Time')
plt.ylabel('# of Questions')

labels = [x.strftime("%b, %Y") for x in df_all_absolute.index]
ax.set_xticks(labels)
ax.set_xticklabels(labels, rotation=90)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.title.set_visible(False)
# plt.axis('off')
# plt.grid()
plt.savefig(file_name, dpi=1000, bbox_inches = "tight")
plt.show()
file_name = os.path.join(ROOT_dir, "Output", "Topic_absolute_impact.png")

ax = df_all_absolute.drop(columns='All').plot(figsize=(10,6))
ax.xaxis.label.set_visible(False)
# plt.xlabel('Time')
plt.ylabel('# of Questions')

labels = [x.strftime("%b, %Y") for x in df_all_absolute.index]
ax.set_xticks(labels)
ax.set_xticklabels(labels, rotation=90)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# plt.grid()
plt.savefig(file_name, dpi=1000, bbox_inches = "tight")
plt.show()<jupyter_output><empty_output><jupyter_text>40 topics<jupyter_code># for topic in MAP_merged_tm_quid:
#     print(topic, len(MAP_merged_tm_quid[topic]))


for cat in Map_cat_topic:
    print(cat, Map_cat_topic[cat])
    for tm in Map_cat_topic[cat]:
        print(tm.name, len(tm.ids))

def plot_dataframe(df, file_name, xlablel="Time"):
    ax = df.drop(columns='All').plot(figsize=(10,6))
    ax.xaxis.label.set_visible(False)
    plt.xlabel(xlablel)
    plt.ylabel('# of Questions')
    labels = [x.strftime("%b, %Y") for x in df.index]
    ax.set_xticks(labels)
    ax.set_xticklabels(labels, rotation=90)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    plt.savefig(file_name, dpi=1000, bbox_inches = "tight")
    plt.show()

def get_initial_df():
    df = All_Questions_df[['Id', 'CreationDate']]
    df.CreationDate = df.CreationDate.apply(pd.to_datetime)
    df = df.resample(SAMPLING, on="CreationDate").count()[["Id"]]
    df.columns = ['All']
    return df

pd.options.mode.chained_assignment = None<jupyter_output><empty_output><jupyter_text>Covid19<jupyter_code>SAMPLING = '3MS'
file_name = os.path.join(ROOT_dir, "Output", "topic_evolution.csv")
# fig_file_name = os.path.join(ROOT_dir, "Output", "Topics_absolute_impact.png")


for cat in Map_cat_topic:
    # print(cat, Map_cat_topic[cat])
    df = get_initial_df()
    for tm in Map_cat_topic[cat]:
        topic = tm.name
    # for topic in MAP_merged_tm_quid:
        # print("%s Categories has %d questions" % (high_cat, len(Map_cat_ques[high_cat])))
        qids = MAP_merged_tm_quid[topic]
        df_cat = All_Questions_df[All_Questions_df['Id'].astype(int).isin(qids)]
        # print("%s ==========> %d" % (topic, len(df_cat)))
        # df_cat = Map_cat_ques_df['Customization']
        df_cat.CreationDate = df_cat.CreationDate.apply(pd.to_datetime)
        df_cat = df_cat.append(pd.Series(data=[DUMMY_DATE], index = ['CreationDate']), ignore_index=True)
        df_cat = df_cat.resample(SAMPLING, on='CreationDate').count()[["Id"]]
        df_cat.columns = [topic]
        df = df.merge(df_cat, how='left', on='CreationDate', validate='one_to_one')
        Map_cat_ques_df[topic] = df_cat
    # print(df.head())

    # df.to_csv(file_name, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
    fig_file_name = "%s_absolute_impact.png" % cat
    fig_file_name = os.path.join(ROOT_dir, "Output", fig_file_name)
    plot_dataframe(df, fig_file_name, cat)
    # break<jupyter_output><empty_output><jupyter_text>Prepandemic vs post pandemic<jupyter_code>df_pandemic = pd.DataFrame()
# pre_pandemic = []
# post_pandemic = []
for high_cat in High_cats:
    # print("%s Categories has %d questions" % (high_cat, len(Map_cat_ques[high_cat])))
    qids = Map_cat_ques[high_cat]
    df_cat = All_Questions_df[All_Questions_df['Id'].astype(int).isin(qids)]
    print("%s ==========> %d" % (high_cat, len(df_cat)))
    # # df_cat = Map_cat_ques_df['Customization']
    df_cat.CreationDate = df_cat.CreationDate.apply(pd.to_datetime)
    df_pre_pandemic = len(df_cat[(df_cat['CreationDate'] > '2018-03-01') & (df_cat['CreationDate'] < '2020-2-29')]) / 24
    df_post_pandemic = len(df_cat[(df_cat['CreationDate'] > '2020-03-01') & (df_cat['CreationDate'] < '2021-04-30')]) / 14

    dic = {}
    dic["Cat"] = high_cat
    dic["Pre_pandemic"] = int(df_pre_pandemic)
    dic["Post_pandemic"] = int(df_post_pandemic)
    

    df_pandemic = df_pandemic.append(pd.Series(dic), ignore_index = True) 

    # last_month = len(df_cat[(df_cat['CreationDate'] > '2021-03-01') & (df_cat['CreationDate'] < '2026-3-31')])
    # print(last_month)
    # pre_pandemic.append(int(df_pre_pandemic))
    # post_pandemic.append(int(df_post_pandemic))
    # print(df_pre_pandemic)
    # print(df_post_pandemic)
    # break

print(df_pandemic)

file_name = os.path.join(ROOT_dir, "output", "pandemic.csv")
df_pandemic.to_csv(file_name, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)<jupyter_output><empty_output><jupyter_text>Tags<jupyter_code>a = [1, 3]
b = [ 4, 5]
c = a + b
print(c)
for tag in sorted(Map_tag_qids, key = lambda topic: len(Map_tag_qids[topic]), reverse=True):
    print(tag, len(Map_tag_qids[tag]))

SAMPLING = '3MS'
file_name = os.path.join(ROOT_dir, "Output", "tag_evolution.csv")
fig_file_name = os.path.join(ROOT_dir, "Output", "Tag_absolute_impact.png")

df = get_initial_df()
count = 0
for tag in sorted(Map_tag_qids, key = lambda topic: len(Map_tag_qids[topic]), reverse=True):
    qids = Map_tag_qids[tag]
    print(tag, len(qids))
    df_cat = All_Questions_df[All_Questions_df['Id'].astype(int).isin(qids)]
    # print("%s ==========> %d" % (topic, len(df_cat)))
    # df_cat = Map_cat_ques_df['Customization']
    df_cat.CreationDate = df_cat.CreationDate.apply(pd.to_datetime)
    df_cat = df_cat.append(pd.Series(data=[DUMMY_DATE], index = ['CreationDate']), ignore_index=True)
    df_cat = df_cat.resample(SAMPLING, on='CreationDate').count()[["Id"]]
    df_cat.columns = [tag]
    df = df.merge(df_cat, how='left', on='CreationDate', validate='one_to_one')
    count += 1
    if(count > 15):
        break

# df.to_csv(file_name, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
plot_dataframe(df, fig_file_name)
    # break<jupyter_output><empty_output><jupyter_text>Platform wise<jupyter_code>PLATFORMS = {'Salesforce': ['salesforce', 'visualforce', 'salesforce-lightning', 'salesforce-service-cloud', 'lwc', 
                            'salesforce-communities', 'salesforce-marketing-cloud', 'salesforce-chatter', 
                            'salesforce-development', 'salesforce-ios-sdk'],
             "Oracle Apex": ['apex-code', 'apex-code', 'apex', 'oracle-apex-5', 'oracle-apex-5.1', 'oracle-apex-19.1', 'oracle-apex-18.2', 'apex-trigger'],
             "Lotus Software": ['lotus-domino', 'lotus-notes'],
             'Filemaker': ['filemaker'],
             'Microsoft PowerApps': ['powerapps', 'powerapps-canvas', 'powerapps-formula', 'powerapps-modeldriven', 'powerapps-selected-items', 'powerapps-collection'],
             'Service Now': ['servicenow', 'servicenow-rest-api'],
             'Tibco': ['tibco'],
             'Zoho Creator': ['zoho'],
             'OutSystems': ['outsystems'],
             'Pega': ['pega'],
             'Process Maker': ['processmaker'],
             'Mendix': ['mendix']
             }
Map_Platfoms_quids = {}
for platform in PLATFORMS:
    Map_Platfoms_quids[platform] = []
    for tag in PLATFORMS[platform]:
        Map_Platfoms_quids[platform] = Map_Platfoms_quids[platform] + Map_tag_qids[tag]

# for platform in PLATFORMS:
#     print(platform, len(Map_Platfoms_quids[platform]))

for platform in sorted(Map_Platfoms_quids, key = lambda platform: len(Map_Platfoms_quids[platform]), reverse=True):
    print(platform, len(Map_Platfoms_quids[platform]))

save_obj(Map_Platfoms_quids, "Map_Platfoms_quids")

SAMPLING = '3MS'
file_name = os.path.join(ROOT_dir, "Output", "platform_evolution.csv")
fig_file_name = os.path.join(ROOT_dir, "Output", "platform_absolute_impact.png")

df = get_initial_df()
count = 0
for platform in sorted(Map_Platfoms_quids, key = lambda topic: len(Map_Platfoms_quids[topic]), reverse=True):
    qids = Map_Platfoms_quids[platform]
    print(platform, len(qids))
    df_cat = All_Questions_df[All_Questions_df['Id'].astype(int).isin(qids)]
    # print("%s ==========> %d" % (topic, len(df_cat)))
    # df_cat = Map_cat_ques_df['Customization']
    df_cat.CreationDate = df_cat.CreationDate.apply(pd.to_datetime)
    df_cat = df_cat.append(pd.Series(data=[DUMMY_DATE], index = ['CreationDate']), ignore_index=True)
    df_cat = df_cat.resample(SAMPLING, on='CreationDate').count()[["Id"]]
    df_cat.columns = [platform]
    df = df.merge(df_cat, how='left', on='CreationDate', validate='one_to_one')

# df.to_csv(file_name, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
plot_dataframe(df, fig_file_name)
    # break<jupyter_output><empty_output><jupyter_text>Relative impact<jupyter_code>df_all_relative = df_all_absolute.copy(deep=True)
for high_cat in High_cats:
    df_all_relative[high_cat] = df_all_relative[high_cat] / df_all_relative['All']
# print(df_all_relative.head())


file_name = os.path.join(ROOT_dir, "Output", "Topic_relative_impact.png")

ax = df_all_relative.drop(columns='All').plot(figsize=(10,6))
ax.xaxis.label.set_visible(False)
# plt.xlabel('Time')
plt.ylabel('# of Questions')

y_labels = [ str(i)+"%" for i in range(0, 51, 10)]
labels = [x.strftime("%b, %Y") for x in df_all_relative.index]
print(type(labels[0]))
print(labels)
ax.set_xticks(labels)
ax.set_xticklabels(labels, rotation=90)
# ax.set_yticks(y_labels)
ax.set_yticklabels(y_labels)



ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# plt.grid()
plt.savefig(file_name, dpi=1000, bbox_inches = "tight")
plt.show()
print(len(All_Questions_df))<jupyter_output><empty_output><jupyter_text>RQ: Popularity difficulty<jupyter_code>class Info:
    def __init__(self):
        self.view = 0.0
        self.favorite = 0.0
        self.score = 0.0
        self.viewN = 0.0
        self.favoriteN = 0.0
        self.scoreN = 0.0
        self.fusedP = 0.0
    def __str__(self):
        return ("View: %.2f, Favorite: %.2f, Score: %.2f, ViewN: %.2f, FavoriteN: %.2f, ScoreN: %.2f, FusedP: %.2f" 
              % ( self.view, self.favorite,  self.score, self.viewN,  self.favoriteN, self.scoreN, self.fusedP))

class InfoD:
    def __init__(self):
        self.pct_qwo_acc = 0.0
        self.med_hours = 0.0

        self.pct_qwo_accN = 0.0
        self.med_hoursN = 0.0
        self.fusedD = 0.0
    def __str__(self):
        return ("pct_qwo_acc: %.2f, med_hours: %.2f, pct_qwo_accN: %.2f, med_hours: %.2f, fusedD: %.2f" 
              % ( self.pct_qwo_acc, self.med_hours,  self.pct_qwo_accN, self.med_hoursN,  self.fusedD))
a = Info()
a.view = 5
b = Info ()
b.view = 10

print(a)
# print(a.view, b.view)<jupyter_output><empty_output><jupyter_text>Popularity<jupyter_code>View_mean_sum = 0
Favorite_mean_sum = 0
Score_mean_sum = 0
Map_topic_pop_diff = {}
for topic in MAP_merged_tm_quid:
    quids = MAP_merged_tm_quid[topic]
    # print(quids)
    # print(type(quids[0]))
    df = DF_question_nss_final_tag[DF_question_nss_final_tag['Id'].astype(int).isin(quids)]
    # print(len(df))

    info = Info()
    info.view = round(pd.to_numeric(df['ViewCount']).fillna(0).astype(int).mean(), 1)
    info.favorite =  round(pd.to_numeric(df['FavoriteCount']).fillna(0).astype(int).mean(), 1)
    info.score =  round(pd.to_numeric(df['Score']).fillna(0).astype(int).mean(), 1)
    # print(info)
    View_mean_sum += info.view
    Favorite_mean_sum += info.favorite
    Score_mean_sum += info.score

    Map_topic_pop_diff[topic] = info
    assert len(quids) == len(df)
    # break
print("view_sum: %.2f favorite_sum: %.2f score_sum: %.2f" % (View_mean_sum, Favorite_mean_sum, Score_mean_sum))

for topic in MAP_merged_tm_quid:
    info = Map_topic_pop_diff[topic]
    info.viewN = info.view * 40.0 / View_mean_sum
    info.favoriteN = info.favorite * 40.0 / Favorite_mean_sum
    info.scoreN = info.score * 40.0 / Score_mean_sum
    info.fusedP = round((info.viewN + info.favoriteN + info.scoreN) / 3.0, 2)

# for topic in MAP_merged_tm_quid:
#     print(Map_topic_pop_diff[topic])

for key in sorted(Map_topic_pop_diff, key = lambda topic: Map_topic_pop_diff[topic].fusedP, reverse=True):
    print(Map_topic_pop_diff[key])
save_obj(Map_topic_pop_diff, "Map_topic_pop_diff")<jupyter_output><empty_output><jupyter_text>Generate topic popularity table<jupyter_code>def get_high_cat(topic):
    for tm in TMs:
        if tm.name == topic:
            return tm.high_cat
    return None
DF_pop = pd.DataFrame()

topics = []
categories = []
fusedPs = []
scores = []
favorites = []
views = []



for key in sorted(Map_topic_pop_diff, key = lambda topic: Map_topic_pop_diff[topic].fusedP, reverse=True):
    info = Map_topic_pop_diff[key]
    topics.append(key)
    categories.append(get_high_cat(key))
    fusedPs.append(info.fusedP)
    views.append(info.view)
    favorites.append(info.favorite)
    scores.append(info.score)



DF_pop['Topic'] = topics
DF_pop['Category'] = categories
DF_pop['FusedP'] = fusedPs
DF_pop['#View'] = views
DF_pop['#Favorite'] = favorites
DF_pop['#Score'] = scores

print(df_pop)
save_obj(DF_pop, "DF_pop")
file_name = os.path.join(ROOT_dir, "output", "Topic_popularity.csv")
save_df(DF_pop, file_name=file_name)
TMs[0].name<jupyter_output><empty_output><jupyter_text>Difficulty<jupyter_code>Map_topic_diff = {}
# for topic in MAP_merged_tm_quid:
#     Map_topic_diff[topic] = InfoD()
def get_acc_time(ques_id):
    question = DF_question_nss_final_tag[DF_question_nss_final_tag.Id == str(ques_id)].iloc[0]
    # print(type(question))
    question_time = pd.to_datetime(question.CreationDate)
    # print(type(question_time))
    # print("question_time: %s" % (question_time) )
    answer_id = question['AcceptedAnswerId']
    answer = DF_answer_nss_final_tag[DF_answer_nss_final_tag.Id == str(answer_id)].iloc[0]
    answer_time = pd.to_datetime(answer.CreationDate)
    # print("answer_time: %s" % (answer_time) )
    time_taken = (answer_time - question_time) / (pd.Timedelta(hours=1))
    # print(time_taken)
    return time_taken

    # print(question_time, answer_time, time_taken)


print(get_acc_time(22940521))<jupyter_output><empty_output><jupyter_text>Generate topic difficulty table Get questions ids with acc<jupyter_code>df_ques_acc = DF_question_nss_final_tag[DF_question_nss_final_tag['AcceptedAnswerId'].astype(str).str.len() > 0]
print(len(df_ques_acc))
set_ques_acc_ids = set(df_ques_acc['Id'].tolist())
# DF_question_nss_final_tag.dtypes
import statistics

pct_qwo_acc_sum = 0.0
med_hours_sum = 0.0

for topic in MAP_merged_tm_quid:
    quids = MAP_merged_tm_quid[topic]

    time_taken_list = []
    acc_count = 0
    for q in quids:
        q = str(q)
        # print(q)
        # print(type(q))
        if str(q) in set_ques_acc_ids:
            acc_count += 1
            time = get_acc_time(q)
            time_taken_list.append(time)
            # print(time)
            # break
    info = InfoD() 

    info.pct_qwo_acc = round(((len(quids) - acc_count)) * 100.0 / len(quids), 0)
    info.med_hours = statistics.median(time_taken_list)
    pct_qwo_acc_sum += info.pct_qwo_acc
    med_hours_sum += info.med_hours

    Map_topic_diff[topic] = info
    # print(info)

    # break

for topic in MAP_merged_tm_quid:
    info = Map_topic_diff[topic]
    info.pct_qwo_accN = info.pct_qwo_acc * 40.0 / pct_qwo_acc_sum
    info.med_hoursN = info.med_hours * 40.0 / med_hours_sum
    # info.scoreN = info.score * 40.0 / Score_mean_sum
    info.fusedD = round((info.pct_qwo_accN + info.med_hoursN) / 2.0, 2)

for key in sorted(Map_topic_diff, key = lambda topic: Map_topic_diff[topic].fusedD, reverse=True):
    print(Map_topic_diff[key])
save_obj(Map_topic_diff, "Map_topic_diff")
DF_diff = pd.DataFrame()

topics = []
categories = []
fusedDs = []
pct_qwo_acc_list = []
med_hours_list = []




for key in sorted(Map_topic_diff, key = lambda topic: Map_topic_diff[topic].fusedD, reverse=True):
    print(Map_topic_diff[key])
    info = Map_topic_diff[key]
    topics.append(key)
    categories.append(get_high_cat(key))

    fusedDs.append(info.fusedD)
    med_hours_list.append(info.med_hours)
    pct_qwo_acc_list.append(info.pct_qwo_acc)
    # favorites.append(info.favorite)
    # scores.append(info.score)



DF_diff['Topic'] = topics
DF_diff['Category'] = categories
DF_diff['FusedD'] = fusedDs
DF_diff['Med Hrs. to Acc'] = med_hours_list
DF_diff['W/O Acc.'] = pct_qwo_acc_list


# print(DF_diff)
save_obj(DF_diff, "DF_diff")
file_name = os.path.join(ROOT_dir, "output", "Topic_difficulty.csv")
save_df(DF_diff, file_name=file_name)<jupyter_output><empty_output><jupyter_text>Statistical significant<jupyter_code>import scipy.stats as stats

Map_pop = {'view': [], 'favorite': [], 'score': []}
Map_diff = {'pct_wo_acc': [], 'med_hours': []}

for topic in MAP_merged_tm_quid:
    infoP = Map_topic_pop_diff[topic]
    infoD = Map_topic_diff[topic]
    Map_pop['view'].append(infoP.viewN)
    Map_pop['favorite'].append(infoP.favoriteN)
    Map_pop['score'].append(infoP.scoreN)

    Map_diff['pct_wo_acc'].append(infoD.pct_qwo_accN)
    Map_diff['med_hours'].append(infoD.med_hoursN)

for diff in Map_diff:
    for pop in Map_pop:
        X1 = Map_pop[pop]
        X2 = Map_diff[diff]
        tau, p_value = stats.kendalltau(X1, X2)
        issig = False
        if p_value <= 0.05:
            issig = True
        print ("%s => %s: %.2f/%.5f %s" % (pop, diff, tau, p_value, issig))

print("# " * 15)

for diff in Map_diff:
    for pop in Map_pop:
        X1 = Map_pop[pop]
        X2 = Map_diff[diff]
        tau, p_value = stats.kendalltau(X1, X2)
        issig = False
        if p_value <= 0.05:
            issig = True
        print ("%.2f/%.2f & " % ( tau, p_value), end="")
    print()<jupyter_output><empty_output><jupyter_text>Bubble charts<jupyter_code>topics = []
total_questions = []
fusedP = []
fusedD = []
for topic in MAP_merged_tm_quid:
    topics.append(topic)
    total_questions.append(len(MAP_merged_tm_quid[topic]))
    fusedP.append(Map_topic_pop_diff[topic].fusedP)
    fusedD.append(Map_topic_diff[topic].fusedD)

df_topic_bubble = pd.DataFrame()
df_topic_bubble['topics'] = topics
df_topic_bubble['total_questions'] = total_questions
df_topic_bubble['fusedP'] = fusedP
df_topic_bubble['fusedD'] = fusedD
print(df_topic_bubble)
file_name = os.path.join(ROOT_dir, "output", "topic_bubble.csv")
save_df(df_topic_bubble, file_name)
save_obj(df_topic_bubble, "df_topic_bubble")
categories = []
total_questions = []
fusedP = []
fusedD = []
for cat in Map_cat_topic:
    categories.append(cat)
    q_total = 0
    fusedP_total = 0
    fusedD_total = 0
    for tm in Map_cat_topic[cat]:
        # print(tm.name)
        q_total += len(Map_cat_ques[cat])
        fusedP_total += Map_topic_pop_diff[tm.name].fusedP
        fusedD_total += Map_topic_diff[tm.name].fusedD
    total_questions.append(q_total)
    fusedP.append(fusedP_total)
    fusedD.append(fusedD_total)

df_cat_bubble = pd.DataFrame()
df_cat_bubble['Categories'] = categories
df_cat_bubble['total_questions'] = total_questions
df_cat_bubble['fusedP'] = fusedP
df_cat_bubble['fusedD'] = fusedD
print(df_cat_bubble)
file_name = os.path.join(ROOT_dir, "output", "category_bubble.csv")
save_df(df_cat_bubble, file_name)
save_obj(df_cat_bubble, "df_cat_bubble")<jupyter_output><empty_output><jupyter_text>Table of topic stats<jupyter_code>df_topic_stat = pd.DataFrame()
topics = []
num_ques = []
num_ans = []
for topic in MAP_merged_tm_quid:
    topics.append(topic)
    num_ques.append(len(MAP_merged_tm_quid[topic]))
    num_ans.append(len(MAP_merged_tm_ansid[topic]))
    # print(topic)

df_topic_stat['Topic'] = topics
df_topic_stat['#Question'] = num_ques
df_topic_stat['#Answer'] = num_ans

file_name = os.path.join(ROOT_dir, "output", "topic_stats.csv")
save_df(df_topic_stat, file_name)
save_obj(df_topic_stat, "df_topic_stat")
print(len(df_topic_stat))<jupyter_output><empty_output><jupyter_text>RQ types of questions<jupyter_code>File_annotation = os.path.join(ROOT_dir, "dataset", "RQ_annotation.csv")<jupyter_output><empty_output><jupyter_text>Labeling file<jupyter_code>df_rq_annotation = pd.read_csv(File_annotation).fillna("")
print(len(df_rq_annotation))<jupyter_output><empty_output><jupyter_text>Category => type of questions<jupyter_code>def print_map(map_tmp):
    print(map_tmp)
    total = 0
    for qtype in map_tmp:
        total += map_tmp[qtype]

    for qtype in map_tmp:
        pct =  round(map_tmp[qtype] * 100.0 / total, 1)
        print("Type: %s pct: %.1f" % (qtype, pct))
    for qtype in map_tmp:
        pct =  round(map_tmp[qtype] * 100.0 / total, 1)
        print(" %.1f\%% & " % (pct), end="")
    print()

Map_type_to_question = {'How': 0,  "What": 0, "Why": 0, "Others": 0}
Map_cat_to_type = {}
for high_cat in High_cats:
    Map_cat_to_type[high_cat] = 0
    ques_ids = Map_sample[high_cat]
    map_tmp = {'How': 0,  "What": 0, "Why": 0, "Others": 0}
    for quid in ques_ids:
        label = df_rq_annotation[df_rq_annotation['Id'] == quid].iloc[0]
        type1 = label.Type
        type2 = label.Type2
        # print(type1)
        # print(type(type1))
        Map_type_to_question[type1] = Map_type_to_question[type1] + 1
        map_tmp[type1] = map_tmp[type1] + 1
        # print(type2)
        if(len(type2) > 2):
            Map_type_to_question[type2] = Map_type_to_question[type2] + 1
            map_tmp[type2] = map_tmp[type2] + 1

    #     print(label.Type)
    
    print("Category: %s and %s" % (high_cat, map_tmp))
    print_map(map_tmp)
    print("#" * 10)
    # break
# print(Map_type_to_question)

print_map(Map_type_to_question)

# total = 0
# for qtype in Map_type_to_question:
#     total += Map_type_to_question[qtype]

# for qtype in Map_type_to_question:
#     pct =  round(Map_type_to_question[qtype] * 100.0 / total, 0)
#     print("Type: %s pct: %.1f" % (qtype, pct))

# save_obj(Map_question_type, "Map_question_type")<jupyter_output><empty_output><jupyter_text>RQ SDLC<jupyter_code>File_annotation = os.path.join(ROOT_dir, "dataset", "RQ_annotation.csv")
SDLC_phases = ["Requirement Analysis & Planning", "Application Design", "Implementation", "Testing", "Deployment", "Maintenance"]<jupyter_output><empty_output><jupyter_text>Read annotation file<jupyter_code>df_rq_annotation = pd.read_csv(File_annotation).fillna("")
print(len(df_rq_annotation))
df_SDLC = df_rq_annotation[['Id', 'SDLC']]
df_SDLC_type = df_rq_annotation[['Id', 'SDLC', 'Type', 'Type2']]<jupyter_output><empty_output><jupyter_text>SDLC stats<jupyter_code>dist = df_SDLC['SDLC'].value_counts(normalize=True)
print(type(dist))
print(dist)
dist = df_SDLC['SDLC'].value_counts()
print(type(dist))
print(dist)<jupyter_output><empty_output><jupyter_text>SDLC => Type<jupyter_code>def get_stat_SDLC_type(df, label):
    # print(len(df))
    total = len(df)
    s1 = df['Type'].value_counts()
    s2 = df['Type2'].dropna().value_counts()
    how_count = s1['How']
    if "How" in s2:
        how_count += s2['How']
    what_count = s1['What']
    if "What" in s2:
        what_count += s2['What']
    why_count = s1['Why'] 
    if "Why" in s2:
        why_count += s2['Why']
    other_count = s1['Others']
    if "Others" in s2:
        other_count += s2['Others']
    # print(other_count)

    total_pct = round(total * 100.0/471, 0)
    how_pct = round(how_count * 100.0/total, 0)
    what_pct = round(what_count * 100.0/total, 0)
    why_pct = round(why_count * 100.0/total, 0)
    other_pct = round(other_count * 100.0/total, 0)

    print("%s(%.0f\\%%) & %.0f\\%% & %.0f\\%% & %.0f\\%% & %.0f\\%% \\\\" % (label, total_pct, how_pct, what_pct, why_pct, other_pct))


for phase in SDLC_phases:
    df = df_SDLC_type[df_SDLC_type['SDLC'] == phase]
    get_stat_SDLC_type(df, phase)
# print(type(df))
# print(len(df))
# print(df.sample)
# get_stat_SDLC_type(df, "Requirement")<jupyter_output><empty_output><jupyter_text>Category => SDLC<jupyter_code>def get_stat_cat_sdlc(df, label):
    # print(len(df), label)
    counts = df['SDLC'].value_counts()

    print("\\textbf{%s} & \\sixbars" % (label), end="")
    for phase in SDLC_phases:
        val = 0
        if phase in counts:
            val = counts[phase]
        print("{%d}" % val, end="")
    print("  \\\\", end="")

    print()

for high_cat in High_cats:
    Map_cat_to_type[high_cat] = 0
    ques_ids = Map_sample[high_cat]
    df = df_rq_annotation[df_rq_annotation["Id"].isin(ques_ids)]
    # print(len(df))
    get_stat_cat_sdlc(df, high_cat)

    # break
    # map_tmp = {'How': 0,  "What": 0, "Why": 0, "Others": 0}
    # for quid in ques_ids:
    #     label = df_rq_annotation[df_rq_annotation['Id'] == quid].iloc[0]
    #     type1 = label.Type
    #     type2 = label.Type2
    #     # print(type1)
    #     # print(type(type1))
    #     Map_type_to_question[type1] = Map_type_to_question[type1] + 1
    #     map_tmp[type1] = map_tmp[type1] + 1
    #     # print(type2)
    #     if(len(type2) > 2):
    #         Map_type_to_question[type2] = Map_type_to_question[type2] + 1
    #         map_tmp[type2] = map_tmp[type2] + 1

    # #     print(label.Type)
    
    # print("Category: %s and %s" % (high_cat, map_tmp))
    # print_map(map_tmp)
    # print("#" * 10)<jupyter_output><empty_output><jupyter_text>Graph: Significance va Relevance of final tags<jupyter_code>df = pd.read_csv('data/tags_stat.csv')
df.head()
fields = ['tags_relevant', 'tags_recommended_count']
colors = ['#4C72B0', '#55A868']
labels = ['Total relevant', 'Total recomended']

# figure and axis
fig, ax = plt.subplots(1, figsize=(19.20, 10.80))

# plot bars
left = len(df) * [0]
for idx, name in enumerate(fields):
    plt.barh(df.index, df[name], left=left, color=colors[idx], height=0.50)
    left = left + df[name]

# title, legend, labels
plt.legend(labels, loc='upper right', frameon=False, prop={'family': 'serif'})

# remove spines
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)

# Add y-axis labels
ytick_labels = []
for _, df_entry in df.iterrows():
    ytick_labels.append(
        f"\u03bc = {df_entry.significance}, \u03BD = {df_entry.relevance}: {df_entry.tags_relavant_pct}%")
plt.yticks(np.arange(0, len(df.index)), ytick_labels)

# adjust limits and draw grid lines
plt.ylim(-0.5, ax.get_yticks()[-1] + 0.5)
ax.set_axisbelow(True)

plt.savefig('figures/significane_relevance.png', dpi=300)<jupyter_output><empty_output><jupyter_text>Graph Genration  Evolution of top 10 platforms over time<jupyter_code>def plot_dataframe(df, file_name, xlablel="Time"):
    ax = df.drop(columns='All').plot(figsize=(10,6))
    ax.xaxis.label.set_visible(False)
    plt.xlabel(xlablel)
    plt.ylabel('# of Questions')
    labels = [x.strftime("%b, %Y") for x in df.index]
    ax.set_xticks(labels)
    ax.set_xticklabels(labels, rotation=90)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    plt.savefig(file_name, dpi=1000, bbox_inches = "tight")
    plt.show()

def get_initial_df(sampling='3MS'):
    df = all_q_df[['Id', 'CreationDate']]
    df.CreationDate = df.CreationDate.apply(pd.to_datetime)
    df = df.resample(sampling, on="CreationDate").count()[["Id"]]
    df.columns = ['All']
    return df

SAMPLING = '4MS'
DUMMY_DATE = pd.to_datetime('2008-8-1 18:18:37.777')

df = get_initial_df(SAMPLING)

count = 0
for platform_name in top_10_plats_to_qid:
    qids = top_10_plats_to_qid[platform_name]
    # print(platform_name, len(qids))
    df_cat = all_q_df[all_q_df['Id'].astype(int).isin(qids)]
    # print("%s ==========> %d" % (platform_name, len(df_cat)))
    df_cat.CreationDate = df_cat.CreationDate.apply(pd.to_datetime)
    df_cat = df_cat.append(pd.Series(data=[DUMMY_DATE], index = ['CreationDate']), ignore_index=True)
    df_cat = df_cat.resample(SAMPLING, on='CreationDate').count()[["Id"]]
    df_cat.columns = [platform_name]
    df = df.merge(df_cat, how='left', on='CreationDate', validate='one_to_one')

# df.to_csv(file_name, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
plot_dataframe(df, 'top5platform_evolution')<jupyter_output><empty_output><jupyter_text>Category Distribution in top 5 platforms<jupyter_code>cat_to_qid = load_obj('map_cat_to_ques')

top_5_plats_to_qid = {}
for platform_name in sorted(plat_to_qid_df, key=lambda k: len(plat_to_qid_df[k]), reverse=True)[:5]:
    top_5_plats_to_qid[platform_name] = plat_to_qid_df[platform_name]
    
# for key in top_5_plats_to_qid:
#     print(key, len(top_5_plats_to_qid[key]))
platforms = []
categories = []
questions = []

for platform_name in top_5_plats_to_qid:
    for category_name in cat_to_qid:
        intersection = list(set(top_5_plats_to_qid[platform_name]) & set(cat_to_qid[category_name]))
        # print(platform_name, category_name, len(intersection))
        platforms.append(platform_name)
        categories.append(category_name)
        questions.append(len(intersection))
    # print()

plat_to_cat_df = pd.DataFrame()
plat_to_cat_df['Platform'] = platforms
plat_to_cat_df['Category'] = categories
plat_to_cat_df['Count'] = questions

file_name = os.path.join("output", "Plat_To_Cateogory_Distribution.csv")
save_df(plat_to_cat_df, file_name=file_name)

cat_to_qid = load_obj('map_cat_to_ques')

for category_name in cat_to_qid:
    print(category_name, len(cat_to_qid[category_name]))

base_q_df = load_obj('questions')
base_q_df['Id'] = base_q_df['Id'].astype(int)
base_ans_df = load_obj('answers')

def get_acc_time(ques_id):
    question = base_q_df[base_q_df.Id == ques_id].iloc[0]
    # print(type(question))
    question_time = pd.to_datetime(question.CreationDate)
    # print(type(question_time))
    # print("question_time: %s" % (question_time) )
    answer_id = question['AcceptedAnswerId']
    # print(answer_id)
    answer = base_ans_df[base_ans_df.Id == str(answer_id)].iloc[0]
    answer_time = pd.to_datetime(answer.CreationDate)
    # print("answer_time: %s" % (answer_time) )
    time_taken = (answer_time - question_time) / (pd.Timedelta(hours=1))
    # print(time_taken)
    return time_taken

    # print(question_time, answer_time, time_taken)


print(get_acc_time(22940521))

df_ques_acc = base_q_df[base_q_df['AcceptedAnswerId'].astype(str).str.len() > 0]
# print(len(df_ques_acc))
set_ques_acc_ids = set(df_ques_acc['Id'].tolist())<jupyter_output><empty_output><jupyter_text>Overall popularity and diffuculty<jupyter_code>all_q_df = load_obj('questions')
base_ans_df = load_obj('answers')
print("# Posts (questions):", len(all_q_df))

avg_view = round(pd.to_numeric(all_q_df['ViewCount']).dropna().astype(int).mean(), 1)
print("Avg View", avg_view)

avg_fav = round(pd.to_numeric(all_q_df['FavoriteCount']).dropna().astype(int).mean(), 1)
print("Avg Favorite (droped n/a)", avg_fav)
avg_fav = round(pd.to_numeric(all_q_df['FavoriteCount']).fillna(0).astype(int).mean(), 1)
print("Avg Favorite (filling n/a favorits with zero)", avg_fav)

avg_score = round(pd.to_numeric(all_q_df['Score']).dropna().astype(int).mean(), 1)
print("Avg Score", avg_score)

import statistics

time_taken_list = []
acc_count = 0
for _, el in all_q_df.iterrows():
        if int(el['Id']) in set_ques_acc_ids:
                acc_count += 1
                time = get_acc_time(int(el['Id']))
                time_taken_list.append(time)


# print(acc_count)
pct_qwo_acc = round(((len(all_q_df) - acc_count)) * 100.0 / len(all_q_df), 0)
print("% W/o Acct Ans", pct_qwo_acc)
med_hours = statistics.median(time_taken_list)
print("Med Hrs to Acc.", med_hours)<jupyter_output><empty_output><jupyter_text>Comparison of Question Types Between Different Domains<jupyter_code>rq_df = pd.read_csv('data/RQ_annotation.csv')
ss = '''
{'How': 271, 'What': 87, 'Why': 68, 'Others': 61}
Type: How pct: 55.6
Type: What pct: 17.9
Type: Why pct: 14.0
Type: Others pct: 12.5
'''
print(ss)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1393973489942095, 'fraction_numerical': 0.01048327341584566, 'mean_word_length': 4.772136860446698, 'pattern_counts': {'":': 20, '<': 275, '<?xml version=': 0, '>': 294, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '65', 'dir': 'jupyter-structured-clean-dedup', 'id': '504982', 'license': 'MIT', 'n_tokens_mistral': 28935, 'n_tokens_neox': 27777, 'n_words': 4345, 'path': 'Codes/LCSD_Challenges_EMSE.ipynb', 'repo_name': 'al-alamin/LCSD_challenge_EMSE', 'size': '72878'}