<jupyter_start><jupyter_text>Code for Drawing All the Plots  Load CSV Data<jupyter_code>import csv

csvData = []

with open('result/repo_info_selected_edited.csv') as csvFile:
    csvReader = csv.DictReader(csvFile)
    for row in csvReader:
        csvData.append(row)<jupyter_output><empty_output><jupyter_text>Load JSON Data<jupyter_code>import json

jsonData = []

with open('temp/repo_info_selected.json') as jsonFile:
    jsonData = json.load(jsonFile)<jupyter_output><empty_output><jupyter_text>Initialize Matplotlib for Producing Figures in Latex<jupyter_code>import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42<jupyter_output><empty_output><jupyter_text>Distribution of Comment Density for Different Language<jupyter_code>import matplotlib.pyplot as plt

langs = ['Python', 'Java', 'C++', 'JavaScript', 'Go']
dist = {}
for lang in langs:
    dist[lang] = [float(row['comment_ratio']) for row in csvData if row['language'] == lang]
plt.hist(dist['JavaScript'], 10)
plt.show()<jupyter_output><empty_output><jupyter_text>Average comment density of different language<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

def extract(lang, key):
    return np.array([float(row[key]) for row in csvData if row['language'] == lang])

langs = ['Python', 'Java', 'C++', 'JavaScript', 'Go']
comment_densities = [extract(lang, 'comment_ratio') for lang in langs]
avgs = [np.average(a) for a in comment_densities]
medians = [np.median(a) for a in comment_densities]
variances = [np.var(a) for a in comment_densities]
avgs
medians
variances
from scipy import stats
# Conduct Wilcoxon rank-sum test for all languages
pvalues = []
for i in range(len(langs)):
    row = []
    for j in range(len(langs)):
        lang1 = langs[i]
        lang2 = langs[j]
        statistics, pvalue = stats.ranksums(comment_densities[i], comment_densities[j])
        row.append(pvalue)
    pvalues.append(row)
pvalues
plt.figure(figsize=(4,3))
plt.bar([1,6,11,16,21], avgs, width=2, align='center', label='Average')
plt.bar([3,8,13,18,23], medians, width=2, align='center', label='Median')
plt.xticks([2,7,12,17,22], langs)
plt.xlabel('Programming Language')
plt.ylabel('Comment Density')
plt.legend()
# plt.title('Comment Density in Different Languages')
plt.savefig('result/figures/cd_by_lang.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Difference in Comment Density for Different Type of Projects<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

def extract(lang, type_):
    return [float(row['comment_ratio']) 
         for row in csvData 
         if row['language'] == lang and row['type'] == type_]

types = ['tutorial', 'framework', 'library', 'app']
cd_java = [np.array(extract('Java', t)) for t in types];
cd_js = [np.array(extract('JavaScript', t)) for t in types];
avgs_java = [np.average(a) for a in cd_java]
avgs_js = [np.average(a) for a in cd_js]

tmp = extract('JavaScript', 'framework')
tmp.extend(extract('JavaScript', 'library'))
np.mean(tmp)
avgs_js
from scipy import stats
# Conduct Wilcoxon rank-sum test for Java
pvalues = []
for i in range(len(types)):
    row = []
    for j in range(len(types)):
        type1 = types[i]
        type2 = types[j]
        statistics, pvalue = stats.ranksums(cd_java[i], cd_java[j])
        row.append(pvalue)
    pvalues.append(row)
pvalues
plt.bar([i for i in range(0, 4)], avgs_js, width=0.4, label="JavaScript")
plt.bar([i+0.4 for i in range(0, 4)], avgs_java, width=0.4, label="Java")
plt.xticks([i+0.2 for i in range(0, 4)], types)
plt.title('Average Comment Density in Different Type of Projects')
plt.xlabel('Project Type')
plt.ylabel('Comment Density')
plt.legend()
plt.savefig('result/figures/avg_cd_java_js.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Relationship between Comment Density and Number of Contributors<jupyter_code>import matplotlib.pyplot as plt

langs = ['Python', 'Java', 'C++', 'JavaScript', 'Go']
comment_densities = [[float(row['comment_ratio']) 
                          for row in csvData if row['language'] == lang] 
                     for lang in langs]
contributors = [[float(row['contributors']) 
                     for row in csvData if row['language'] == lang] 
                for lang in langs]

markers = ['o', '^', '8', 's', '*']
plt.figure(figsize=(4,3))
for i in range(len(langs)):
    plt.scatter(contributors[i], comment_densities[i], label=langs[i], marker=markers[i])
plt.legend()
plt.xlabel('Contributors')
plt.ylabel('Comment Density')
# plt.title('Relationship between Contributors and Comment Density')
plt.savefig('result/figures/cd_contrb.pdf', bbox_inches='tight')
plt.show()
langs = ['Python', 'Java', 'C++', 'JavaScript', 'Go']
comment_densities = [[float(row['comment_ratio']) 
                          for row in csvData if row['language'] == lang] 
                     for lang in langs]
core_contributors = {
    'Python': [],
    'Java': [],
    'C++': [],
    'JavaScript': [],
    'Go': []
}

for row in csvData:
    for data in jsonData:
        if data['full_name'] != row['full_name']:
            continue
        core_contrib = 0
        contributor_list = data['contributors']
        leader_contribution = contributor_list[0]['contributions']
        for c in contributor_list:
            if c['contributions'] > 1 and c['contributions'] > 0.05 * leader_contribution:
                core_contrib += 1
        core_contributors[row['language']].append(core_contrib)
   
markers = ['o', '^', '8', 's', '*']
for i in range(len(langs)):
    plt.scatter(core_contributors[langs[i]], comment_densities[i], s=15, label=langs[i], marker=markers[i])
plt.legend()
plt.xlabel('Core Contributors')
plt.ylabel('Comment Density')
plt.title('Relationship between Core Contributors and Comment Density')
plt.savefig('result/figures/cd_core_contrib.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Relationship between Project Size and Comment Density<jupyter_code>import matplotlib.pyplot as plt
import math

langs = ['Python', 'Java', 'C++', 'JavaScript', 'Go']
comment_densities = [[float(row['comment_ratio']) 
                          for row in csvData if row['language'] == lang] 
                     for lang in langs]
lines_of_code = [[float(row['lines_code']) 
                     for row in csvData if row['language'] == lang] 
                for lang in langs]
log_loc = [[math.log(i+1) for i in a] for a in lines_of_code]

markers = ['o', '^', '8', 's', '*']
for i in range(len(langs)):
    plt.scatter(log_loc[i], comment_densities[i], label=langs[i], marker=markers[i])
plt.legend()
plt.xlabel('Lines of Code(in log)')
plt.ylabel('Comment Density')
plt.title('Relationship between LOC and Comment Density')
plt.savefig('result/figures/cd_loc.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15127388535031847, 'fraction_numerical': 0.009698899826288361, 'mean_word_length': 4.4964200477326965, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '226640', 'license': 'MIT', 'n_tokens_mistral': 2499, 'n_tokens_neox': 2324, 'n_words': 565, 'path': 'ESEC:FSE19-SRC/visualization.ipynb', 'repo_name': 'hehao98/CommentAnalysis', 'size': '6908'}