<jupyter_start><jupyter_text>Kubernetes activity reflected in GitHub act|ivityPlot metrics related to GitHub repo activity over time, including number of starred repos, commits etc.Notebook is meant to be *very* portable - no use of distributed FS or processing.<jupyter_code># All imports required for the notebook - use conda or pip to install missing libs
import requests
import re
import pandas as pd
import sqlite3
import matplotlib.pyplot as plot
# Define functions to be used for plotting

def subset(org_and_repo):
    org, repo = org_and_repo.split('/')
    return data.loc[(data.organization == org) & (data.repository == repo)]

def merge_repos(repos, metric='stargazers'):
    data = pd.DataFrame(columns=['Date'])
    for repo in repos:
        repo_data = subset(repo)
        repo_data = repo_data[['date', metric]].rename(columns={metric: repo, 'date': 'Date'})
        data = data.merge(repo_data, on='Date', how='outer')
    return data.sort_values(by='Date')

def plot_repos(title, repos, metric='stargazers'):
    plot_data = merge_repos(repos, metric)
    plot_data.plot(x='Date', y=repos, title=title, figsize=(15,10))

def plot_all_repos(title, org, metric='stargazers'):
    repo_data = data.loc[data.organization == org]
    repos = [org + '/' + repo for repo in repo_data.repository.unique()]
    plot_repos(title, repos, metric)<jupyter_output><empty_output><jupyter_text>Load data from last survey<jupyter_code>conn = sqlite3.connect('../../data/github.db')
data = pd.read_sql_query("""
SELECT strftime('%Y-%m-01', date) AS date, organization, repository,
MAX(stargazers) AS stargazers, MAX(forks) AS forks
FROM github_stats
GROUP BY organization, repository, strftime('%Y-%m', date)
""", conn)<jupyter_output><empty_output><jupyter_text>Gather the latest in search keywords"Awesome Operators" is a community-maintained list of K8s operators> Operators are Kubernetes native applications. We define native as being both managed using the Kubernetes APIs via kubectl and ran on Kubernetes as containers. Operators take advantage of Kubernetes’s extensibility to deliver the automation advantages of cloud services like provisioning, scaling, and backup/restore while being able to run anywhere that Kubernetes can run.Let's grab the latest<jupyter_code>f =open('../../data/awesome_operators.txt', 'w')
awesome_re = re.compile("\[([^\]]+/[^\]]+)\]")
r = requests.get('https://raw.githubusercontent.com/operator-framework/awesome-operators/master/README.md')
operators = awesome_re.findall(r.text)

for operator in operators:
    f.write(operator + '\n')
f.close()<jupyter_output><empty_output><jupyter_text>Top repos by stars (number of stargazers)<jupyter_code>awesome_repo_names = [l.rstrip('\n') for l in open('../../data/awesome_operators.txt').readlines()]
def name_and_stars(name):
    awesome_data = subset(name)
    max_stars = awesome_data['stargazers'].max()
    return (name, max_stars)
awesome_data = pd.DataFrame(map(name_and_stars, awesome_repo_names), columns=['repository', 'stargazers']).dropna()

n = 10
top_n_operators = awesome_data.sort_values(by='stargazers', ascending=False).head(n)
top_n_operators<jupyter_output><empty_output><jupyter_text>Top stargazers visualized<jupyter_code>top_n_operators.plot(x='repository', y='stargazers', kind='bar', figsize=(20,10), title='Github stars for operators listed in awesome-operators')<jupyter_output><empty_output><jupyter_text>Analytics over timeThe following plots show various categories of activity over time<jupyter_code>repos = [
    'tensorflow/tensorflow',
    'apache/incubator-mxnet',
    'Microsoft/CNTK',
    'BVLC/caffe',
    'keras-team/keras',
    'Theano/Theano',
    'amzn/amazon-dsstne',
    'pytorch/pytorch'
]
plot_repos('Machine Learning', repos)
repos = [
    'kubeflow/kubeflow',
    'tensorflow/tfx'
]
plot_repos('Machine Learning Platforms', repos)
repos = [
    'apache/spark',
    'apache/ignite',
    'apache/flink',
    'apache/beam',
    'apache/storm',
    'apache/samza'
]
plot_repos('Big Data', repos)
repos = [
    'fnproject/fn',
    'openfaas/faas',
    'apache/incubator-openwhisk',
    'fission/fission',
    'apex/apex',
    'serverless/serverless',
    'kubeless/kubeless',
    'knative/serving'
]

plot_repos('Functions as a Service', repos)
repos = [
    'mesosphere/marathon',
    'hashicorp/nomad',
    'kubernetes/kubernetes',
    'docker/swarm',
    'docker/swarmkit'
]

plot_repos('Container Orchestration', repos)
repos = [
    'jupyter/notebook',
    'apache/zeppelin',
    'jupyterhub/jupyterhub'
]

plot_repos('Data Science Notebooks', repos)
plot_all_repos('Kubernetes', 'kubernetes')
plot_all_repos('Operator Framework', 'operator-framework')
repos = [
    'operator-framework/operator-sdk',
    'kubernetes-sigs/kubebuilder',
    'kubernetes-sigs/controller-runtime',
    'kudobuilder/kudo',
    'bluek8s/kubedirector',
    'kubedb/operator',
    'rook/operator-kit',
    'GoogleCloudPlatform/metacontroller',
    'zalando-incubator/kopf'
]

plot_repos('Operator Tools', repos)
plot_all_repos('Kubernetes SIGs', 'kubernetes-sigs')
most_stars = data.groupby(['organization', 'repository']).max().sort_values(['stargazers'])
most_stars
plot_all_repos('Gardener', 'gardener')
repos = [
    'istio/istio',
    'linkerd/linkerd2'
]
plot_repos('Service Mesh', repos)
repos = [
    'operator-framework/operator-sdk',
    'kubernetes-sigs/kubebuilder',
    'kudobuilder/kudo',
    'bluek8s/kubedirector',
    'kubedb/operator',
    'rook/operator-kit',
    'GoogleCloudPlatform/metacontroller',
    'zalando-incubator/kopf'
]

plot_data = merge_repos(repos, 'forks')
plot_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13766832034018428, 'fraction_numerical': 0.0031892274982282067, 'mean_word_length': 5.1626637554585155, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '119009', 'license': 'Apache-2.0', 'n_tokens_mistral': 1998, 'n_tokens_neox': 1898, 'n_words': 402, 'path': 'notebooks/k8s-analytics/github-stats.ipynb', 'repo_name': 'gregoryg/cdsw-ds-for-telco', 'size': '5644'}