<jupyter_start><jupyter_text>Use pyquery-ql.pySend a graphql query to GitHuband pretty print output.Supports Python 3.6+<jupyter_code>import json
import os
import pprint

import requests
# get api token and set authorization
api_token = os.environ['GITHUB_API_TOKEN']
headers = {'Authorization': f'token {api_token}'}
# set url to a graphql endpoint
url = 'https://api.github.com/graphql'
# add a json query
query = """
{
  organization(login: "jupyterhub") {
    repositories(first: 30) {
      totalCount
      edges {
        node {
          name
          url
          issues(states: OPEN) {
            totalCount
          }
          pullRequests(states: OPEN) {
            totalCount
          }
        }
      }
    }
  }
}
"""
# submit the request
r = requests.post(url=url, json={'query': query}, headers=headers)
r.text
data = json.loads(r.text)
import requests
import json
from itertools import zip_longest
import pandas as pd
import numpy as np
#This is a little helper funciton that will take x of something, and let you chunk through a subset
#Note that 'n' is the chunk size
def _grouper(n, iterable, fillvalue=None):
    args = [iter(iterable)] * n
    return zip_longest(fillvalue=fillvalue, *args)
#Since you want a report by month, lets just create a function to get all the chat IDs for a two digit month and 4 digit year

def get_ids(data):
    all_ids = []
    response = json.loads(data)
    for id in response['data']:
        all_ids.append('id')
    return all_ids
all_ids = []
for id in data['data']:
    print(id)
    all_ids.append(id)
    
    for id in data['data'][all_ids[0]]:
        print(id)
        all_ids.append(id)
        print(all_ids)
        
    for id in data['data'][all_ids[0]][all_ids[1]]:
        print(id)
        all_ids.append(id)
    
    print(all_ids)
    print(data['data']['edges'])

def build_csv_data(queried_id_list):
  all_chats_for_csv = []
  for chunk in _grouper(50, queried_id_list):
    filtered = [id for id in chunk if id is not None] #Only needed for last set of 50 that will have some sort of remainder filled with 'None' values by the grouper() function
    url = "{0}/chats".format(config['base_path'])
    stringified_ids = ",".join(filtered)
    params = {"ids":stringified_ids}
    r = s.get(url, params=params)
    if r.status_code == 200:
      response = json.loads(r.text)
      docs = response['docs']
      for k in docs:
        doc_obj = docs[k] #This is the 'doc' object. Note that some keys may not exist for all items. Duration does not exist for offline messages for example
        all_chats_for_csv.append(doc_obj)
    else:
      print("error getting bulk chats")
  return all_chats_for_csv
#grab the ids for the specified range NOTE!! This is where you can change the year/month for the date range.
queried_id_list = get_chat_ids_for_month(2016, 10)

#turn those into raw objects
csv_obj_data = build_csv_data(queried_id_list)

#create a list to hold our rows
all_rows_as_obj = []

#turn the raw objects into nice and tasty ones that pandas can digest
for record in csv_obj_data:
  try:
    response_time = record.get('response_time', {})
    count = record.get('count',{})
    csv_obj = {
      "id": record['id'],
      "agent": ",".join(record.get('agent_names', "N/A")),
      "visitor": record['visitor']['name'],
      "department": record['department_name'],
      "url": record.get('webpath', "N/A"),
      "missed": record.get('missed', "N/A"),
      "resp_first": response_time.get('first', []),
      "resp_max": response_time.get('max', []),
      "resp_avg": response_time.get('avg', []),
      "start timestamp": record['session']['start_date'],
      "end timestamp": record['session']['end_date'],
      "total messages": count.get('total', 0),
      "Agent Msg Count": count.get('agent', 0),
      "Visitor Msg Count": count.get('visitor', 0),
      "rating": record.get('rating',[]),
      "ticket_id": record['zendesk_ticket_id']
    }
  except KeyError as e:
    print(e)
    print("key error occured for record with id: {}".format(record['id']))
    print(record)
    pass
  all_rows_as_obj.append(csv_obj)

#with prettier objects our column names can just come from a call to the first object's keys
col_keys = all_rows_as_obj[0].keys()

#create the dataframe, list comprehension creates an array of rows based on the column names we got
#Let pandas do the hard part with csvs
csv_frame = pd.DataFrame([[i[j] for j in col_keys] for i in all_rows_as_obj], columns=col_keys)

#coerce ticket_id to be int not float
csv_frame['ticket_id'] = csv_frame['ticket_id'].fillna(0).astype(np.int64)

#fill blank values with the string 'N/A'
csv_frame = csv_frame.fillna("N/A")

#output to a file
csv_frame.to_csv('october_stats.csv')<jupyter_output><empty_output><jupyter_text>Walk the JSON response contents<jupyter_code>issue_data = data['data']
org = issue_data['organization']
repos = org['repositories']
edges = repos['edges']
# edges[0]['node']
# print(edges[0]['node']['name'] + '---' + edges[0]['node']['url'])
# for edge in edges:
#    pprint.pprint(edge)<jupyter_output><empty_output><jupyter_text>Bring into pandas<jupyter_code>import pandas as pd
df = pd.DataFrame.from_records(edges)
df.columns
# df['node']<jupyter_output><empty_output><jupyter_text>Generate basic report of total open issues<jupyter_code>print(f"{'Repo':30} {'OpenIssues':11} {'OpenPRs':9} {'URL'}")

line = 26
while line > 0:
    lineout = f"{df['node'][line]['name']:30} {(df['node'][line]['issues']['totalCount']):8} {(df['node'][line]['pullRequests']['totalCount']):8}     {df['node'][line]['url']}"
    print(lineout)
    line -= 1
df.dtypes
df.head()
df.index
df.columns
df.values
df.sort_index
# output data to a csv
# df.to_csv('issue_data.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13788324961025464, 'fraction_numerical': 0.007794907327212887, 'mean_word_length': 3.897370653095844, 'pattern_counts': {'":': 17, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '62197', 'license': 'MIT', 'n_tokens_mistral': 1987, 'n_tokens_neox': 1938, 'n_words': 540, 'path': 'notebooks/use-pyquery-ql-Copy1.ipynb', 'repo_name': 'willingc/pyquery-ql', 'size': '5773'}