<commit_before>#!/usr/bin/env python3

import re
import os
import sys
from glob import iglob
import urllib.parse
import html.parser
import itertools

import inflection
from bs4 import BeautifulSoup

# Unless you're really going to hack on the API generator, don't hammer
# the developer site needlessly. Currently, mirroring the whole site
# results in about 26MB on disk. I may eventually do some more intelligent
# scraping, downloading, and disk caching to avoid this wget.

# That said, here is my usual process:
# $ wget -mk https://developer.zendesk.com/rest_api/docs/core/introduction
# $ cp -R developer.zendesk.com developer.zendesk.com.orig
# $ ./clean.py
# $ cp -R developer.zendesk.com developer.zendesk.com.cleaned
#
# For each patch.* file
# cd developer.zendesk.com
# $ patch -p1 < ../patch.file
#
# $ cd ..
# $ ./api_gen.py

# If I find that more patching is needed to get what I want, then:
# $ vim edit whatever
# $ diff -r -u developer.zendesk.com developer.zendesk.com.cleaned > api_gen.patch.new
# If that all checks out, then api_gen.patch.new becomes the new api_gen.patch

# Then, once all is well:
# cp zdesk_api.py ../zdesk/

html_parser = html.parser.HTMLParser()

skip_files=[
    'introduction',
    'changes_roadmap',
    'side_loading',
    ]

api_actions = [
    'autocomplete',
    'clone',
    'compact',
    'create',
    'destroy',
    'detect',
    'execute',
    'exports',
    'import',
    'imports',
    'make',
    'mark',
    'me',
    'merge',
    'preview',
    'recover',
    'redact',
    'reorder',
    'search',
    'show',
    'verify',
    ]

with open('api_template.py', 'r') as template_file:
    template = template_file.read()

#os.chdir('old.developer.zendesk.com/documentation/rest_api')
#os.chdir('developer.zendesk.com/rest_api/docs/core')

api_items = {}
duplicate_api_items = {}
for doc_file in iglob(os.path.join('developer.zendesk.com', 'rest_api', 'docs', '*', '*')):
    if '.html' in doc_file or '.orig' in doc_file or os.path.split(doc_file)[1] in skip_files:
        continue

    with open(doc_file, 'r') as doc:
        soup = BeautifulSoup(doc)

    for code in soup.find_all(['code', 'pre']):
        text = code.get_text()<commit_msg>add 'update' to api_gen api_actions<commit_after>#!/usr/bin/env python3

import re
import os
import sys
from glob import iglob
import urllib.parse
import html.parser
import itertools

import inflection
from bs4 import BeautifulSoup

# Unless you're really going to hack on the API generator, don't hammer
# the developer site needlessly. Currently, mirroring the whole site
# results in about 26MB on disk. I may eventually do some more intelligent
# scraping, downloading, and disk caching to avoid this wget.

# That said, here is my usual process:
# $ wget -mk https://developer.zendesk.com/rest_api/docs/core/introduction
# $ cp -R developer.zendesk.com developer.zendesk.com.orig
# $ ./clean.py
# $ cp -R developer.zendesk.com developer.zendesk.com.cleaned
#
# For each patch.* file
# cd developer.zendesk.com
# $ patch -p1 < ../patch.file
#
# $ cd ..
# $ ./api_gen.py

# If I find that more patching is needed to get what I want, then:
# $ vim edit whatever
# $ diff -r -u developer.zendesk.com developer.zendesk.com.cleaned > api_gen.patch.new
# If that all checks out, then api_gen.patch.new becomes the new api_gen.patch

# Then, once all is well:
# cp zdesk_api.py ../zdesk/

html_parser = html.parser.HTMLParser()

skip_files=[
    'introduction',
    'changes_roadmap',
    'side_loading',
    ]

api_actions = [
    'autocomplete',
    'clone',
    'compact',
    'create',
    'destroy',
    'detect',
    'execute',
    'exports',
    'import',
    'imports',
    'make',
    'mark',
    'me',
    'merge',
    'preview',
    'recover',
    'redact',
    'reorder',
    'search',
    'show',
    'update',
    'verify',
    ]

with open('api_template.py', 'r') as template_file:
    template = template_file.read()

#os.chdir('old.developer.zendesk.com/documentation/rest_api')
#os.chdir('developer.zendesk.com/rest_api/docs/core')

api_items = {}
duplicate_api_items = {}
for doc_file in iglob(os.path.join('developer.zendesk.com', 'rest_api', 'docs', '*', '*')):
    if '.html' in doc_file or '.orig' in doc_file or os.path.split(doc_file)[1] in skip_files:
        continue

    with open(doc_file, 'r') as doc:
        soup = BeautifulSoup(doc)

    for code in soup.find_all(['code', 'pre']):
        text = code.get_text()
{'fraction_non_alphanumeric': 0.13640449438202248, 'fraction_numerical': 0.002696629213483146, 'mean_word_length': 3.3722986247544204, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7441952', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1533, 'n_words': 426}