<commit_before>#!/usr/bin/env python3

from __future__ import print_function
import json
import os
import sys
from urllib.request import Request, urlopen

sys.path.append(
  os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))

from lib.util import store_artifact, scoped_cwd, safe_mkdir, get_out_dir, \
  ELECTRON_DIR

OUT_DIR     = get_out_dir()

BASE_URL = 'https://electron-metadumper.herokuapp.com/?version='

version = sys.argv[1]
authToken = os.getenv('META_DUMPER_AUTH_HEADER')

def is_json(myjson):
  try:
    json.loads(myjson)
  except ValueError:
    return False
  return True

def get_content(retry_count = 5):
  try:
    request = Request(
      BASE_URL + version,
      headers={"Authorization" : authToken}
    )

    proposed_content = urlopen(
      request
    ).read()

    if is_json(proposed_content):
      return proposed_content
    print("bad attempt")
    raise Exception("Failed to fetch valid JSON from the metadumper service")
  except Exception as e:
    if retry_count == 0:
      raise e
    return get_content(retry_count - 1)

def main():
  if not authToken or authToken == "":
    raise Exception("Please set META_DUMPER_AUTH_HEADER")
  # Upload the index.json.
  with scoped_cwd(ELECTRON_DIR):
    safe_mkdir(OUT_DIR)
    index_json = os.path.relpath(os.path.join(OUT_DIR, 'index.json'))

    new_content = get_content()

    with open(index_json, "wb") as f:
      f.write(new_content)

    store_artifact(OUT_DIR, 'headers/dist/', [index_json])


if __name__ == '__main__':
  sys.exit(main())<commit_msg>build: remove ending slash upload-index-json (#34125)<commit_after>#!/usr/bin/env python3

from __future__ import print_function
import json
import os
import sys
from urllib.request import Request, urlopen

sys.path.append(
  os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))

from lib.util import store_artifact, scoped_cwd, safe_mkdir, get_out_dir, \
  ELECTRON_DIR

OUT_DIR     = get_out_dir()

BASE_URL = 'https://electron-metadumper.herokuapp.com/?version='

version = sys.argv[1]
authToken = os.getenv('META_DUMPER_AUTH_HEADER')

def is_json(myjson):
  try:
    json.loads(myjson)
  except ValueError:
    return False
  return True

def get_content(retry_count = 5):
  try:
    request = Request(
      BASE_URL + version,
      headers={"Authorization" : authToken}
    )

    proposed_content = urlopen(
      request
    ).read()

    if is_json(proposed_content):
      return proposed_content
    print("bad attempt")
    raise Exception("Failed to fetch valid JSON from the metadumper service")
  except Exception as e:
    if retry_count == 0:
      raise e
    return get_content(retry_count - 1)

def main():
  if not authToken or authToken == "":
    raise Exception("Please set META_DUMPER_AUTH_HEADER")
  # Upload the index.json.
  with scoped_cwd(ELECTRON_DIR):
    safe_mkdir(OUT_DIR)
    index_json = os.path.relpath(os.path.join(OUT_DIR, 'index.json'))

    new_content = get_content()

    with open(index_json, "wb") as f:
      f.write(new_content)

    store_artifact(OUT_DIR, 'headers/dist', [index_json])


if __name__ == '__main__':
  sys.exit(main())
{'fraction_non_alphanumeric': 0.11461951373539628, 'fraction_numerical': 0.00473634354278497, 'mean_word_length': 3.7496251874062967, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2881190', 'n_tokens_mistral': 1130, 'n_tokens_neox': 1095, 'n_words': 248}