<commit_before>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package commands

import (
	"cocoon/db"
	"encoding/json"
	"io/ioutil"

	"golang.org/x/net/context"

	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

// RefreshGithubCommitsResult pulls down the latest GitHub commit data and
// generates checklists for the bots to run through.
type RefreshGithubCommitsResult struct {
	Results []CommitSyncResult
}

// CommitSyncResult describes what happened to a specific commit during the
// sync.
type CommitSyncResult struct {
	Commit  string
	Outcome string
}

// RefreshGithubCommits returns the information about the latest GitHub commits.
func RefreshGithubCommits(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error) {
	httpClient := urlfetch.Client(cocoon.Ctx)

	// Fetch data from GitHub
	githubResp, err := httpClient.Get("https://api.github.com/repos/flutter/flutter/commits")

	if err != nil {
		return nil, err
	}

	defer githubResp.Body.Close()
	commitData, err := ioutil.ReadAll(githubResp.Body)

	if err != nil {
		return nil, err
	}<commit_msg>skip bad github responses (#48)<commit_after>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package commands

import (
	"cocoon/db"
	"encoding/json"
	"fmt"
	"io/ioutil"

	"golang.org/x/net/context"

	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

// RefreshGithubCommitsResult pulls down the latest GitHub commit data and
// generates checklists for the bots to run through.
type RefreshGithubCommitsResult struct {
	Results []CommitSyncResult
}

// CommitSyncResult describes what happened to a specific commit during the
// sync.
type CommitSyncResult struct {
	Commit  string
	Outcome string
}

// RefreshGithubCommits returns the information about the latest GitHub commits.
func RefreshGithubCommits(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error) {
	httpClient := urlfetch.Client(cocoon.Ctx)

	// Fetch data from GitHub
	githubResp, err := httpClient.Get("https://api.github.com/repos/flutter/flutter/commits")

	if err != nil {
		return nil, err
	}

	if githubResp.StatusCode != 200 {
		return nil, fmt.Errorf("GitHub API responded with a non-200 HTTP status: %v", githubResp.StatusCode)
	}

	defer githubResp.Body.Close()
	commitData, err := ioutil.ReadAll(githubResp.Body)

	if err != nil {
		return nil, err
	}
{'fraction_non_alphanumeric': 0.10331965684446102, 'fraction_numerical': 0.005967922417008579, 'mean_word_length': 4.23828125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6593461', 'n_tokens_mistral': 919, 'n_tokens_neox': 843, 'n_words': 262}