<commit_before>package scraper

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sort"
	"time"

	"github.com/EVE-Tools/emdr-to-nsq/lib/emds"
	"github.com/EVE-Tools/market-streamer/lib/locations/citadels"
	"github.com/EVE-Tools/market-streamer/lib/locations/locationCache"
	"github.com/EVE-Tools/market-streamer/lib/marketTypes"
	"github.com/Sirupsen/logrus"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/v1"
	"github.com/klauspost/compress/zlib"
	"golang.org/x/net/context"
)

type esiOrder goesiv1.GetMarketsRegionIdOrders200Ok

var esiClient goesi.APIClient
var esiPublicContext context.Context

// Initialize initializes the scraper
func Initialize(clientID string, secretKey string, refreshToken string) {
	httpClient := &http.Client{
		Timeout: time.Duration(time.Second * 10),
	}

	// Requests to citadel's markets are authenticated - we're just using a default key for retrieving public markets<commit_msg>Switch scraper to fork, too<commit_after>package scraper

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sort"
	"time"

	"github.com/EVE-Tools/emdr-to-nsq/lib/emds"
	"github.com/EVE-Tools/goesi"
	"github.com/EVE-Tools/goesi/v1"
	"github.com/EVE-Tools/market-streamer/lib/locations/citadels"
	"github.com/EVE-Tools/market-streamer/lib/locations/locationCache"
	"github.com/EVE-Tools/market-streamer/lib/marketTypes"
	"github.com/Sirupsen/logrus"
	"github.com/klauspost/compress/zlib"
	"golang.org/x/net/context"
)

type esiOrder goesiv1.GetMarketsRegionIdOrders200Ok

var esiClient goesi.APIClient
var esiPublicContext context.Context

// Initialize initializes the scraper
func Initialize(clientID string, secretKey string, refreshToken string) {
	httpClient := &http.Client{
		Timeout: time.Duration(time.Second * 10),
	}

	// Requests to citadel's markets are authenticated - we're just using a default key for retrieving public markets
{'fraction_non_alphanumeric': 0.12338362068965517, 'fraction_numerical': 0.007543103448275862, 'mean_word_length': 4.895238095238096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269300', 'n_tokens_mistral': 711, 'n_tokens_neox': 684, 'n_words': 121}