<commit_before>package main

import (
	"math/rand"
	"sort"
	"strconv"
	"time"
)

var text string = "Attend to hear 6 stellar #mobile #startups at #OF12 Entrepreneur Idol show 2day,  " +
"http://t.co/HtzEMgAC @TiEcon @sv_entrepreneur @500!"

func contains(s []int, e int) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

type Entity struct {
	start int
	end int
	html string
}

func (e Entity) Less()

func testEntities() map[Entity]bool {
	entities := make(map[Entity]bool)
	entities[Entity{25, 32, "<#mobile>"}] = true
	entities[Entity{33, 42, "<#startups>"}] = true
	entities[Entity{46, 51, "<#OF12>"}] = true
	entities[Entity{82, 102, "<http://t.co/HtzEMgAC>"}] = true
	entities[Entity{103, 110, "<@TiEcon>"}] = true
	entities[Entity{111, 127, "<@sv_entrepreneur>"}] = true
	entities[Entity{128, 132, "<@500>"}] = true
	return entities
}


func createEntityList() []map[Entity]bool {
	length := len(text)
	entitiesList := make([]map[Entity]bool, 1000)
	for i := 0; i < 1000; i++ {
		entities := make(map[Entity]bool)
		total := rand.Intn(10)
		indices := make([]int, total * 2)
		for j := 0; j < total * 2; j++ {
			var next int
			for {<commit_msg>accidentally checked in something in progress<commit_after>package main

import (
	"math/rand"
	"sort"
	"strconv"
	"time"
)

var text string = "Attend to hear 6 stellar #mobile #startups at #OF12 Entrepreneur Idol show 2day,  " +
"http://t.co/HtzEMgAC @TiEcon @sv_entrepreneur @500!"

func contains(s []int, e int) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

type Entity struct {
	start int
	end int
	html string
}

func testEntities() map[Entity]bool {
	entities := make(map[Entity]bool)
	entities[Entity{25, 32, "<#mobile>"}] = true
	entities[Entity{33, 42, "<#startups>"}] = true
	entities[Entity{46, 51, "<#OF12>"}] = true
	entities[Entity{82, 102, "<http://t.co/HtzEMgAC>"}] = true
	entities[Entity{103, 110, "<@TiEcon>"}] = true
	entities[Entity{111, 127, "<@sv_entrepreneur>"}] = true
	entities[Entity{128, 132, "<@500>"}] = true
	return entities
}


func createEntityList() []map[Entity]bool {
	length := len(text)
	entitiesList := make([]map[Entity]bool, 1000)
	for i := 0; i < 1000; i++ {
		entities := make(map[Entity]bool)
		total := rand.Intn(10)
		indices := make([]int, total * 2)
		for j := 0; j < total * 2; j++ {
			var next int
			for {
{'fraction_non_alphanumeric': 0.17647058823529413, 'fraction_numerical': 0.05126050420168067, 'mean_word_length': 4.232967032967033, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3166949', 'n_tokens_mistral': 1029, 'n_tokens_neox': 935, 'n_words': 251}