<commit_before>package gocassos

import (
	"errors"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/gocql/gocql"
)

type HttpInputHandler struct {
	w           http.ResponseWriter
	r           *http.Request
	o           *Object
	failure     bool
	in_progress sync.WaitGroup
}

type HttpOutputHandler struct {
	HttpInputHandler

	t        *os.File
	mode     int
	pipeline chan int64

	waiting_for_chunk int64
	chunk_status      []bool
}

type ReadAndWriteAt interface {
	ReadAt([]byte, int64) (int, error)
	WriteAt([]byte, int64) (int, error)
	Close() error
}

type InputHandler interface {
	Read([]byte) (int, error)
	Close() error
}

type OutputHandler interface {
	WriteAt([]byte, int64) (int, error)
	Close() error
}

type Object struct {
	ClientId string

	Objectname string
	Updated    int64
	Nodetag    gocql.UUID
	id         string

	NumChunks  int64
	ChunkSize  int64
	ObjectSize int64
	Expiration time.Time

	finished_chunks chan int64
	fetcher_control chan struct{}
	pusher_control  chan struct{}

	failure     bool
	in_progress sync.WaitGroup

	lookup_start time.Time
	LookupTime   time.Duration
	fetch_start  time.Time
	FetchTime    time.Duration
	push_start   time.Time
	PushTime     time.Duration

	OutputHandler OutputHandler
	InputHandler  InputHandler

	cfg *ObjectStorage
}

var consistencies = map[string]gocql.Consistency{
	"localquorum": gocql.LocalQuorum,
	"one":         gocql.One,
	"quorum":      gocql.Quorum,
	"any":         gocql.Any,
	"all":         gocql.All,
}

type ObjectStorage struct {
	Conn           *gocql.Session
	ScrubGraceTime int
	AllowUpdates   bool
	PopulatePaths  bool
	ChunkSize      int64

	write_consistency_str []string
	write_consistency     []gocql.Consistency
	read_consistency_str  []string
	read_consistency      []gocql.Consistency

	ConcurrentGetsPerObj int
	ConcurrentPutsPerObj int

	in_progress sync.WaitGroup
}

type Logger struct {
	prefix string
	level  int
}

const (
	StreamMode = iota
	BatchMode
	HeadMode
)

var ErrNullReference = errors.New("Null object reference")
var ErrRefused = errors.New("Update refused")
var ErrNotFound = errors.New("Not Found")
var ErrChunksFailed = errors.New("Failed to push chunks")
var ErrCassandraNotConnected = errors.New("Cassandra not connected")<commit_msg>Added transfer mode names map<commit_after>package gocassos

import (
	"errors"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/gocql/gocql"
)

type HttpInputHandler struct {
	w           http.ResponseWriter
	r           *http.Request
	o           *Object
	failure     bool
	in_progress sync.WaitGroup
}

type HttpOutputHandler struct {
	HttpInputHandler

	t        *os.File
	mode     int
	pipeline chan int64

	waiting_for_chunk int64
	chunk_status      []bool
}

type ReadAndWriteAt interface {
	ReadAt([]byte, int64) (int, error)
	WriteAt([]byte, int64) (int, error)
	Close() error
}

type InputHandler interface {
	Read([]byte) (int, error)
	Close() error
}

type OutputHandler interface {
	WriteAt([]byte, int64) (int, error)
	Close() error
}

type Object struct {
	ClientId string

	Objectname string
	Updated    int64
	Nodetag    gocql.UUID
	id         string

	NumChunks  int64
	ChunkSize  int64
	ObjectSize int64
	Expiration time.Time

	finished_chunks chan int64
	fetcher_control chan struct{}
	pusher_control  chan struct{}

	failure     bool
	in_progress sync.WaitGroup

	lookup_start time.Time
	LookupTime   time.Duration
	fetch_start  time.Time
	FetchTime    time.Duration
	push_start   time.Time
	PushTime     time.Duration

	OutputHandler OutputHandler
	InputHandler  InputHandler

	cfg *ObjectStorage
}

var consistencies = map[string]gocql.Consistency{
	"localquorum": gocql.LocalQuorum,
	"one":         gocql.One,
	"quorum":      gocql.Quorum,
	"any":         gocql.Any,
	"all":         gocql.All,
}

type ObjectStorage struct {
	Conn           *gocql.Session
	ScrubGraceTime int
	AllowUpdates   bool
	PopulatePaths  bool
	ChunkSize      int64

	write_consistency_str []string
	write_consistency     []gocql.Consistency
	read_consistency_str  []string
	read_consistency      []gocql.Consistency

	ConcurrentGetsPerObj int
	ConcurrentPutsPerObj int

	in_progress sync.WaitGroup
}

type Logger struct {
	prefix string
	level  int
}

const (
	StreamMode = iota
	BatchMode
	HeadMode
)

var TransferModes = map[string]int{
	"stream": StreamMode,
	"batch":  BatchMode,
	"head":   HeadMode,
}

var ErrNullReference = errors.New("Null object reference")
var ErrRefused = errors.New("Update refused")
var ErrNotFound = errors.New("Not Found")
var ErrChunksFailed = errors.New("Failed to push chunks")
var ErrCassandraNotConnected = errors.New("Cassandra not connected")
{'fraction_non_alphanumeric': 0.07871468621953849, 'fraction_numerical': 0.009488893681259434, 'mean_word_length': 3.624127617148554, 'pattern_counts': {'":': 13, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6222009', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1575, 'n_words': 402}