<commit_before>// Package jq provides go bindings for libjq providing a streaming filter of
// JSON documents.
//
// This package provides a thin layer on top of stedolan's libjq -- it would
// likely be helpful to read through the wiki pages about it:
//
// jv: the JSON value type https://github.com/stedolan/jq/wiki/C-API:-jv
//
// libjq: https://github.com/stedolan/jq/wiki/C-API:-libjq
package jq

/*
To install
$ ./configure --disable-maintainer-mode --prefix=$PWD/BUILD
$ make install-libLTLIBRARIES install-includeHEADERS
*/

/*
#cgo CFLAGS: -I ${SRCDIR}/../jq-1.5/BUILD/include
#cgo LDFLAGS: ${SRCDIR}/../jq-1.5/BUILD/lib/libjq.a

#include <jq.h>
#include <jv.h>

#include <stdlib.h>

void install_jq_error_cb(jq_state *jq, unsigned long long id);
*/
import "C"
import (
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"unsafe"
)

// Jq encapsulates the state needed to interface with the libjq C library
type Jq struct {
	_state       *C.struct_jq_state
	errorStoreId uint64
}

// New initializes a new JQ object and the underlying C library.
func New() (*Jq, error) {
	jq := new(Jq)

	var err error
	jq._state, err = C.jq_init()

	if err != nil {
		return nil, err
	} else if jq == nil {
		return nil, errors.New("jq_init returned nil -- out of memory?")
	}

	return jq, nil
}

// Close the handle to libjq and free C resources
func (jq *Jq) Close() {
	if jq._state != nil {
		C.jq_teardown(&jq._state)
		jq._state = nil
	}
	if jq.errorStoreId != 0 {
		globalErrorChannels.Delete(jq.errorStoreId)
		jq.errorStoreId = 0
	}
}

// We cant pass many things over the Go/C boundary, so instead of passing the error channel we pass an opaque indentifier (a 64bit int as it turns out) and use that to look up in a global variable
type globalErrorChannelsState struct {
	sync.RWMutex
	idCounter uint64
	channels  map[uint64]chan<- error
}

func (e *globalErrorChannelsState) Add(c chan<- error) uint64 {
	newID := atomic.AddUint64(&e.idCounter, 1)
	e.RWMutex.Lock()
	defer e.RWMutex.Unlock()
	e.channels[newID] = c
	return newID
}

func (e *globalErrorChannelsState) Get(id uint64) chan<- error {
	e.RWMutex.RLock()
	defer e.RWMutex.RUnlock()
	c, ok := e.channels[id]
	if !ok {
		panic(fmt.Sprintf("Tried to get error channel #%d out of store but it wasn't there!", id))
	}
	return c
}

func (e *globalErrorChannelsState) Delete(id uint64) {
	e.RWMutex.Lock()
	defer e.RWMutex.Unlock()<commit_msg>fixup! Fix Go 1.6 go/cgo pointer constraints<commit_after>// Package jq provides go bindings for libjq providing a streaming filter of
// JSON documents.
//
// This package provides a thin layer on top of stedolan's libjq -- it would
// likely be helpful to read through the wiki pages about it:
//
// jv: the JSON value type https://github.com/stedolan/jq/wiki/C-API:-jv
//
// libjq: https://github.com/stedolan/jq/wiki/C-API:-libjq
package jq

/*
To install
$ ./configure --disable-maintainer-mode --prefix=$PWD/BUILD
$ make install-libLTLIBRARIES install-includeHEADERS
*/

/*
#cgo CFLAGS: -I ${SRCDIR}/../jq-1.5/BUILD/include
#cgo LDFLAGS: ${SRCDIR}/../jq-1.5/BUILD/lib/libjq.a

#include <jq.h>
#include <jv.h>

#include <stdlib.h>

void install_jq_error_cb(jq_state *jq, unsigned long long id);
*/
import "C"
import (
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"unsafe"
)

// Jq encapsulates the state needed to interface with the libjq C library
type Jq struct {
	_state       *C.struct_jq_state
	errorStoreId uint64
}

// New initializes a new JQ object and the underlying C library.
func New() (*Jq, error) {
	jq := new(Jq)

	var err error
	jq._state, err = C.jq_init()

	if err != nil {
		return nil, err
	} else if jq == nil {
		return nil, errors.New("jq_init returned nil -- out of memory?")
	}

	return jq, nil
}

// Close the handle to libjq and free C resources
func (jq *Jq) Close() {
	if jq._state != nil {
		C.jq_teardown(&jq._state)
		jq._state = nil
	}
	if jq.errorStoreId != 0 {
		globalErrorChannels.Delete(jq.errorStoreId)
		jq.errorStoreId = 0
	}
}

// We cant pass many things over the Go/C boundary, so instead of passing the error channel we pass an opaque indentifier (a 64bit int as it turns out) and use that to look up in a global variable
type errorLookupState struct {
	sync.RWMutex
	idCounter uint64
	channels  map[uint64]chan<- error
}

func (e *errorLookupState) Add(c chan<- error) uint64 {
	newID := atomic.AddUint64(&e.idCounter, 1)
	e.RWMutex.Lock()
	defer e.RWMutex.Unlock()
	e.channels[newID] = c
	return newID
}

func (e *errorLookupState) Get(id uint64) chan<- error {
	e.RWMutex.RLock()
	defer e.RWMutex.RUnlock()
	c, ok := e.channels[id]
	if !ok {
		panic(fmt.Sprintf("Tried to get error channel #%d out of store but it wasn't there!", id))
	}
	return c
}

func (e *errorLookupState) Delete(id uint64) {
	e.RWMutex.Lock()
	defer e.RWMutex.Unlock()
{'fraction_non_alphanumeric': 0.1285803888772737, 'fraction_numerical': 0.010035542546518922, 'mean_word_length': 3.5345971563981045, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7067505', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1757, 'n_words': 557}