<commit_before>package types

import (
	"encoding/json"
	"github.com/pkg/errors"
	"golang.org/x/text/currency"
	"strconv"
	"time"
)

// UnmarshalJSON implements json.Unmarshaller for a Journey
func (j *Journey) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Duration  int64 `json:"duration"`
		Transfers *uint `json:"nb_transfers"`

		Departure string `json:"departure_date_time"`
		Requested string `json:"requested_date_time"`
		Arrival   string `json:"arrival_date_time"`

		Sections *[]Section `json:"sections"`

		From PlaceCountainer `json:"from"`
		To   PlaceCountainer `json:"to"`

		Type *JourneyQualification `json:"type"`

		Fare *Fare `json:"fare"`

		Status *JourneyStatus `json:"status"`
	}{
		Transfers: &j.Transfers,
		Sections:  &j.Sections,
		Type:      &j.Type,
		Fare:      &j.Fare,
		Status:    &j.Status,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// As the given duration is in second, let's multiply it by one second to have the correct value
	j.Duration = time.Duration(data.Duration) * time.Second

	// For departure, requested and arrival, we use parseDateTime
	j.Departure, err = parseDateTime(data.Departure)
	if err != nil {
		return unmarshalErr(err, "Departure", "departure_date_time", data.Departure, "parseDateTime failed")
	}
	j.Requested, err = parseDateTime(data.Requested)
	if err != nil {
		return unmarshalErr(err, "Requested", "requested_date_time", data.Requested, "parseDateTime failed")
	}
	j.Arrival, err = parseDateTime(data.Arrival)
	if err != nil {
		return unmarshalErr(err, "Arrival", "arrival_date_time", data.Arrival, "parseDateTime failed")
	}

	// For the places, we directly use the embedded type !
	j.From, err = data.From.Place()
	if err != nil {
		return unmarshalErr(err, "From", "from", data.From, " .Place() failed")
	}
	j.To, err = data.To.Place()
	if err != nil {
		return unmarshalErr(err, "To", "to", data.To, " .Place() failed")<commit_msg>types: in cases of coordinates form of addresses IDs, no need for query escaping<commit_after>package types

import (
	"encoding/json"
	"github.com/pkg/errors"
	"golang.org/x/text/currency"
	"strconv"
	"time"
)

// UnmarshalJSON implements json.Unmarshaller for a Journey
func (j *Journey) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Duration  int64 `json:"duration"`
		Transfers *uint `json:"nb_transfers"`

		Departure string `json:"departure_date_time"`
		Requested string `json:"requested_date_time"`
		Arrival   string `json:"arrival_date_time"`

		Sections *[]Section `json:"sections"`

		From PlaceCountainer `json:"from"`
		To   PlaceCountainer `json:"to"`

		Type *JourneyQualification `json:"type"`

		Fare *Fare `json:"fare"`

		Status *JourneyStatus `json:"status"`
	}{
		Transfers: &j.Transfers,
		Sections:  &j.Sections,
		Type:      &j.Type,
		Fare:      &j.Fare,
		Status:    &j.Status,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// As the given duration is in second, let's multiply it by one second to have the correct value
	j.Duration = time.Duration(data.Duration) * time.Second

	// For departure, requested and arrival, we use parseDateTime
	j.Departure, err = parseDateTime(data.Departure)
	if err != nil {
		return unmarshalErr(err, "Departure", "departure_date_time", data.Departure, "parseDateTime failed")
	}
	j.Requested, err = parseDateTime(data.Requested)
	if err != nil {
		return unmarshalErr(err, "Requested", "requested_date_time", data.Requested, "parseDateTime failed")
	}
	j.Arrival, err = parseDateTime(data.Arrival)
	if err != nil {
		return unmarshalErr(err, "Arrival", "arrival_date_time", data.Arrival, "parseDateTime failed")
	}

	// For the places, we directly use the embedded type !
	// Warning: it is possible for a countainer to be empty, in those cases a nil Place will be present
	j.From, err = data.From.Place()
	if err != nil {
		return unmarshalErr(err, "From", "from", data.From, " .Place() failed")
	}
	j.To, err = data.To.Place()
	if err != nil {
		return unmarshalErr(err, "To", "to", data.To, " .Place() failed")
{'fraction_non_alphanumeric': 0.12602150537634407, 'fraction_numerical': 0.0008602150537634409, 'mean_word_length': 4.26727066817667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083988', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1530, 'n_words': 475}