<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit/aws"
	"github.com/trackit/trackit/aws/s3"
	"github.com/trackit/trackit/cache"
	"github.com/trackit/trackit/db"
	"github.com/trackit/trackit/models"
)

const iso8601DateFormat = "2006-01-02"

// taskIngest ingests billing data for a given BillRepository and AwsAccount.
func taskIngestLimit(ctx context.Context) error {
	args := flag.Args()
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Debug("Running task 'ingest-limit'.", map[string]interface{}{
		"args": args,
	})
	if len(args) != 3 {
		return errors.New("taskIngest requires three integer arguments")
	} else if aa, err := strconv.Atoi(args[0]); err != nil {
		return err
	} else if br, err := strconv.Atoi(args[1]); err != nil {
		return err
	} else if dateUpperLimit, err := time.Parse(iso8601DateFormat, args[2]); err != nil {
		return err
	} else {
		logger.Debug("Launching ingest billing", map[string]interface{}{
			"date": dateUpperLimit,
		})
		return ingestBillingDataForBillRepositoryLimit(ctx, aa, br, dateUpperLimit)
	}
}

// ingestBillingDataForBillRepository ingests the billing data for a<commit_msg>Added debug<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit/aws"
	"github.com/trackit/trackit/aws/s3"
	"github.com/trackit/trackit/cache"
	"github.com/trackit/trackit/db"
	"github.com/trackit/trackit/models"
)

const iso8601DateFormat = "2006-01-02"

// taskIngest ingests billing data for a given BillRepository and AwsAccount.
func taskIngestLimit(ctx context.Context) error {
	args := flag.Args()
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Debug("Running task 'ingest-limit'.", map[string]interface{}{
		"args": args,
	})
	if len(args) != 3 {
		return errors.New("taskIngest requires three integer arguments")
	} else if aa, err := strconv.Atoi(args[0]); err != nil {
		return err
	} else if br, err := strconv.Atoi(args[1]); err != nil {
		return err
	} else if dateUpperLimit, err := time.Parse(iso8601DateFormat, args[2]); err != nil {
		logger.Debug("Error while decoding date", map[string]interface{}{
			"date": dateUpperLimit,
			"err":  err,
		})
		return err
	} else {
		logger.Debug("Launching ingest billing", map[string]interface{}{
			"date": dateUpperLimit,
		})
		return ingestBillingDataForBillRepositoryLimit(ctx, aa, br, dateUpperLimit)
	}
}

// ingestBillingDataForBillRepository ingests the billing data for a
{'fraction_non_alphanumeric': 0.12436028659160696, 'fraction_numerical': 0.014841351074718526, 'mean_word_length': 3.9606598984771573, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '88060', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1260, 'n_words': 371}