<commit_before>// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"io"

	"github.com/m3db/m3/src/x/unsafe"

	"fmt"
	golog "log"
	"os"
	"time"

	"github.com/m3db/m3/src/dbnode/persist"
	"github.com/m3db/m3/src/dbnode/persist/fs"
	"github.com/m3db/m3/src/query/util/json"
	"github.com/m3db/m3/src/x/ident"

	"github.com/pborman/getopt"
	"go.uber.org/zap"
)

func main() {
	var (
		optPathPrefix = getopt.StringLong("path-prefix", 'p', "/var/lib/m3db", "Path prefix [e.g. /var/lib/m3db]")
		optOutputFile = getopt.StringLong("output-file", 'o', "", "Output JSON file of line delimited JSON objects for each segment")
	)
	getopt.Parse()

	logConfig := zap.NewDevelopmentConfig()
	log, err := logConfig.Build()
	if err != nil {
		golog.Fatalf("unable to create logger: %+v", err)
	}

	if *optPathPrefix == "" || *optOutputFile == "" {
		getopt.Usage()
		os.Exit(1)
	}

	run(runOptions{
		filePathPrefix: *optPathPrefix,
		outputFilePath: *optOutputFile,
		log:            log,
	})

}

type runOptions struct {
	filePathPrefix string
	outputFilePath string
	log            *zap.Logger
}

func run(opts runOptions) {
	log := opts.log

	fsOpts := fs.NewOptions().
		SetFilePathPrefix(opts.filePathPrefix)

	indexDirPath := fs.IndexDataDirPath(opts.filePathPrefix)

	namespaces, err := dirFiles(indexDirPath)
	if err != nil {
		log.Fatal("could not read namespaces", zap.Error(err))
	}

	// Get all fileset files.
	log.Info("discovered namespaces", zap.Strings("namespaces", namespaces))

	out, err := os.Create(opts.outputFilePath)
	if err != nil {
		log.Fatal("unable to create output file",
			zap.String("file", opts.outputFilePath),
			zap.Error(err))
	}

	for _, namespace := range namespaces {
		log.Info("reading segments", zap.String("namespace", namespace))
		ns := ident.StringID(namespace)

		readNamespaceSegments(out, ns, fsOpts, log)

		// Separate by endline.
		if _, err := out.WriteString("\n"); err != nil {
			log.Fatal("could not write endline", zap.Error(err))
		}
	}
}

func readNamespaceSegments(
	out io.Writer,
	nsID ident.ID,
	fsOpts fs.Options,
	log *zap.Logger,
) {
	infoFiles := fs.ReadIndexInfoFiles(fsOpts.FilePathPrefix(), nsID,
		fsOpts.InfoReaderBufferSize())

	for _, infoFile := range infoFiles {
		if err := infoFile.Err.Error(); err != nil {
			log.Error("unable to read index info file",
				zap.Stringer("namespace", nsID),
				zap.Error(err),
				zap.String("filepath", infoFile.Err.Filepath()),
			)
			continue
		}

		segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{
			ReaderOptions: fs.IndexReaderOpenOptions{
				Identifier:  infoFile.ID,
				FileSetType: persist.FileSetFlushType,
			},
			FilesystemOptions: fsOpts,
		})
		if err != nil {
			log.Error("unable to read segments from index fileset",
				zap.Stringer("namespace", nsID),
				zap.Error(err),
				zap.Time("blockStart", time.Unix(0, infoFile.Info.BlockStart)),
				zap.Int("volumeIndex", infoFile.ID.VolumeIndex),
			)
			continue
		}

		for i, seg := range segments {
			jw := json.NewWriter(out)
			jw.BeginObject()

			jw.BeginObjectField("namespace")
			jw.WriteString(nsID.String())

			jw.BeginObjectField("blockStart")
			jw.WriteString(time.Unix(0, infoFile.Info.BlockStart).Format(time.RFC3339))

			jw.BeginObjectField("volumeIndex")<commit_msg>[read_index_segments] Add log that outlines which segment being processed (#2825)<commit_after>// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"io"

	"github.com/m3db/m3/src/x/unsafe"

	"fmt"
	golog "log"
	"os"
	"time"

	"github.com/m3db/m3/src/dbnode/persist"
	"github.com/m3db/m3/src/dbnode/persist/fs"
	"github.com/m3db/m3/src/query/util/json"
	"github.com/m3db/m3/src/x/ident"

	"github.com/pborman/getopt"
	"go.uber.org/zap"
)

func main() {
	var (
		optPathPrefix = getopt.StringLong("path-prefix", 'p', "/var/lib/m3db", "Path prefix [e.g. /var/lib/m3db]")
		optOutputFile = getopt.StringLong("output-file", 'o', "", "Output JSON file of line delimited JSON objects for each segment")
	)
	getopt.Parse()

	logConfig := zap.NewDevelopmentConfig()
	log, err := logConfig.Build()
	if err != nil {
		golog.Fatalf("unable to create logger: %+v", err)
	}

	if *optPathPrefix == "" || *optOutputFile == "" {
		getopt.Usage()
		os.Exit(1)
	}

	run(runOptions{
		filePathPrefix: *optPathPrefix,
		outputFilePath: *optOutputFile,
		log:            log,
	})

}

type runOptions struct {
	filePathPrefix string
	outputFilePath string
	log            *zap.Logger
}

func run(opts runOptions) {
	log := opts.log

	fsOpts := fs.NewOptions().
		SetFilePathPrefix(opts.filePathPrefix)

	indexDirPath := fs.IndexDataDirPath(opts.filePathPrefix)

	namespaces, err := dirFiles(indexDirPath)
	if err != nil {
		log.Fatal("could not read namespaces", zap.Error(err))
	}

	// Get all fileset files.
	log.Info("discovered namespaces", zap.Strings("namespaces", namespaces))

	out, err := os.Create(opts.outputFilePath)
	if err != nil {
		log.Fatal("unable to create output file",
			zap.String("file", opts.outputFilePath),
			zap.Error(err))
	}

	for _, namespace := range namespaces {
		log.Info("reading segments", zap.String("namespace", namespace))
		ns := ident.StringID(namespace)

		readNamespaceSegments(out, ns, fsOpts, log)

		// Separate by endline.
		if _, err := out.WriteString("\n"); err != nil {
			log.Fatal("could not write endline", zap.Error(err))
		}
	}
}

func readNamespaceSegments(
	out io.Writer,
	nsID ident.ID,
	fsOpts fs.Options,
	log *zap.Logger,
) {
	infoFiles := fs.ReadIndexInfoFiles(fsOpts.FilePathPrefix(), nsID,
		fsOpts.InfoReaderBufferSize())

	for _, infoFile := range infoFiles {
		if err := infoFile.Err.Error(); err != nil {
			log.Error("unable to read index info file",
				zap.Stringer("namespace", nsID),
				zap.Error(err),
				zap.String("filepath", infoFile.Err.Filepath()),
			)
			continue
		}

		log.Info("reading block segments",
			zap.String("namespace", nsID.String()),
			zap.String("blockStart", infoFile.ID.BlockStart.String()),
			zap.Int64("blockStartUnixNano", infoFile.ID.BlockStart.UnixNano()),
			zap.Strings("files", infoFile.AbsoluteFilePaths))

		segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{
			ReaderOptions: fs.IndexReaderOpenOptions{
				Identifier:  infoFile.ID,
				FileSetType: persist.FileSetFlushType,
			},
			FilesystemOptions: fsOpts,
		})
		if err != nil {
			log.Error("unable to read segments from index fileset",
				zap.Stringer("namespace", nsID),
				zap.Error(err),
				zap.Time("blockStart", time.Unix(0, infoFile.Info.BlockStart)),
				zap.Int("volumeIndex", infoFile.ID.VolumeIndex),
			)
			continue
		}

		for i, seg := range segments {
			jw := json.NewWriter(out)
			jw.BeginObject()

			jw.BeginObjectField("namespace")
			jw.WriteString(nsID.String())

			jw.BeginObjectField("blockStart")
			jw.WriteString(time.Unix(0, infoFile.Info.BlockStart).Format(time.RFC3339))

			jw.BeginObjectField("volumeIndex")
{'fraction_non_alphanumeric': 0.13090909090909092, 'fraction_numerical': 0.005730027548209366, 'mean_word_length': 3.946049046321526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5099308', 'n_tokens_mistral': 3264, 'n_tokens_neox': 2975, 'n_words': 809}