<commit_before>// Package to read and write FASTA format files
package fasta

// Modified under the terms of GPL3 from
// <NAME> github.com/kortschak/BioGo

import (
	"bufio"
	"bytes"
	"errors"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/node/file"
	"github.com/MG-RAST/Shock/shock-server/node/file/format/seq"
	"io"
	"os"
	"regexp"
)

var (
	Regex = regexp.MustCompile(`^[\n\r]*>\S+[\S\t ]*[\n\r]+[A-Za-z\- ]+`)
)

// Fasta sequence format reader type.
type Reader struct {
	f file.SectionReader
	r *bufio.Reader
}

// Returns a new fasta format reader using f.
func NewReader(f file.SectionReader) seq.ReadRewinder {
	return &Reader{
		f: f,
		r: nil,
	}
}

// Read a single sequence and return it or an error.
func (self *Reader) Read() (sequence *seq.Seq, err error) {
	if self.r == nil {
		self.r = bufio.NewReader(self.f)
	}
	var prev, read, label, body []byte
	var eof bool
	for {
		read, err = self.r.ReadBytes('>')
		// non eof error
		if err != nil {
			if err == io.EOF {
				eof = true
			} else {
				return
			}
		}
		if len(prev) > 0 {
			read = append(prev, read...)
		}
		// only have '>'
		if len(read) == 1 {
			if eof {
				break
			} else {
				continue
			}
		}
		// found an embedded '>'
		if !bytes.Contains(read, []byte{'\n'}) {
			prev = read
			continue
		}
		// process lines
		read = bytes.TrimSpace(bytes.TrimRight(read, ">"))
		lines := bytes.Split(read, []byte{'\n'})
		if len(lines) > 1 {
			label = lines[0]
			body = bytes.Join(lines[1:], []byte{})
		}
		break
	}
	if len(label) > 0 && len(body) > 0 {
		sequence = seq.New(label, body, nil)
	} else {
		err = errors.New("Invalid fasta entry")
	}
	if eof {
		err = io.EOF
	}
	return
}

// Read a single sequence and return read offset for indexing.
func (self *Reader) GetReadOffset() (n int, err error) {
	if self.r == nil {
		self.r = bufio.NewReader(self.f)
	}
	n = 0
	var read []byte
	var eof bool
	for {
		read, err = self.r.ReadBytes('>')
		// non eof error
		if err != nil {
			if err == io.EOF {
				eof = true
			} else {
				return
			}
		}
		// handle embedded '>'
		if (len(read) > 1) && bytes.Contains(read, []byte{'\n'}) {
			// check for sequence
			lines := bytes.Split(bytes.TrimSpace(bytes.TrimRight(read, ">")), []byte{'\n'})
			seq := bytes.Join(lines[1:], []byte{})
			if len(seq) == 0 {
				err = errors.New("Invalid fasta entry:\n" + string(read))
				return
			}
			if eof {
				n += len(read)
				err = io.EOF
			} else {
				n += len(read) - 1
				err = self.r.UnreadByte()
			}
			break
		} else {
			n += len(read)
		}
		if eof {
			err = io.EOF
			break
		}
	}
	return
}

// seek sequences which add up to a size close to the configured chunk size (conf.CHUNK_SIZE, e.g. 1M)<commit_msg>fix debug output<commit_after>// Package to read and write FASTA format files
package fasta

// Modified under the terms of GPL3 from
// <NAME> github.com/kortschak/BioGo

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"

	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/node/file"
	"github.com/MG-RAST/Shock/shock-server/node/file/format/seq"
)

var (
	Regex = regexp.MustCompile(`^[\n\r]*>\S+[\S\t ]*[\n\r]+[A-Za-z\- ]+`)
)

// Fasta sequence format reader type.
type Reader struct {
	f file.SectionReader
	r *bufio.Reader
}

// Returns a new fasta format reader using f.
func NewReader(f file.SectionReader) seq.ReadRewinder {
	return &Reader{
		f: f,
		r: nil,
	}
}

// Read a single sequence and return it or an error.
func (self *Reader) Read() (sequence *seq.Seq, err error) {
	if self.r == nil {
		self.r = bufio.NewReader(self.f)
	}
	var prev, read, label, body []byte
	var eof bool
	for {
		read, err = self.r.ReadBytes('>')
		// non eof error
		if err != nil {
			if err == io.EOF {
				eof = true
			} else {
				return
			}
		}
		if len(prev) > 0 {
			read = append(prev, read...)
		}
		// only have '>'
		if len(read) == 1 {
			if eof {
				break
			} else {
				continue
			}
		}
		// found an embedded '>'
		if !bytes.Contains(read, []byte{'\n'}) {
			prev = read
			continue
		}
		// process lines
		read = bytes.TrimSpace(bytes.TrimRight(read, ">"))
		lines := bytes.Split(read, []byte{'\n'})
		if len(lines) > 1 {
			label = lines[0]
			body = bytes.Join(lines[1:], []byte{})
		}
		break
	}
	if len(label) > 0 && len(body) > 0 {
		sequence = seq.New(label, body, nil)
	} else {
		err = errors.New("Invalid fasta entry")
	}
	if eof {
		err = io.EOF
	}
	return
}

// Read a single sequence and return read offset for indexing.
func (self *Reader) GetReadOffset() (n int, err error) {
	if self.r == nil {
		self.r = bufio.NewReader(self.f)
	}
	n = 0
	var read []byte
	var eof bool
	for {
		read, err = self.r.ReadBytes('>')
		// non eof error
		if err != nil {
			if err == io.EOF {
				eof = true
			} else {
				return
			}
		}
		// handle embedded '>'
		if (len(read) > 1) && bytes.Contains(read, []byte{'\n'}) {
			// check for sequence
			lines := bytes.Split(bytes.TrimSpace(bytes.TrimRight(read, ">")), []byte{'\n'})
			seq := bytes.Join(lines[1:], []byte{})
			if len(seq) == 0 {
				showLen := len(read)
				if showLen > 50 {
					showLen = 50
				}
				err = fmt.Errorf("Invalid fasta entry: %s", read[0:showLen])
				return
			}
			if eof {
				n += len(read)
				err = io.EOF
			} else {
				n += len(read) - 1
				err = self.r.UnreadByte()
			}
			break
		} else {
			n += len(read)
		}
		if eof {
			err = io.EOF
			break
		}
	}
	return
}

// seek sequences which add up to a size close to the configured chunk size (conf.CHUNK_SIZE, e.g. 1M)
{'fraction_non_alphanumeric': 0.16867253457876774, 'fraction_numerical': 0.005927788755164361, 'mean_word_length': 2.532994923857868, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7477418', 'n_tokens_mistral': 2511, 'n_tokens_neox': 2191, 'n_words': 580}