<commit_before>package python

import (
	"errors"
	"io"
	"bufio"
//	"fmt"

	"github.com/daviddengcn/go-villa"
	"github.com/daviddengcn/sgrep/parser"
)

type Parser struct{}

var (
	EOF_UNEXPECTED = errors.New("EOF unexpected")
	InvalidFormat  = errors.New("Invalid format")
)

func init() {
	sparser.Register(".py", func() (sparser.Parser, error) {
		return Parser{}, nil
	})
}

const (
	ST_INDENT = iota
	ST_CONTENT
)

func (Parser) Parse(in io.Reader, rcvr sparser.Receiver) error {
	lineNumber := 1
	s := bufio.NewScanner(in)
	var indents villa.IntSlice
	indents.Add(-1)
	buffers := make([][]byte, 1)
	for ;s.Scan(); lineNumber++ {
		line := s.Bytes()
		indent := 0
		lineloop:
		for i, b := range line {
			switch rune(b) {
			case ' ':
				indent++
			case '\t':
				indent += 8 - indent % 8
			default:
				if rune(b) == '#' {
					// ignore comments
					continue
				}
				rg := sparser.Range {
					MinOffs: i,
					MaxOffs: len(line) - 1,
					MinLine: lineNumber,
					MaxLine: lineNumber,
				}
				
				for i <= indents[len(indents) - 1] {
					if err := rcvr.EndLevel(nil, nil); err != nil {
						return err
					}
					indents.Pop()
					buffers = buffers[:len(buffers)-1]
				}
				
				indents.Add(i)
				if len(indents) > len(buffers) {
					buffers = append(buffers, nil)
				}
				buffers[len(indents) - 1] = append(buffers[len(indents) - 1][:0], line...)
				if err := rcvr.StartLevel(buffers[len(indents) - 1], &rg); err != nil {
					return err
				}
				break lineloop
			}
		}
	}
	
	return s.Err()
}<commit_msg>Fix comment problem for python<commit_after>package python

import (
	"bufio"
	"errors"
	"io"
//	"fmt"

	"github.com/daviddengcn/go-villa"
	"github.com/daviddengcn/sgrep/parser"
)

type Parser struct{}

var (
	EOF_UNEXPECTED = errors.New("EOF unexpected")
	InvalidFormat  = errors.New("Invalid format")
)

func init() {
	sparser.Register(".py", func() (sparser.Parser, error) {
		return Parser{}, nil
	})
}

const (
	ST_INDENT = iota
	ST_CONTENT
)

func (Parser) Parse(in io.Reader, rcvr sparser.Receiver) error {
	lineNumber := 1
	s := bufio.NewScanner(in)
	var indents villa.IntSlice
	indents.Add(-1)
	buffers := make([][]byte, 1)
	for ; s.Scan(); lineNumber++ {
		line := s.Bytes()
		indent := 0
	lineloop:
		for i, b := range line {
			switch b {
			case ' ':
				indent++
			case '\t':
				indent += 8 - indent%8
			default:
				if b == '#' {
					// ignore comments
					break lineloop
				}
				rg := sparser.Range{
					MinOffs: i,
					MaxOffs: len(line) - 1,
					MinLine: lineNumber,
					MaxLine: lineNumber,
				}

				for i <= indents[len(indents)-1] {
					if err := rcvr.EndLevel(nil, nil); err != nil {
						return err
					}
					indents.Pop()
					buffers = buffers[:len(buffers)-1]
				}

				indents.Add(i)
				if len(indents) > len(buffers) {
					buffers = append(buffers, nil)
				}
				buffers[len(indents)-1] = append(buffers[len(indents)-1][:0], line...)
				if err := rcvr.StartLevel(buffers[len(indents)-1], &rg); err != nil {
					return err
				}
				break lineloop
			}
		}
	}

	return s.Err()
}
{'fraction_non_alphanumeric': 0.16117455138662318, 'fraction_numerical': 0.008482871125611746, 'mean_word_length': 2.658711217183771, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779370', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1186, 'n_words': 260}