<commit_before>// Usage: colsplit {separator} {filename} {column-filename-format}
//
// Splits columns of a CSV file into a separate file for each column.
// Quoted fields, if any, are unquoted in the process.
//
// Separator can be specified as any string.
//
// Filename is the name of the input file.
//
// Column-filename-format is a format string for fmt.Printf(), which specifies
// the schema for name of the output files. This schema should contain substring '%d'
// exactly once, and this is going to be replaced by the number of the column, starting with 0.
// 
package main

import (
	"os"
	"bytes"
	"io/ioutil"
	"unicode/utf8"
	"fmt"
)

func consumeQuoted(line []byte, ln int) (field []byte, t []byte) {
	inidx, outidx := 0, 0
	
	// line[0] == '"'
	line = line[1:]
	
	for inidx < len(line) {
		r, size := utf8.DecodeRune(line[inidx:])
		
		if r == '"' {
			inidx += size
			r, size = utf8.DecodeRune(line[inidx:])
			
			if r != '"' {
				return line[:outidx], line[inidx:]
			}
		}
		
		inidx += size
		outidx += utf8.EncodeRune(line[outidx:], r)
	}
	
	fmt.Fprintln(os.Stderr, "Quoted end of line on line %d.\n", ln)
	
	return line[:outidx], nil
}

func head(line []byte, ln int, separator []byte) (h []byte, t []byte) {
	if line[0] == '"' {
		field, line := consumeQuoted(line, ln)
		
		if !bytes.HasPrefix(line, separator) && len(line) > 0 {
			fmt.Fprintln(os.Stderr, "Malformed quotation on line %d.\n", ln)
		} else {
			line = line[len(separator):]<commit_msg>Another bug.<commit_after>// Usage: colsplit {separator} {filename} {column-filename-format}
//
// Splits columns of a CSV file into a separate file for each column.
// Quoted fields, if any, are unquoted in the process.
//
// Separator can be specified as any string.
//
// Filename is the name of the input file.
//
// Column-filename-format is a format string for fmt.Printf(), which specifies
// the schema for name of the output files. This schema should contain substring '%d'
// exactly once, and this is going to be replaced by the number of the column, starting with 0.
// 
package main

import (
	"os"
	"bytes"
	"io/ioutil"
	"unicode/utf8"
	"fmt"
)

func consumeQuoted(line []byte, ln int) (field []byte, t []byte) {
	inidx, outidx := 0, 0
	
	// line[0] == '"'
	line = line[1:]
	
	for inidx < len(line) {
		r, size := utf8.DecodeRune(line[inidx:])
		
		if r == '"' {
			inidx += size
			r, size = utf8.DecodeRune(line[inidx:])
			
			if r != '"' {
				return line[:outidx], line[inidx:]
			}
		}
		
		inidx += size
		outidx += utf8.EncodeRune(line[outidx:], r)
	}
	
	fmt.Fprintln(os.Stderr, "Quoted end of line on line %d.\n", ln)
	
	return line[:outidx], nil
}

func head(line []byte, ln int, separator []byte) (h []byte, t []byte) {
	if line[0] == '"' {
		field, line := consumeQuoted(line, ln)
		
		if bytes.HasPrefix(line, separator) {
			line = line[len(separator):]
		} else if len(line) > 0 {
			fmt.Fprintln(os.Stderr, "Malformed quotation on line %d.\n", ln)
{'fraction_non_alphanumeric': 0.15696887686062247, 'fraction_numerical': 0.007442489851150203, 'mean_word_length': 3.194326241134752, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '311189', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1043, 'n_words': 351}