<commit_before>package docconv

import (
	"encoding/csv"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"

	"github.com/360EntSecGroup-Skylar/excelize"
	"github.com/tealeg/xlsx"
)

// convertWithTealegXlsx used tealeg/xlsx to convert XLSX to CSV
func convertWithTealegXlsx(f *LocalFile) (body string, err error) {
	fileStat, err := f.Stat()
	if err != nil {
		err = fmt.Errorf("error convert with 'tealeg/xlsx' on getting file stats: %v", err)
		return
	}
	xlsFile, err := xlsx.OpenReaderAt(f, fileStat.Size())
	if err != nil {
		err = fmt.Errorf("error convert with 'tealeg/xlsx' on xlsx parsing: %v", err)
		return
	}
	for _, sheet := range xlsFile.Sheets {
		for rowIndex, row := range sheet.Rows {
			for cellIndex, cell := range row.Cells {
				text := cell.String()
				body += text
				if cellIndex < len(row.Cells)-1 {<commit_msg>fix: change name of xls module<commit_after>package docconv

import (
	"encoding/csv"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"

	"github.com/qax-os/excelize"
	"github.com/tealeg/xlsx"
)

// convertWithTealegXlsx used tealeg/xlsx to convert XLSX to CSV
func convertWithTealegXlsx(f *LocalFile) (body string, err error) {
	fileStat, err := f.Stat()
	if err != nil {
		err = fmt.Errorf("error convert with 'tealeg/xlsx' on getting file stats: %v", err)
		return
	}
	xlsFile, err := xlsx.OpenReaderAt(f, fileStat.Size())
	if err != nil {
		err = fmt.Errorf("error convert with 'tealeg/xlsx' on xlsx parsing: %v", err)
		return
	}
	for _, sheet := range xlsFile.Sheets {
		for rowIndex, row := range sheet.Rows {
			for cellIndex, cell := range row.Cells {
				text := cell.String()
				body += text
				if cellIndex < len(row.Cells)-1 {
{'fraction_non_alphanumeric': 0.13845223700120918, 'fraction_numerical': 0.003022974607013301, 'mean_word_length': 3.7017045454545454, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5761329', 'n_tokens_mistral': 692, 'n_tokens_neox': 647, 'n_words': 169}