<commit_before>package virt

import (
	"context"
	"errors"
	"io"

	"github.com/jimmyfrasche/etlite/internal/internal/errint"
	"github.com/jimmyfrasche/etlite/internal/internal/synth"
	"github.com/jimmyfrasche/etlite/internal/token"
)

type ImportSpec struct {
	Pos           token.Position
	Temp          bool
	Table, Frame  string
	Header        []string
	Limit, Offset int
}

func (s ImportSpec) Valid() error {
	if s.Table == "" {
		return errint.New("table did not provide name")
	}
	return nil
}

func (m *Machine) readHeader(frame string, header []string) ([]string, error) {
	dec := m.decoder
	if dec == nil {
		return nil, errint.New("no decoder available when importing")
	}
	inHeader, err := dec.ReadHeader(frame, header)
	if err != nil {
		return nil, err
	}
	return inHeader, nil
}

func Import(s ImportSpec) Instruction {
	return func(ctx context.Context, m *Machine) error {
		if err := s.Valid(); err != nil {
			return err
		}

		inHeader, err := m.readHeader(s.Frame, s.Header)
		if err != nil {
			return err
		}

		//no header specified, use the one from the decoder
		if len(s.Header) == 0 {
			if len(inHeader) == 0 {
				return errors.New("no header specified and none returned by " + m.decoder.Name() + " format")
			}
			s.Header = inHeader
		}

		ddl := synth.CreateTable(s.Temp, s.Table, s.Header)<commit_msg>not much point in ImportSpec.Valid at this point, delete<commit_after>package virt

import (
	"context"
	"errors"
	"io"

	"github.com/jimmyfrasche/etlite/internal/internal/errint"
	"github.com/jimmyfrasche/etlite/internal/internal/synth"
	"github.com/jimmyfrasche/etlite/internal/token"
)

type ImportSpec struct {
	Pos           token.Position
	Temp          bool
	Table, Frame  string
	Header        []string
	Limit, Offset int
}

func (m *Machine) readHeader(frame string, header []string) ([]string, error) {
	dec := m.decoder
	if dec == nil {
		return nil, errint.New("no decoder available when importing")
	}
	inHeader, err := dec.ReadHeader(frame, header)
	if err != nil {
		return nil, err
	}
	return inHeader, nil
}

func Import(s ImportSpec) Instruction {
	return func(ctx context.Context, m *Machine) error {
		inHeader, err := m.readHeader(s.Frame, s.Header)
		if err != nil {
			return err
		}

		//no header specified, use the one from the decoder
		if len(s.Header) == 0 {
			if len(inHeader) == 0 {
				return errors.New("no header specified and none returned by " + m.decoder.Name() + " format")
			}
			s.Header = inHeader
		}

		ddl := synth.CreateTable(s.Temp, s.Table, s.Header)
{'fraction_non_alphanumeric': 0.12628255722178375, 'fraction_numerical': 0.0015785319652722968, 'mean_word_length': 3.3481989708404805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1519916', 'n_tokens_mistral': 951, 'n_tokens_neox': 871, 'n_words': 245}