<commit_before>import { v4 as uuid } from 'uuid';
import { joinList, toString } from '@terascope/utils';
import {
    DataTypeFieldConfig, DataTypeFields, Maybe
} from '@terascope/types';
import { Builder } from '../builder';
import {
    Vector, isVector, VectorType
} from '../vector';
import { ColumnTransformFn, TransformMode } from './interfaces';
import { ReadableData, WritableData } from '../data';

const _vectorIds = new WeakMap<Vector<any>, string>();
export function getVectorId(vector: Vector<any>): string {
    const id = _vectorIds.get(vector);
    if (id) return id;
    const newId = uuid();
    _vectorIds.set(vector, newId);
    return newId;
}

/**
 * Map over the Vector
*/
export function mapVector<T, R = T>(
    vector: Vector<T>,
    transform: ColumnTransformFn<T, R>,
    config?: Partial<DataTypeFieldConfig>,
): Vector<R> {
    const builder = Builder.make<R>(
        { ...vector.config, ...config, ...transform.output },
        WritableData.make(vector.size),
        vector.childConfig
    );

    if (transform.mode === TransformMode.NONE) {
        return mapVectorNone(vector, builder);
    }

    if (transform.mode === TransformMode.EACH) {
        return mapVectorEach(
            vector, builder, transform.fn
        );
    }

    if (transform.mode === TransformMode.EACH_VALUE) {
        return mapVectorEachValue(
            vector, builder, transform.fn
        );
    }

    throw new Error(`Unknown transformation ${toString(transform)}`);
}

export function mapVectorNone<T, R = T>(
    vector: Vector<T>,
    builder: Builder<R>,
): Vector<R> {
    for (const { value, indices } of vector.data.values) {
        builder.mset(value, indices);
    }
    return builder.toVector();
}

export function mapVectorEach<T, R = T>(
    vector: Vector<T>,
    builder: Builder<R>,
    fn: (value: Maybe<T|Vector<T>>) => Maybe<R|Vector<R>>,
): Vector<R> {
    for (const { value, indices } of vector.data.values) {
        builder.mset(fn(value), indices);
    }
    return builder.toVector();
}

export function mapVectorEachValue<T, R = T>(
    vector: Vector<T>,
    builder: Builder<R>,
    fn: (value: T) => Maybe<R>,
): Vector<R> {
    for (const { value, indices } of vector.data.values) {
        if (isVector<T>(value)) {
            const values: Maybe<R>[] = [];
            for (const val of value) {
                values.push(val != null ? fn(val as any) : null);
            }
            builder.mset(values, indices);
        } else {
            builder.mset(
                fn(value),
                indices,
            );
        }
    }
    return builder.toVector();
}

export function validateFieldTransformArgs<A extends Record<string, any>>(
    schema?: DataTypeFields, requiredArgs?: string[], args?: Partial<A>
): A {
    if (!schema) return {} as any;

    const result = { ...args } as A;

    for (const [name, config] of Object.entries(schema)) {
        const field = name as keyof A;

        const required = requiredArgs?.includes(name);
        if (required && result[field] == null) {
            throw new Error(`Missing required parameter ${field}`);
        }

        const builder = Builder.make(config, WritableData.make(0));
        if (builder.valueFrom && result[field] != null) {
            result[field] = builder.valueFrom(result[field], builder) as any;
        }
    }

    return result;
}

const emptyData = new ReadableData<any>(WritableData.make(0));
<commit_msg>Fix mapVectorEachValue<commit_after>import { v4 as uuid } from 'uuid';
import { joinList, toString } from '@terascope/utils';
import {
    DataTypeFieldConfig, DataTypeFields, Maybe
} from '@terascope/types';
import { Builder } from '../builder';
import {
    Vector, isVector, VectorType
} from '../vector';
import { ColumnTransformFn, TransformMode } from './interfaces';
import { ReadableData, WritableData } from '../data';

const _vectorIds = new WeakMap<Vector<any>, string>();
export function getVectorId(vector: Vector<any>): string {
    const id = _vectorIds.get(vector);
    if (id) return id;
    const newId = uuid();
    _vectorIds.set(vector, newId);
    return newId;
}

/**
 * Map over the Vector
*/
export function mapVector<T, R = T>(
    vector: Vector<T>,
    transform: ColumnTransformFn<T, R>,
    config?: Partial<DataTypeFieldConfig>,
): Vector<R> {
    const builder = Builder.make<R>(
        { ...vector.config, ...config, ...transform.output },
        WritableData.make(vector.size),
        vector.childConfig
    );

    if (transform.mode === TransformMode.NONE) {
        return mapVectorNone(vector, builder);
    }

    if (transform.mode === TransformMode.EACH) {
        return mapVectorEach(
            vector, builder, transform.fn
        );
    }

    if (transform.mode === TransformMode.EACH_VALUE) {
        return mapVectorEachValue(
            vector, builder, transform.fn
        );
    }

    throw new Error(`Unknown transformation ${toString(transform)}`);
}

export function mapVectorNone<T, R = T>(
    vector: Vector<T>,
    builder: Builder<R>,
): Vector<R> {
    for (const { value, indices } of vector.data.values) {
        builder.mset(value, indices);
    }
    return builder.toVector();
}

export function mapVectorEach<T, R = T>(
    vector: Vector<T>,
    builder: Builder<R>,
    fn: (value: Maybe<T|Vector<T>>) => Maybe<R|Vector<R>>,
): Vector<R> {
    let i = 0;
    for (const value of vector) {
        builder.set(i++, fn(value));
    }
    return builder.toVector();
}

export function mapVectorEachValue<T, R = T>(
    vector: Vector<T>,
    builder: Builder<R>,
    fn: (value: T) => Maybe<R>,
): Vector<R> {
    function _mapValue(value: T|Vector<T>): Maybe<R> {
        if (!isVector<T>(value)) return fn(value);
        const values: Maybe<R>[] = [];
        for (const val of value) {
            values.push(val != null ? fn(val as any) : null);
        }
        return values as any;
    }

    for (const { value, indices } of vector.data.values) {
        builder.mset(_mapValue(value), indices);
    }
    return builder.toVector();
}

export function validateFieldTransformArgs<A extends Record<string, any>>(
    schema?: DataTypeFields, requiredArgs?: string[], args?: Partial<A>
): A {
    if (!schema) return {} as any;

    const result = { ...args } as A;

    for (const [name, config] of Object.entries(schema)) {
        const field = name as keyof A;

        const required = requiredArgs?.includes(name);
        if (required && result[field] == null) {
            throw new Error(`Missing required parameter ${field}`);
        }

        const builder = Builder.make(config, WritableData.make(0));
        if (builder.valueFrom && result[field] != null) {
            result[field] = builder.valueFrom(result[field], builder) as any;
        }
    }

    return result;
}

const emptyData = new ReadableData<any>(WritableData.make(0));

{'fraction_non_alphanumeric': 0.1452991452991453, 'fraction_numerical': 0.0010140518615094887, 'mean_word_length': 2.7850877192982457, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '396166', 'n_tokens_mistral': 2089, 'n_tokens_neox': 2068, 'n_words': 545}