<commit_before>/*
 * Copyright 2013-2016 Netherlands Forensic Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package nl.minvenj.nfi.ddrx;

import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

import nl.minvenj.nfi.ddrx.data.Environment;
import nl.minvenj.nfi.ddrx.expression.value.UnaryValueExpression;
import nl.minvenj.nfi.ddrx.expression.value.Value;
import nl.minvenj.nfi.ddrx.expression.value.ValueExpression;
import nl.minvenj.nfi.ddrx.expression.value.ValueOperation;

public class Callback {

    public static ValueExpression crc32(ValueExpression target) {
        return new UnaryValueExpression(target) {
            @Override
            public Value eval(final Environment env) {
                return _op.eval(env).operation(new ValueOperation() {
                    @Override
                    public Value execute(final Value value) {
                        CRC32 crc = new CRC32();
                        crc.update(value.getValue());
                        final long crcValue = crc.getValue();
                        return new Value(value.getEncoding().getByteOrder().apply(new byte[] { (byte)((crcValue & 0xff000000) >> 24),
                                                                                               (byte)((crcValue & 0xff0000) >> 16),
                                                                                               (byte)((crcValue & 0xff00) >> 8),
                                                                                               (byte)(crcValue & 0xff) }), value.getEncoding());
                    }
                });
            }
        };
    }
    
    public static ValueExpression inflate(ValueExpression target) {
        return new UnaryValueExpression(target) {
            @Override
            public Value eval(Environment env) {
                return _op.eval(env).operation(new ValueOperation() {
                    @Override
                    public Value execute(Value value) {
                        Inflater inf = new Inflater(true);
                        inf.setInput(value.getValue());
                        final byte[] tmp = new byte[value.getValue().length];
                        try {
                            final int size = inf.inflate(tmp);
                            final byte[] out = new byte[size];
                            System.arraycopy(tmp, 0, out, 0, size);
                            return new Value(out, value.getEncoding());
                        } catch (DataFormatException e) {
                            throw new RuntimeException(e);
                        }
                    }
                });
            }
        };
    }

}<commit_msg>When an exception occurs during decompression in callback, an empty value is returned instead of throwing a RuntimeException. Should be accompanied by error logging to notify the error source and potential skipping of data as a result of the returned empty value.:<commit_after>/*
 * Copyright 2013-2016 Netherlands Forensic Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package nl.minvenj.nfi.ddrx;

import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

import nl.minvenj.nfi.ddrx.data.Environment;
import nl.minvenj.nfi.ddrx.expression.value.UnaryValueExpression;
import nl.minvenj.nfi.ddrx.expression.value.Value;
import nl.minvenj.nfi.ddrx.expression.value.ValueExpression;
import nl.minvenj.nfi.ddrx.expression.value.ValueOperation;

public class Callback {

    public static ValueExpression crc32(ValueExpression target) {
        return new UnaryValueExpression(target) {
            @Override
            public Value eval(final Environment env) {
                return _op.eval(env).operation(new ValueOperation() {
                    @Override
                    public Value execute(final Value value) {
                        CRC32 crc = new CRC32();
                        crc.update(value.getValue());
                        final long crcValue = crc.getValue();
                        return new Value(value.getEncoding().getByteOrder().apply(new byte[] { (byte)((crcValue & 0xff000000) >> 24),
                                                                                               (byte)((crcValue & 0xff0000) >> 16),
                                                                                               (byte)((crcValue & 0xff00) >> 8),
                                                                                               (byte)(crcValue & 0xff) }), value.getEncoding());
                    }
                });
            }
        };
    }
    
    public static ValueExpression inflate(ValueExpression target) {
        return new UnaryValueExpression(target) {
            @Override
            public Value eval(Environment env) {
                return _op.eval(env).operation(new ValueOperation() {
                    @Override
                    public Value execute(Value value) {
                        Inflater inf = new Inflater(true);
                        inf.setInput(value.getValue());
                        final byte[] tmp = new byte[value.getValue().length];
                        try {
                            final int size = inf.inflate(tmp);
                            final byte[] out = new byte[size];
                            System.arraycopy(tmp, 0, out, 0, size);
                            return new Value(out, value.getEncoding());
                        } catch (DataFormatException e) {
                            return new Value(new byte[] {}, value.getEncoding());
                        }
                    }
                });
            }
        };
    }

}
{'fraction_non_alphanumeric': 0.0849194729136164, 'fraction_numerical': 0.012591508052708639, 'mean_word_length': 1.5132450331125828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5815619', 'n_tokens_mistral': 1642, 'n_tokens_neox': 1539, 'n_words': 461}