<commit_before>/*
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package com.github.sakserv.storm.scheme;

import backtype.storm.spout.Scheme;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

import java.util.Arrays;
import java.util.List;

public class AvroScheme implements Scheme {

        private static final long serialVersionUID = -2990121166902741545L;

        @Override
        public List<Object> deserialize(byte[] bytes) {
            String protocolVersion = Byte.toString(bytes[0]);
            String mutationTypeIdx = Byte.toString(bytes[1]);
            String mutationType = "";
            switch (mutationTypeIdx) {
                case "1":   mutationType = "insert";
                case "2":   mutationType = "update";
                case "3":   mutationType = "delete";
            }
            String msg = "";
            for (Byte theByte : bytes) {
                msg = msg + Byte.toString(theByte);
            }
            
            return new Values(protocolVersion, mutationType, msg);
        }

        @Override
            public Fields getOutputFields() {
            return new Fields("version", "type", "msg");
        }
}<commit_msg>more bytes conversion<commit_after>/*
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package com.github.sakserv.storm.scheme;

import backtype.storm.spout.Scheme;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

import java.util.Arrays;
import java.util.List;

public class AvroScheme implements Scheme {

        private static final long serialVersionUID = -2990121166902741545L;

        @Override
        public List<Object> deserialize(byte[] bytes) {
            String protocolVersion = Byte.toString(bytes[0]);
            String mutationTypeIdx = Byte.toString(bytes[1]);
            String mutationType = "";
            switch (mutationTypeIdx) {
                case "1":   mutationType = "insert"; break;
                case "2":   mutationType = "update"; break;
                case "3":   mutationType = "delete"; break;
            }
            
            
            byte[] newByteArray = Arrays.copyOfRange(bytes, 2, bytes.length - 1);
            String msg = new String(newByteArray);
            
            return new Values(protocolVersion, mutationType, msg);
        }

        @Override
            public Fields getOutputFields() {
            return new Fields("version", "type", "msg");
        }
}
{'fraction_non_alphanumeric': 0.09119496855345911, 'fraction_numerical': 0.016580903373356205, 'mean_word_length': 2.4136585365853658, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '416151', 'n_tokens_mistral': 948, 'n_tokens_neox': 899, 'n_words': 310}