<commit_before>use std::env;
use std::fs::{File, read_dir};
use std::io::Read;
use std::path::PathBuf;
use serde::de::Deserialize;
use serde_protobuf::descriptor::{Descriptors, MessageDescriptor};
use serde_protobuf::de::Deserializer;
use serde_value::Value;
use protobuf::{CodedInputStream, parse_from_reader};
use protobuf::descriptor::{FileDescriptorSet, FileDescriptorProto};

pub fn process_single(read: &mut Read) {
    let mut descriptors = Descriptors::new();
    let mut message_descriptors = Vec::new();

    for fdset_path in discover_fdsets() {
        let mut fdset_file = File::open(fdset_path.as_path()).unwrap();
        let fdset: FileDescriptorSet = parse_from_reader(&mut fdset_file).unwrap();
        descriptors.add_file_set_proto(&fdset);
        let fdset_protos: &[FileDescriptorProto] = fdset.get_file();
        for file_proto in fdset_protos.iter() {
            for message_proto in file_proto.get_message_type().iter() {
                println!("{:?}", message_proto);
                message_descriptors.push(MessageDescriptor::from_proto(&fdset_path.to_string_lossy().into_owned().as_str(), message_proto));
            }
        }
    }

    descriptors.resolve_refs();

    let byte_is = CodedInputStream::new(read);

    for md in message_descriptors {
        println!("ATTEMPT TO DESERIALIE WITH: {:?}\n\n", md);
        let mut deserializer = Deserializer::new(&descriptors, &md, byte_is);
        let value = Value::deserialize(&mut deserializer).unwrap();
        println!("{:?}", value);
    }
}

pub fn process_stream(read: &mut Read) {
    let mut stream = CodedInputStream::new(read);

    loop {
        match stream.eof() {
            Err(e) => panic!(e),
            Ok(true) => break,
            Ok(false) => break, 
            //todo: actually do deserialization here
        }
    }
}

fn discover_fdsets() -> Vec<PathBuf> {
    let mut fdset_files = Vec::new();

    let mut home = env::home_dir().expect("Could not find $HOME");
    home.push(".pq");
    let paths = read_dir(home.as_path()).unwrap();
<commit_msg>Read consumes message - somehow retain<commit_after>use std::env;
use std::fs::{File, read_dir};
use std::io::Read;
use std::path::PathBuf;
use serde::de::Deserialize;
use serde_protobuf::descriptor::{Descriptors, MessageDescriptor};
use serde_protobuf::de::Deserializer;
use serde_value::Value;
use protobuf::{CodedInputStream, parse_from_reader};
use protobuf::descriptor::{FileDescriptorSet, FileDescriptorProto};

pub fn process_single(read: &mut Read) {
    let mut descriptors = Descriptors::new();
    let mut message_descriptors = Vec::new();

    for fdset_path in discover_fdsets() {
        let mut fdset_file = File::open(fdset_path.as_path()).unwrap();
        let fdset: FileDescriptorSet = parse_from_reader(&mut fdset_file).unwrap();
        descriptors.add_file_set_proto(&fdset);
        let fdset_protos: &[FileDescriptorProto] = fdset.get_file();
        for file_proto in fdset_protos.iter() {
            for message_proto in file_proto.get_message_type().iter() {
                println!("{:?}", message_proto);
                message_descriptors.push(MessageDescriptor::from_proto(&fdset_path.to_string_lossy().into_owned().as_str(), message_proto));
            }
        }
    }

    descriptors.resolve_refs();

    for md in message_descriptors {
        let stream = CodedInputStream::new(read);
        println!("ATTEMPT TO DESERIALIE WITH\n:\t{:?}\n\n", md);
        let mut deserializer = Deserializer::new(&descriptors, &md, stream);
        let value = Value::deserialize(&mut deserializer).unwrap();
        println!("DESERIALIZED\n:\t{:?}\n\n", value);
    }
}

pub fn process_stream(read: &mut Read) {
    let mut stream = CodedInputStream::new(read);

    loop {
        match stream.eof() {
            Err(e) => panic!(e),
            Ok(true) => break,
            Ok(false) => break, 
            //todo: actually do deserialization here
        }
    }
}

fn discover_fdsets() -> Vec<PathBuf> {
    let mut fdset_files = Vec::new();

    let mut home = env::home_dir().expect("Could not find $HOME");
    home.push(".pq");
    let paths = read_dir(home.as_path()).unwrap();

{'fraction_non_alphanumeric': 0.14422158548233047, 'fraction_numerical': 0.0, 'mean_word_length': 3.461128860489883, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983911', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1320, 'n_words': 271}