<commit_before>#![feature(const_fn)]
#![feature(custom_derive, plugin)]
#![plugin(serde_macros)]

extern crate serde;
extern crate serde_json;

use serde::Deserialize;
use std::fmt;
use std::io::{self, Read};
use std::convert;
use std::collections::HashMap;
use std::net::{
    TcpListener,
    TcpStream,
    SocketAddr,
};
use std::sync::{
    self,
    Mutex,
    Arc,
};
use std::sync::mpsc::{
    channel,
    Sender,
    Receiver,
    TryRecvError,
};
use std::thread::{
    self,
    JoinHandle,
};

#[derive(Debug)]
pub enum Error {
    Io(io::Error),
    Json(serde_json::Error),
    Sender,
    Unimplemented,
    Impossible
}

impl convert::From<serde_json::Error> for Error {
    fn from(err: serde_json::Error) -> Error {
        match err {
            serde_json::Error::IoError(err) => Error::Io(err),
            err => Error::Json(err),
        }
    }
}

impl convert::From<io::Error> for Error {
    fn from(err: io::Error) -> Error {
        Error::Io(err)
    }
}

impl<T> convert::From<sync::mpsc::SendError<T>> for Error {
    fn from(_: sync::mpsc::SendError<T>) -> Error {
        Error::Sender
    }
}

pub type Result<T> = std::result::Result<T, Error>;

pub fn handle_conn<F, Request, Reply>(mut stream: TcpStream, f: F) -> Result<()>
    where Request: 'static + fmt::Debug + Send + serde::de::Deserialize + serde::ser::Serialize,
          Reply: 'static + fmt::Debug + serde::ser::Serialize,
          F: 'static + Clone + Serve<Request, Reply>
{
    let read_stream = try!(stream.try_clone());
    let mut de = serde_json::Deserializer::new(read_stream.bytes());
    let stream = Arc::new(Mutex::new(stream));
    loop {
        let request_packet: Packet<Request> = try!(Packet::deserialize(&mut de));
        match request_packet {
            Packet::Shutdown => {
                let stream = stream.clone();
                let mut my_stream = stream.lock().unwrap();
                try!(serde_json::to_writer(&mut *my_stream, &request_packet));
                break;
            },
            Packet::Message(id, message) => {
                let f = f.clone();
                let arc_stream = stream.clone();
                thread::spawn(move || {
                    let reply = f.serve(&message).unwrap();
                    let reply_packet = Packet::Message(id, reply);
                    let mut my_stream = arc_stream.lock().unwrap();
                    serde_json::to_writer(&mut *my_stream, &reply_packet).unwrap();
                });
            },
        }
    }
    Ok(())
}


pub struct Shutdown {
    tx: Sender<()>,
    join_handle: JoinHandle<()>,
    addr: SocketAddr,
}


impl Shutdown {
    fn wait(self) {
        self.join_handle.join().unwrap();
    }

    fn shutdown(self) {
        self.tx.send(()).expect(&line!().to_string());<commit_msg>Clean up those unused thingies<commit_after>#![feature(const_fn)]
#![feature(custom_derive, plugin)]
#![plugin(serde_macros)]

extern crate serde;
extern crate serde_json;

use serde::Deserialize;
use std::fmt;
use std::io::{self, Read};
use std::convert;
use std::collections::HashMap;
use std::net::{
    TcpListener,
    TcpStream,
    SocketAddr,
};
use std::sync::{
    self,
    Mutex,
    Arc,
};
use std::sync::mpsc::{
    channel,
    Sender,
    TryRecvError,
};
use std::thread::{
    self,
    JoinHandle,
};

#[derive(Debug)]
pub enum Error {
    Io(io::Error),
    Json(serde_json::Error),
    Sender,
    Unimplemented,
    Impossible
}

impl convert::From<serde_json::Error> for Error {
    fn from(err: serde_json::Error) -> Error {
        match err {
            serde_json::Error::IoError(err) => Error::Io(err),
            err => Error::Json(err),
        }
    }
}

impl convert::From<io::Error> for Error {
    fn from(err: io::Error) -> Error {
        Error::Io(err)
    }
}

impl<T> convert::From<sync::mpsc::SendError<T>> for Error {
    fn from(_: sync::mpsc::SendError<T>) -> Error {
        Error::Sender
    }
}

pub type Result<T> = std::result::Result<T, Error>;

pub fn handle_conn<F, Request, Reply>(stream: TcpStream, f: F) -> Result<()>
    where Request: 'static + fmt::Debug + Send + serde::de::Deserialize + serde::ser::Serialize,
          Reply: 'static + fmt::Debug + serde::ser::Serialize,
          F: 'static + Clone + Serve<Request, Reply>
{
    let read_stream = try!(stream.try_clone());
    let mut de = serde_json::Deserializer::new(read_stream.bytes());
    let stream = Arc::new(Mutex::new(stream));
    loop {
        let request_packet: Packet<Request> = try!(Packet::deserialize(&mut de));
        match request_packet {
            Packet::Shutdown => {
                let stream = stream.clone();
                let mut my_stream = stream.lock().unwrap();
                try!(serde_json::to_writer(&mut *my_stream, &request_packet));
                break;
            },
            Packet::Message(id, message) => {
                let f = f.clone();
                let arc_stream = stream.clone();
                thread::spawn(move || {
                    let reply = f.serve(&message).unwrap();
                    let reply_packet = Packet::Message(id, reply);
                    let mut my_stream = arc_stream.lock().unwrap();
                    serde_json::to_writer(&mut *my_stream, &reply_packet).unwrap();
                });
            },
        }
    }
    Ok(())
}


pub struct Shutdown {
    tx: Sender<()>,
    join_handle: JoinHandle<()>,
    addr: SocketAddr,
}


impl Shutdown {
    pub fn wait(self) {
        self.join_handle.join().unwrap();
    }

    pub fn shutdown(self) {
        self.tx.send(()).expect(&line!().to_string());
{'fraction_non_alphanumeric': 0.16166814551907718, 'fraction_numerical': 0.0, 'mean_word_length': 2.511526479750779, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1393540', 'n_tokens_mistral': 1832, 'n_tokens_neox': 1816, 'n_words': 371}