<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License.  You may obtain a copy
// of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
// License for the specific language governing permissions and limitations
// under the License.

use failure::Fallible;
use nix::unistd;
use nix::sys::{self, signal};
use signal_hook;
use std::cmp;
use std::fs;
use std::io::{self, Read};
use std::os::unix::io as unix_io;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
use std::time;
use std::thread;

/// Converts a `nix::Error` that we expect to carry an errno to an `io::Error`.
fn nix_to_io_error(err: nix::Error) -> io::Error {
    match err {
        nix::Error::Sys(errno) => io::Error::from_raw_os_error(errno as i32),
        e => panic!("Did not expect to get an error without an errno from a nix call: {:?}", e),
    }
}

/// A scope-owned file with support for multiple non-owner readers in different threads.
///
/// A `ShareableFile` object owns the file passed to it at construction time and will close the
/// underlying file handle when the object is dropped.
///
/// Concurrent views into this same file, obtained via the `reader` method, do not own the
/// file handle.  Such views must accept the fact that the handle can be closed at any time, a
/// condition that is simply exposed as if the handle reached EOF.  Concurrent views can safely be
/// moved across threads.
pub struct ShareableFile {
    /// Underlying file descriptor shared across all views of this file.
    fd: unix_io::RawFd,

    /// Write ends of the pipes used by `ShareableFileReader` to know if the file has been closed.
    watchers: Vec<unix_io::RawFd>,

    /// Whether the file descriptor has already been closed or not.
    ///
    /// In principle, we don't need this field: concurrent readers of this file will get their
    /// selects abruptly terminated with an error and we should be able to rely on `EBADF` to tell
    /// that this happened because of us closing the file descriptor.  But it seems better to track
    /// our own closed condition so that we can distinguish actual errors from expected errors.
    closed: Arc<AtomicBool>,
}

impl ShareableFile {
    /// Constructs a new `ShareableFile` from an open file and takes ownership of it.
    pub fn from(file: fs::File) -> ShareableFile {
        use std::os::unix::io::IntoRawFd;
        ShareableFile {
            fd: file.into_raw_fd(),
            watchers: vec!(),
            closed: Arc::from(AtomicBool::new(false)),
        }
    }

    /// Returns an unowned view of the file.
    ///
    /// Users of this file must accept that the file can be closed at any time by the owner.
    pub fn reader(&mut self) -> io::Result<ShareableFileReader> {
        let (notifier, watcher) = unistd::pipe().map_err(nix_to_io_error)?;
        self.watchers.push(watcher);
        Ok(ShareableFileReader {
            fd: self.fd,
            notifier: notifier,
            closed: self.closed.clone(),
        })
    }
}

impl Drop for ShareableFile {
    fn drop(&mut self) {
        debug!("Closing ShareableFile with fd {}", self.fd);

        // We are about to touch file handles used by other threads.  If those threads are blocked
        // on a select call, the call may return an error on some systems due to the closed file
        // descriptors.  If those threads are about to issue a read after a select, the read will
        // fail with a bad file descriptor.  Prepare them about these potential failure conditions
        // before we actually touch anything.
        self.closed.store(true, Ordering::SeqCst);

        for watcher in &self.watchers {
            if let Err(e) = unistd::write(*watcher, &[0]) {
                // This write to a pipe we control really should not have failed.  If it did there
                // is not much we can do other than log an error.  We may get stuck threads on exit
                // though...
                warn!("Failed to tell ShareableFileReader with handle {} of close: {}", *watcher, e)
            }
            if let Err(e) = unistd::close(*watcher) {
                // Closing should really not have failed, but if it did, it does not hurt and there
                // is nothing we can do anyway.
                warn!("Failed to close pipe write end with handle {}: {}", *watcher, e)
            }
        }

        if let Err(e) = unistd::close(self.fd) {
            warn!("Failed to close fd {}: {}", self.fd, e);
        }
    }
}
<commit_msg>Silence EPIPE warning when closing reconfigs<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License.  You may obtain a copy
// of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
// License for the specific language governing permissions and limitations
// under the License.

use failure::Fallible;
use nix::errno::Errno;
use nix::unistd;
use nix::sys::{self, signal};
use signal_hook;
use std::cmp;
use std::fs;
use std::io::{self, Read};
use std::os::unix::io as unix_io;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
use std::time;
use std::thread;

/// Converts a `nix::Error` that we expect to carry an errno to an `io::Error`.
fn nix_to_io_error(err: nix::Error) -> io::Error {
    match err {
        nix::Error::Sys(errno) => io::Error::from_raw_os_error(errno as i32),
        e => panic!("Did not expect to get an error without an errno from a nix call: {:?}", e),
    }
}

/// A scope-owned file with support for multiple non-owner readers in different threads.
///
/// A `ShareableFile` object owns the file passed to it at construction time and will close the
/// underlying file handle when the object is dropped.
///
/// Concurrent views into this same file, obtained via the `reader` method, do not own the
/// file handle.  Such views must accept the fact that the handle can be closed at any time, a
/// condition that is simply exposed as if the handle reached EOF.  Concurrent views can safely be
/// moved across threads.
pub struct ShareableFile {
    /// Underlying file descriptor shared across all views of this file.
    fd: unix_io::RawFd,

    /// Write ends of the pipes used by `ShareableFileReader` to know if the file has been closed.
    watchers: Vec<unix_io::RawFd>,

    /// Whether the file descriptor has already been closed or not.
    ///
    /// In principle, we don't need this field: concurrent readers of this file will get their
    /// selects abruptly terminated with an error and we should be able to rely on `EBADF` to tell
    /// that this happened because of us closing the file descriptor.  But it seems better to track
    /// our own closed condition so that we can distinguish actual errors from expected errors.
    closed: Arc<AtomicBool>,
}

impl ShareableFile {
    /// Constructs a new `ShareableFile` from an open file and takes ownership of it.
    pub fn from(file: fs::File) -> ShareableFile {
        use std::os::unix::io::IntoRawFd;
        ShareableFile {
            fd: file.into_raw_fd(),
            watchers: vec!(),
            closed: Arc::from(AtomicBool::new(false)),
        }
    }

    /// Returns an unowned view of the file.
    ///
    /// Users of this file must accept that the file can be closed at any time by the owner.
    pub fn reader(&mut self) -> io::Result<ShareableFileReader> {
        let (notifier, watcher) = unistd::pipe().map_err(nix_to_io_error)?;
        self.watchers.push(watcher);
        Ok(ShareableFileReader {
            fd: self.fd,
            notifier: notifier,
            closed: self.closed.clone(),
        })
    }
}

impl Drop for ShareableFile {
    fn drop(&mut self) {
        debug!("Closing ShareableFile with fd {}", self.fd);

        // We are about to touch file handles used by other threads.  If those threads are blocked
        // on a select call, the call may return an error on some systems due to the closed file
        // descriptors.  If those threads are about to issue a read after a select, the read will
        // fail with a bad file descriptor.  Prepare them about these potential failure conditions
        // before we actually touch anything.
        self.closed.store(true, Ordering::SeqCst);

        for watcher in &self.watchers {
            if let Err(e) = unistd::write(*watcher, &[0]) {
                // This write to a pipe we control really should not have failed.  If it did there
                // is not much we can do other than log an error.  We may get stuck threads on exit
                // though...
                //
                // TODO(jmmv): We expect the write to fail if we had any ShareableFileReader that
                // has already been dropped, as that drop would have closed the read end of the
                // pipe.  Note that this means that, if we create/drop ShareableFileReaders non-stop
                // for this ShareableFile, we'll exhaust the open file descriptors because we are
                // leaking the write ends of these pipes.  This should be fixed, but it's not a big
                // deal because we don't do this in sandboxfs.
                if e.as_errno().expect("Must have been a system error") != Errno::EPIPE {
                    warn!("Failed to tell ShareableFileReader with handle {} of close: {}",
                        *watcher, e)
                }
            }
            if let Err(e) = unistd::close(*watcher) {
                // Closing should really not have failed, but if it did, it does not hurt and there
                // is nothing we can do anyway.
                warn!("Failed to close pipe write end with handle {}: {}", *watcher, e)
            }
        }

        if let Err(e) = unistd::close(self.fd) {
            warn!("Failed to close fd {}: {}", self.fd, e);
        }
    }
}

{'fraction_non_alphanumeric': 0.10119215244532057, 'fraction_numerical': 0.002065145968271848, 'mean_word_length': 2.4794252122795557, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695541', 'n_tokens_mistral': 2948, 'n_tokens_neox': 2894, 'n_words': 1302}