#
std/sys/pal/unix/
kernel_copy.rs

    1//! This module contains specializations that can offload `io::copy()` operations on file descriptor
    2//! containing types (`File`, `TcpStream`, etc.) to more efficient syscalls than `read(2)` and `write(2)`.
    3//!
    4//! Specialization is only applied to wholly std-owned types so that user code can't observe
    5//! that the `Read` and `Write` traits are not used.
    6//!
    7//! Since a copy operation involves a reader and writer side where each can consist of different types
    8//! and also involve generic wrappers (e.g. `Take`, `BufReader`) it is not practical to specialize
    9//! a single method on all possible combinations.
    10//!
    11//! Instead readers and writers are handled separately by the `CopyRead` and `CopyWrite` specialization
    12//! traits and then specialized on by the `Copier::copy` method.
    13//!
    14//! `Copier` uses the specialization traits to unpack the underlying file descriptors and
    15//! additional prerequisites and constraints imposed by the wrapper types.
    16//!
    17//! Once it has obtained all necessary pieces and brought any wrapper types into a state where they
    18//! can be safely bypassed it will attempt to use the `copy_file_range(2)`,
    19//! `sendfile(2)` or `splice(2)` syscalls to move data directly between file descriptors.
    20//! Since those syscalls have requirements that cannot be fully checked in advance it attempts
    21//! to use them one after another (guided by hints) to figure out which one works and
    22//! falls back to the generic read-write copy loop if none of them does.
    23//! Once a working syscall is found for a pair of file descriptors it will be called in a loop
    24//! until the copy operation is completed.
    25//!
    26//! Advantages of using these syscalls:
    27//!
    28//! * fewer context switches since reads and writes are coalesced into a single syscall
    29//!   and more bytes are transferred per syscall. This translates to higher throughput
    30//!   and fewer CPU cycles, at least for sufficiently large transfers to amortize the initial probing.
    31//! * `copy_file_range` creates reflink copies on CoW filesystems, thus moving less data and
    32//!   consuming less disk space
    33//! * `sendfile` and `splice` can perform zero-copy IO under some circumstances while
    34//!   a naive copy loop would move every byte through the CPU.
    35//!
    36//! Drawbacks:
    37//!
    38//! * copy operations smaller than the default buffer size can under some circumstances, especially
    39//!   on older kernels, incur more syscalls than the naive approach would. As mentioned above
    40//!   the syscall selection is guided by hints to minimize this possibility but they are not perfect.
    41//! * optimizations only apply to std types. If a user adds a custom wrapper type, e.g. to report
    42//!   progress, they can hit a performance cliff.
    43//! * complexity
    44
    45#[cfg(not(any(all(target_os = "linux", target_env = "gnu"), target_os = "hurd")))]
    46use libc::sendfile as sendfile64;
    47#[cfg(any(all(target_os = "linux", target_env = "gnu"), target_os = "hurd"))]
    48use libc::sendfile64;
    49use libc::{EBADF, EINVAL, ENOSYS, EOPNOTSUPP, EOVERFLOW, EPERM, EXDEV};
    50
    51use crate::cmp::min;
    52use crate::fs::{File, Metadata};
    53use crate::io::copy::generic_copy;
    54use crate::io::{
    55    BufRead, BufReader, BufWriter, Error, PipeReader, PipeWriter, Read, Result, StderrLock,
    56    StdinLock, StdoutLock, Take, Write,
    57};
    58use crate::mem::ManuallyDrop;
    59use crate::net::TcpStream;
    60use crate::os::unix::fs::FileTypeExt;
    61use crate::os::unix::io::{AsRawFd, FromRawFd, RawFd};
    62use crate::os::unix::net::UnixStream;
    63use crate::process::{ChildStderr, ChildStdin, ChildStdout};
    64use crate::ptr;
    65use crate::sync::atomic::{Atomic, AtomicBool, AtomicU8, Ordering};
    66use crate::sys::cvt;
    67use crate::sys::fs::CachedFileMetadata;
    68use crate::sys::weak::syscall;
    69
    70#[cfg(test)]
    71mod tests;
    72
    73pub(crate) fn copy_spec<R: Read + ?Sized, W: Write + ?Sized>(
    74    read: &mut R,
    75    write: &mut W,
    76) -> Result<u64> {
    77    let copier = Copier { read, write };
    78    SpecCopy::copy(copier)
    79}
    80
    81/// This type represents either the inferred `FileType` of a `RawFd` based on the source
    82/// type from which it was extracted or the actual metadata
    83///
    84/// The methods on this type only provide hints, due to `AsRawFd` and `FromRawFd` the inferred
    85/// type may be wrong.
    86enum FdMeta {
    87    Metadata(Metadata),
    88    Socket,
    89    Pipe,
    90    /// We don't have any metadata because the stat syscall failed
    91    NoneObtained,
    92}
    93
    94#[derive(PartialEq)]
    95enum FdHandle {
    96    Input,
    97    Output,
    98}
    99
    100impl FdMeta {
    101    fn maybe_fifo(&self) -> bool {
    102        match self {
    103            FdMeta::Metadata(meta) => meta.file_type().is_fifo(),
    104            FdMeta::Socket => false,
    105            FdMeta::Pipe => true,
    106            FdMeta::NoneObtained => true,
    107        }
    108    }
    109
    110    fn potential_sendfile_source(&self) -> bool {
    111        match self {
    112            // procfs erroneously shows 0 length on non-empty readable files.
    113            // and if a file is truly empty then a `read` syscall will determine that and skip the write syscall
    114            // thus there would be benefit from attempting sendfile
    115            FdMeta::Metadata(meta)
    116                if meta.file_type().is_file() && meta.len() > 0
    117                    || meta.file_type().is_block_device() =>
    118            {
    119                true
    120            }
    121            _ => false,
    122        }
    123    }
    124
    125    fn copy_file_range_candidate(&self, f: FdHandle) -> bool {
    126        match self {
    127            // copy_file_range will fail on empty procfs files. `read` can determine whether EOF has been reached
    128            // without extra cost and skip the write, thus there is no benefit in attempting copy_file_range
    129            FdMeta::Metadata(meta) if f == FdHandle::Input && meta.is_file() && meta.len() > 0 => {
    130                true
    131            }
    132            FdMeta::Metadata(meta) if f == FdHandle::Output && meta.is_file() => true,
    133            _ => false,
    134        }
    135    }
    136}
    137
    138/// Returns true either if changes made to the source after a sendfile/splice call won't become
    139/// visible in the sink or the source has explicitly opted into such behavior (e.g. by splicing
    140/// a file into a pipe, the pipe being the source in this case).
    141///
    142/// This will prevent File -> Pipe and File -> Socket splicing/sendfile optimizations to uphold
    143/// the Read/Write API semantics of io::copy.
    144///
    145/// Note: This is not 100% airtight, the caller can use the RawFd conversion methods to turn a
    146/// regular file into a TcpSocket which will be treated as a socket here without checking.
    147fn safe_kernel_copy(source: &FdMeta, sink: &FdMeta) -> bool {
    148    match (source, sink) {
    149        // Data arriving from a socket is safe because the sender can't modify the socket buffer.
    150        // Data arriving from a pipe is safe(-ish) because either the sender *copied*
    151        // the bytes into the pipe OR explicitly performed an operation that enables zero-copy,
    152        // thus promising not to modify the data later.
    153        (FdMeta::Socket, _) => true,
    154        (FdMeta::Pipe, _) => true,
    155        (FdMeta::Metadata(meta), _)
    156            if meta.file_type().is_fifo() || meta.file_type().is_socket() =>
    157        {
    158            true
    159        }
    160        // Data going into non-pipes/non-sockets is safe because the "later changes may become visible" issue
    161        // only happens for pages sitting in send buffers or pipes.
    162        (_, FdMeta::Metadata(meta))
    163            if !meta.file_type().is_fifo() && !meta.file_type().is_socket() =>
    164        {
    165            true
    166        }
    167        _ => false,
    168    }
    169}
    170
    171struct CopyParams(FdMeta, Option<RawFd>);
    172
    173struct Copier<'a, 'b, R: Read + ?Sized, W: Write + ?Sized> {
    174    read: &'a mut R,
    175    write: &'b mut W,
    176}
    177
    178trait SpecCopy {
    179    fn copy(self) -> Result<u64>;
    180}
    181
    182impl<R: Read + ?Sized, W: Write + ?Sized> SpecCopy for Copier<'_, '_, R, W> {
    183    default fn copy(self) -> Result<u64> {
    184        generic_copy(self.read, self.write)
    185    }
    186}
    187
    188impl<R: CopyRead, W: CopyWrite> SpecCopy for Copier<'_, '_, R, W> {
    189    fn copy(self) -> Result<u64> {
    190        let (reader, writer) = (self.read, self.write);
    191        let r_cfg = reader.properties();
    192        let w_cfg = writer.properties();
    193
    194        // before direct operations on file descriptors ensure that all source and sink buffers are empty
    195        let mut flush = || -> Result<u64> {
    196            let bytes = reader.drain_to(writer, u64::MAX)?;
    197            // BufWriter buffered bytes have already been accounted for in earlier write() calls
    198            writer.flush()?;
    199            Ok(bytes)
    200        };
    201
    202        let mut written = 0u64;
    203
    204        if let (CopyParams(input_meta, Some(readfd)), CopyParams(output_meta, Some(writefd))) =
    205            (r_cfg, w_cfg)
    206        {
    207            written += flush()?;
    208            let max_write = reader.min_limit();
    209
    210            if input_meta.copy_file_range_candidate(FdHandle::Input)
    211                && output_meta.copy_file_range_candidate(FdHandle::Output)
    212            {
    213                let result = copy_regular_files(readfd, writefd, max_write);
    214                result.update_take(reader);
    215
    216                match result {
    217                    CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written),
    218                    CopyResult::Error(e, _) => return Err(e),
    219                    CopyResult::Fallback(bytes) => written += bytes,
    220                }
    221            }
    222
    223            // on modern kernels sendfile can copy from any mmapable type (some but not all regular files and block devices)
    224            // to any writable file descriptor. On older kernels the writer side can only be a socket.
    225            // So we just try and fallback if needed.
    226            // If current file offsets + write sizes overflow it may also fail, we do not try to fix that and instead
    227            // fall back to the generic copy loop.
    228            if input_meta.potential_sendfile_source() && safe_kernel_copy(&input_meta, &output_meta)
    229            {
    230                let result = sendfile_splice(SpliceMode::Sendfile, readfd, writefd, max_write);
    231                result.update_take(reader);
    232
    233                match result {
    234                    CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written),
    235                    CopyResult::Error(e, _) => return Err(e),
    236                    CopyResult::Fallback(bytes) => written += bytes,
    237                }
    238            }
    239
    240            if (input_meta.maybe_fifo() || output_meta.maybe_fifo())
    241                && safe_kernel_copy(&input_meta, &output_meta)
    242            {
    243                let result = sendfile_splice(SpliceMode::Splice, readfd, writefd, max_write);
    244                result.update_take(reader);
    245
    246                match result {
    247                    CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written),
    248                    CopyResult::Error(e, _) => return Err(e),
    249                    CopyResult::Fallback(0) => { /* use the fallback below */ }
    250                    CopyResult::Fallback(_) => {
    251                        unreachable!("splice should not return > 0 bytes on the fallback path")
    252                    }
    253                }
    254            }
    255        }
    256
    257        // fallback if none of the more specialized syscalls wants to work with these file descriptors
    258        match generic_copy(reader, writer) {
    259            Ok(bytes) => Ok(bytes + written),
    260            err => err,
    261        }
    262    }
    263}
    264
    265#[rustc_specialization_trait]
    266trait CopyRead: Read {
    267    /// Implementations that contain buffers (i.e. `BufReader`) must transfer data from their internal
    268    /// buffers into `writer` until either the buffers are emptied or `limit` bytes have been
    269    /// transferred, whichever occurs sooner.
    270    /// If nested buffers are present the outer buffers must be drained first.
    271    ///
    272    /// This is necessary to directly bypass the wrapper types while preserving the data order
    273    /// when operating directly on the underlying file descriptors.
    274    fn drain_to<W: Write>(&mut self, _writer: &mut W, _limit: u64) -> Result<u64> {
    275        Ok(0)
    276    }
    277
    278    /// Updates `Take` wrappers to remove the number of bytes copied.
    279    fn taken(&mut self, _bytes: u64) {}
    280
    281    /// The minimum of the limit of all `Take<_>` wrappers, `u64::MAX` otherwise.
    282    /// This method does not account for data `BufReader` buffers and would underreport
    283    /// the limit of a `Take<BufReader<Take<_>>>` type. Thus its result is only valid
    284    /// after draining the buffers via `drain_to`.
    285    fn min_limit(&self) -> u64 {
    286        u64::MAX
    287    }
    288
    289    /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary.
    290    fn properties(&self) -> CopyParams;
    291}
    292
    293#[rustc_specialization_trait]
    294trait CopyWrite: Write {
    295    /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary.
    296    fn properties(&self) -> CopyParams;
    297}
    298
    299impl<T> CopyRead for &mut T
    300where
    301    T: CopyRead,
    302{
    303    fn drain_to<W: Write>(&mut self, writer: &mut W, limit: u64) -> Result<u64> {
    304        (**self).drain_to(writer, limit)
    305    }
    306
    307    fn taken(&mut self, bytes: u64) {
    308        (**self).taken(bytes);
    309    }
    310
    311    fn min_limit(&self) -> u64 {
    312        (**self).min_limit()
    313    }
    314
    315    fn properties(&self) -> CopyParams {
    316        (**self).properties()
    317    }
    318}
    319
    320impl<T> CopyWrite for &mut T
    321where
    322    T: CopyWrite,
    323{
    324    fn properties(&self) -> CopyParams {
    325        (**self).properties()
    326    }
    327}
    328
    329impl CopyRead for File {
    330    fn properties(&self) -> CopyParams {
    331        CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
    332    }
    333}
    334
    335impl CopyRead for &File {
    336    fn properties(&self) -> CopyParams {
    337        CopyParams(fd_to_meta(*self), Some(self.as_raw_fd()))
    338    }
    339}
    340
    341impl CopyWrite for File {
    342    fn properties(&self) -> CopyParams {
    343        CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
    344    }
    345}
    346
    347impl CopyWrite for &File {
    348    fn properties(&self) -> CopyParams {
    349        CopyParams(fd_to_meta(*self), Some(self.as_raw_fd()))
    350    }
    351}
    352
    353impl CopyRead for TcpStream {
    354    fn properties(&self) -> CopyParams {
    355        // avoid the stat syscall since we can be fairly sure it's a socket
    356        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    357    }
    358}
    359
    360impl CopyRead for &TcpStream {
    361    fn properties(&self) -> CopyParams {
    362        // avoid the stat syscall since we can be fairly sure it's a socket
    363        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    364    }
    365}
    366
    367impl CopyWrite for TcpStream {
    368    fn properties(&self) -> CopyParams {
    369        // avoid the stat syscall since we can be fairly sure it's a socket
    370        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    371    }
    372}
    373
    374impl CopyWrite for &TcpStream {
    375    fn properties(&self) -> CopyParams {
    376        // avoid the stat syscall since we can be fairly sure it's a socket
    377        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    378    }
    379}
    380
    381impl CopyRead for UnixStream {
    382    fn properties(&self) -> CopyParams {
    383        // avoid the stat syscall since we can be fairly sure it's a socket
    384        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    385    }
    386}
    387
    388impl CopyRead for &UnixStream {
    389    fn properties(&self) -> CopyParams {
    390        // avoid the stat syscall since we can be fairly sure it's a socket
    391        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    392    }
    393}
    394
    395impl CopyWrite for UnixStream {
    396    fn properties(&self) -> CopyParams {
    397        // avoid the stat syscall since we can be fairly sure it's a socket
    398        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    399    }
    400}
    401
    402impl CopyWrite for &UnixStream {
    403    fn properties(&self) -> CopyParams {
    404        // avoid the stat syscall since we can be fairly sure it's a socket
    405        CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
    406    }
    407}
    408
    409impl CopyRead for PipeReader {
    410    fn properties(&self) -> CopyParams {
    411        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    412    }
    413}
    414
    415impl CopyRead for &PipeReader {
    416    fn properties(&self) -> CopyParams {
    417        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    418    }
    419}
    420
    421impl CopyWrite for PipeWriter {
    422    fn properties(&self) -> CopyParams {
    423        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    424    }
    425}
    426
    427impl CopyWrite for &PipeWriter {
    428    fn properties(&self) -> CopyParams {
    429        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    430    }
    431}
    432
    433impl CopyWrite for ChildStdin {
    434    fn properties(&self) -> CopyParams {
    435        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    436    }
    437}
    438
    439impl CopyRead for ChildStdout {
    440    fn properties(&self) -> CopyParams {
    441        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    442    }
    443}
    444
    445impl CopyRead for ChildStderr {
    446    fn properties(&self) -> CopyParams {
    447        CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
    448    }
    449}
    450
    451impl CopyRead for StdinLock<'_> {
    452    fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
    453        let buf_reader = self.as_mut_buf();
    454        let buf = buf_reader.buffer();
    455        let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))];
    456        let bytes_drained = buf.len();
    457        writer.write_all(buf)?;
    458        buf_reader.consume(bytes_drained);
    459
    460        Ok(bytes_drained as u64)
    461    }
    462
    463    fn properties(&self) -> CopyParams {
    464        CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
    465    }
    466}
    467
    468impl CopyWrite for StdoutLock<'_> {
    469    fn properties(&self) -> CopyParams {
    470        CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
    471    }
    472}
    473
    474impl CopyWrite for StderrLock<'_> {
    475    fn properties(&self) -> CopyParams {
    476        CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
    477    }
    478}
    479
    480impl<T: CopyRead> CopyRead for Take<T> {
    481    fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
    482        let local_limit = self.limit();
    483        let combined_limit = min(outer_limit, local_limit);
    484        let bytes_drained = self.get_mut().drain_to(writer, combined_limit)?;
    485        // update limit since read() was bypassed
    486        self.set_limit(local_limit - bytes_drained);
    487
    488        Ok(bytes_drained)
    489    }
    490
    491    fn taken(&mut self, bytes: u64) {
    492        self.set_limit(self.limit() - bytes);
    493        self.get_mut().taken(bytes);
    494    }
    495
    496    fn min_limit(&self) -> u64 {
    497        min(Take::limit(self), self.get_ref().min_limit())
    498    }
    499
    500    fn properties(&self) -> CopyParams {
    501        self.get_ref().properties()
    502    }
    503}
    504
    505impl<T: ?Sized + CopyRead> CopyRead for BufReader<T> {
    506    fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
    507        let buf = self.buffer();
    508        let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))];
    509        let bytes = buf.len();
    510        writer.write_all(buf)?;
    511        self.consume(bytes);
    512
    513        let remaining = outer_limit - bytes as u64;
    514
    515        // in case of nested bufreaders we also need to drain the ones closer to the source
    516        let inner_bytes = self.get_mut().drain_to(writer, remaining)?;
    517
    518        Ok(bytes as u64 + inner_bytes)
    519    }
    520
    521    fn taken(&mut self, bytes: u64) {
    522        self.get_mut().taken(bytes);
    523    }
    524
    525    fn min_limit(&self) -> u64 {
    526        self.get_ref().min_limit()
    527    }
    528
    529    fn properties(&self) -> CopyParams {
    530        self.get_ref().properties()
    531    }
    532}
    533
    534impl<T: ?Sized + CopyWrite> CopyWrite for BufWriter<T> {
    535    fn properties(&self) -> CopyParams {
    536        self.get_ref().properties()
    537    }
    538}
    539
    540impl CopyRead for CachedFileMetadata {
    541    fn properties(&self) -> CopyParams {
    542        CopyParams(FdMeta::Metadata(self.1.clone()), Some(self.0.as_raw_fd()))
    543    }
    544}
    545
    546impl CopyWrite for CachedFileMetadata {
    547    fn properties(&self) -> CopyParams {
    548        CopyParams(FdMeta::Metadata(self.1.clone()), Some(self.0.as_raw_fd()))
    549    }
    550}
    551
    552fn fd_to_meta<T: AsRawFd>(fd: &T) -> FdMeta {
    553    let fd = fd.as_raw_fd();
    554    let file: ManuallyDrop<File> = ManuallyDrop::new(unsafe { File::from_raw_fd(fd) });
    555    match file.metadata() {
    556        Ok(meta) => FdMeta::Metadata(meta),
    557        Err(_) => FdMeta::NoneObtained,
    558    }
    559}
    560
    561pub(super) enum CopyResult {
    562    Ended(u64),
    563    Error(Error, u64),
    564    Fallback(u64),
    565}
    566
    567impl CopyResult {
    568    fn update_take(&self, reader: &mut impl CopyRead) {
    569        match *self {
    570            CopyResult::Fallback(bytes)
    571            | CopyResult::Ended(bytes)
    572            | CopyResult::Error(_, bytes) => reader.taken(bytes),
    573        }
    574    }
    575}
    576
    577/// Invalid file descriptor.
    578///
    579/// Valid file descriptors are guaranteed to be positive numbers (see `open()` manpage)
    580/// while negative values are used to indicate errors.
    581/// Thus -1 will never be overlap with a valid open file.
    582const INVALID_FD: RawFd = -1;
    583
    584/// Linux-specific implementation that will attempt to use copy_file_range for copy offloading.
    585/// As the name says, it only works on regular files.
    586///
    587/// Callers must handle fallback to a generic copy loop.
    588/// `Fallback` may indicate non-zero number of bytes already written
    589/// if one of the files' cursor +`max_len` would exceed u64::MAX (`EOVERFLOW`).
    590pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> CopyResult {
    591    use crate::cmp;
    592
    593    const NOT_PROBED: u8 = 0;
    594    const UNAVAILABLE: u8 = 1;
    595    const AVAILABLE: u8 = 2;
    596
    597    // Kernel prior to 4.5 don't have copy_file_range
    598    // We store the availability in a global to avoid unnecessary syscalls
    599    static HAS_COPY_FILE_RANGE: Atomic<u8> = AtomicU8::new(NOT_PROBED);
    600
    601    let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
    602        NOT_PROBED => false,
    603        UNAVAILABLE => return CopyResult::Fallback(0),
    604        _ => true,
    605    };
    606
    607    syscall!(
    608        fn copy_file_range(
    609            fd_in: libc::c_int,
    610            off_in: *mut libc::loff_t,
    611            fd_out: libc::c_int,
    612            off_out: *mut libc::loff_t,
    613            len: libc::size_t,
    614            flags: libc::c_uint,
    615        ) -> libc::ssize_t;
    616    );
    617
    618    fn probe_copy_file_range_support() -> u8 {
    619        // In some cases, we cannot determine availability from the first
    620        // `copy_file_range` call. In this case, we probe with an invalid file
    621        // descriptor so that the results are easily interpretable.
    622        match unsafe {
    623            cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
    624                .map_err(|e| e.raw_os_error())
    625        } {
    626            Err(Some(EPERM | ENOSYS)) => UNAVAILABLE,
    627            Err(Some(EBADF)) => AVAILABLE,
    628            Ok(_) => panic!("unexpected copy_file_range probe success"),
    629            // Treat other errors as the syscall
    630            // being unavailable.
    631            Err(_) => UNAVAILABLE,
    632        }
    633    }
    634
    635    let mut written = 0u64;
    636    while written < max_len {
    637        let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
    638        // cap to 1GB chunks in case u64::MAX is passed as max_len and the file has a non-zero seek position
    639        // this allows us to copy large chunks without hitting EOVERFLOW,
    640        // unless someone sets a file offset close to u64::MAX - 1GB, in which case a fallback would be required
    641        let bytes_to_copy = cmp::min(bytes_to_copy as usize, 0x4000_0000usize);
    642        let copy_result = unsafe {
    643            // We actually don't have to adjust the offsets,
    644            // because copy_file_range adjusts the file offset automatically
    645            cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
    646        };
    647
    648        if !have_probed && copy_result.is_ok() {
    649            have_probed = true;
    650            HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
    651        }
    652
    653        match copy_result {
    654            Ok(0) if written == 0 => {
    655                // fallback to work around several kernel bugs where copy_file_range will fail to
    656                // copy any bytes and return 0 instead of an error if
    657                // - reading virtual files from the proc filesystem which appear to have 0 size
    658                //   but are not empty. noted in coreutils to affect kernels at least up to 5.6.19.
    659                // - copying from an overlay filesystem in docker. reported to occur on fedora 32.
    660                return CopyResult::Fallback(0);
    661            }
    662            Ok(0) => return CopyResult::Ended(written), // reached EOF
    663            Ok(ret) => written += ret as u64,
    664            Err(err) => {
    665                return match err.raw_os_error() {
    666                    // when file offset + max_length > u64::MAX
    667                    Some(EOVERFLOW) => CopyResult::Fallback(written),
    668                    Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF))
    669                        if written == 0 =>
    670                    {
    671                        if !have_probed {
    672                            let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) {
    673                                // EPERM can indicate seccomp filters or an
    674                                // immutable file. To distinguish these
    675                                // cases we probe with invalid file
    676                                // descriptors which should result in EBADF
    677                                // if the syscall is supported and EPERM or
    678                                // ENOSYS if it's not available.
    679                                //
    680                                // For EOPNOTSUPP, see below. In the case of
    681                                // ENOSYS, we try to cover for faulty FUSE
    682                                // drivers.
    683                                probe_copy_file_range_support()
    684                            } else {
    685                                AVAILABLE
    686                            };
    687                            HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed);
    688                        }
    689
    690                        // Try fallback io::copy if either:
    691                        // - Kernel version is < 4.5 (ENOSYS¹)
    692                        // - Files are mounted on different fs (EXDEV)
    693                        // - copy_file_range is broken in various ways on RHEL/CentOS 7 (EOPNOTSUPP)
    694                        // - copy_file_range file is immutable or syscall is blocked by seccomp¹ (EPERM)
    695                        // - copy_file_range cannot be used with pipes or device nodes (EINVAL)
    696                        // - the writer fd was opened with O_APPEND (EBADF²)
    697                        // and no bytes were written successfully yet. (All these errnos should
    698                        // not be returned if something was already written, but they happen in
    699                        // the wild, see #91152.)
    700                        //
    701                        // ¹ these cases should be detected by the initial probe but we handle them here
    702                        //   anyway in case syscall interception changes during runtime
    703                        // ² actually invalid file descriptors would cause this too, but in that case
    704                        //   the fallback code path is expected to encounter the same error again
    705                        CopyResult::Fallback(0)
    706                    }
    707                    _ => CopyResult::Error(err, written),
    708                };
    709            }
    710        }
    711    }
    712    CopyResult::Ended(written)
    713}
    714
    715#[derive(PartialEq)]
    716enum SpliceMode {
    717    Sendfile,
    718    Splice,
    719}
    720
    721/// performs splice or sendfile between file descriptors
    722/// Does _not_ fall back to a generic copy loop.
    723fn sendfile_splice(mode: SpliceMode, reader: RawFd, writer: RawFd, len: u64) -> CopyResult {
    724    static HAS_SENDFILE: Atomic<bool> = AtomicBool::new(true);
    725    static HAS_SPLICE: Atomic<bool> = AtomicBool::new(true);
    726
    727    // Android builds use feature level 14, but the libc wrapper for splice is
    728    // gated on feature level 21+, so we have to invoke the syscall directly.
    729    #[cfg(target_os = "android")]
    730    syscall!(
    731        fn splice(
    732            srcfd: libc::c_int,
    733            src_offset: *const i64,
    734            dstfd: libc::c_int,
    735            dst_offset: *const i64,
    736            len: libc::size_t,
    737            flags: libc::c_int,
    738        ) -> libc::ssize_t;
    739    );
    740
    741    #[cfg(target_os = "linux")]
    742    use libc::splice;
    743
    744    match mode {
    745        SpliceMode::Sendfile if !HAS_SENDFILE.load(Ordering::Relaxed) => {
    746            return CopyResult::Fallback(0);
    747        }
    748        SpliceMode::Splice if !HAS_SPLICE.load(Ordering::Relaxed) => {
    749            return CopyResult::Fallback(0);
    750        }
    751        _ => (),
    752    }
    753
    754    let mut written = 0u64;
    755    while written < len {
    756        // according to its manpage that's the maximum size sendfile() will copy per invocation
    757        let chunk_size = crate::cmp::min(len - written, 0x7ffff000_u64) as usize;
    758
    759        let result = match mode {
    760            SpliceMode::Sendfile => {
    761                cvt(unsafe { sendfile64(writer, reader, ptr::null_mut(), chunk_size) })
    762            }
    763            SpliceMode::Splice => cvt(unsafe {
    764                splice(reader, ptr::null_mut(), writer, ptr::null_mut(), chunk_size, 0)
    765            }),
    766        };
    767
    768        match result {
    769            Ok(0) => break, // EOF
    770            Ok(ret) => written += ret as u64,
    771            Err(err) => {
    772                return match err.raw_os_error() {
    773                    Some(ENOSYS | EPERM) => {
    774                        // syscall not supported (ENOSYS)
    775                        // syscall is disallowed, e.g. by seccomp (EPERM)
    776                        match mode {
    777                            SpliceMode::Sendfile => HAS_SENDFILE.store(false, Ordering::Relaxed),
    778                            SpliceMode::Splice => HAS_SPLICE.store(false, Ordering::Relaxed),
    779                        }
    780                        assert_eq!(written, 0);
    781                        CopyResult::Fallback(0)
    782                    }
    783                    Some(EINVAL) => {
    784                        // splice/sendfile do not support this particular file descriptor (EINVAL)
    785                        assert_eq!(written, 0);
    786                        CopyResult::Fallback(0)
    787                    }
    788                    Some(os_err) if mode == SpliceMode::Sendfile && os_err == EOVERFLOW => {
    789                        CopyResult::Fallback(written)
    790                    }
    791                    _ => CopyResult::Error(err, written),
    792                };
    793            }
    794        }
    795    }
    796    CopyResult::Ended(written)
    797}