diff mbox series

[1/4] csu: randomize location of TCB

Message ID 20201128115945.42732-2-toiwoton@gmail.com
State New
Headers show
Series Improved ASLR | expand

Commit Message

Topi Miettinen Nov. 28, 2020, 11:59 a.m. UTC
Use mmap() for allocating TCB except if instructed by tunable
glibc.malloc.use_sbrk. This makes the location of TCB random instead
of always staying predictably next to data segment. When using mmap(),
improve the logic so that allocation of TCB can be assumed to fail
insted of segfaulting.

--
v2: introduce a tunable to use sbrk()
v3:
- refactor mmap() (Adhemerval Zanella)
- rename mmap_internal to mmap_noerrno
---
 csu/libc-tls.c                               | 40 ++++++++++++++++----
 elf/dl-tunables.list                         |  7 ++++
 include/sys/mman.h                           |  5 +++
 manual/tunables.texi                         |  5 +++
 sysdeps/mach/hurd/dl-sysdep.c                | 18 +++++++--
 sysdeps/unix/sysv/linux/mmap.c               | 30 ++++++++++++---
 sysdeps/unix/sysv/linux/mmap64.c             | 23 ++++++++---
 sysdeps/unix/sysv/linux/mmap_internal.h      |  2 +-
 sysdeps/unix/sysv/linux/s390/mmap_internal.h |  2 +-
 9 files changed, 109 insertions(+), 23 deletions(-)

Comments

Topi Miettinen Nov. 29, 2020, 2:13 p.m. UTC | #1
On 28.11.2020 13.59, Topi Miettinen wrote:
> Use mmap() for allocating TCB except if instructed by tunable
> glibc.malloc.use_sbrk. This makes the location of TCB random instead
> of always staying predictably next to data segment. When using mmap(),
> improve the logic so that allocation of TCB can be assumed to fail
> insted of segfaulting.

insted -> instead

> 
> --
> v2: introduce a tunable to use sbrk()
> v3:
> - refactor mmap() (Adhemerval Zanella)
> - rename mmap_internal to mmap_noerrno
> ---
>   csu/libc-tls.c                               | 40 ++++++++++++++++----
>   elf/dl-tunables.list                         |  7 ++++
>   include/sys/mman.h                           |  5 +++
>   manual/tunables.texi                         |  5 +++
>   sysdeps/mach/hurd/dl-sysdep.c                | 18 +++++++--
>   sysdeps/unix/sysv/linux/mmap.c               | 30 ++++++++++++---
>   sysdeps/unix/sysv/linux/mmap64.c             | 23 ++++++++---
>   sysdeps/unix/sysv/linux/mmap_internal.h      |  2 +-
>   sysdeps/unix/sysv/linux/s390/mmap_internal.h |  2 +-
>   9 files changed, 109 insertions(+), 23 deletions(-)
> 
> diff --git a/csu/libc-tls.c b/csu/libc-tls.c
> index c3589f0a7d..0cb6cb2e42 100644
> --- a/csu/libc-tls.c
> +++ b/csu/libc-tls.c
> @@ -25,11 +25,18 @@
>   #include <sys/param.h>
>   #include <array_length.h>
>   #include <list.h>
> +#include <sys/mman.h>
> +#include <sysdep.h>
>   
>   #ifdef SHARED
>    #error makefile bug, this file is for static only
>   #endif
>   
> +#if HAVE_TUNABLES
> +# define TUNABLE_NAMESPACE malloc
> +#endif
> +#include <elf/dl-tunables.h>
> +
>   dtv_t _dl_static_dtv[2 + TLS_SLOTINFO_SURPLUS];
>   
>   
> @@ -135,26 +142,45 @@ __libc_setup_tls (void)
>   
>     /* We have to set up the TCB block which also (possibly) contains
>        'errno'.  Therefore we avoid 'malloc' which might touch 'errno'.
> -     Instead we use 'sbrk' which would only uses 'errno' if it fails.
> -     In this case we are right away out of memory and the user gets
> -     what she/he deserves.  */
> +     Instead we use '__mmap_noerrno' (when available) which does not
> +     use 'errno', except if instructed by tunable
> +     glibc.malloc.use_sbrk to use 'sbrk()' instead. If 'sbrk()' fails,
> +     it will access 'errno' with catastrophic results. */
> +
> +  size_t tlsblock_size;
>   #if TLS_TCB_AT_TP
>     /* Align the TCB offset to the maximum alignment, as
>        _dl_allocate_tls_storage (in elf/dl-tls.c) does using __libc_memalign
>        and dl_tls_static_align.  */
>     tcb_offset = roundup (memsz + GLRO(dl_tls_static_surplus), max_align);
> -  tlsblock = __sbrk (tcb_offset + TLS_INIT_TCB_SIZE + max_align);
> +  tlsblock_size = tcb_offset + TLS_INIT_TCB_SIZE + max_align;
>   #elif TLS_DTV_AT_TP
>     tcb_offset = roundup (TLS_INIT_TCB_SIZE, align ?: 1);
> -  tlsblock = __sbrk (tcb_offset + memsz + max_align
> -		     + TLS_PRE_TCB_SIZE + GLRO(dl_tls_static_surplus));
> -  tlsblock += TLS_PRE_TCB_SIZE;
> +  tlsblock_size = tcb_offset + memsz + max_align
> +	  + TLS_PRE_TCB_SIZE + GLRO(dl_tls_static_surplus);
>   #else
>     /* In case a model with a different layout for the TCB and DTV
>        is defined add another #elif here and in the following #ifs.  */
>   # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
>   #endif
>   
> +#if HAVE_TUNABLES
> +  if (!TUNABLE_GET (use_sbrk, int32_t, NULL))
> +    {
> +      int error = 0;
> +      tlsblock = __mmap_noerrno (NULL, tlsblock_size, PROT_READ | PROT_WRITE,

Forgot to add ALIGN_UP (tlsblock_size, GLRO(dl_pagesize)).

> +				 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, &error);
> +      if (error || tlsblock == MAP_FAILED)
> +        _startup_fatal ("Cannot allocate TCB");
> +    }
> +  else
> +#endif
> +    tlsblock = __sbrk (tlsblock_size);
> +
> +#if TLS_DTV_AT_TP
> +  tlsblock += TLS_PRE_TCB_SIZE;
> +#endif
> +
>     /* Align the TLS block.  */
>     tlsblock = (void *) (((uintptr_t) tlsblock + max_align - 1)
>   		       & ~(max_align - 1));

This probably is not necessary for the mmap() case since it returns page 
aligned pointers, except if the addition above tlsblock += 
TLS_PRE_TCB_SIZE makes the alignment worse than needed.

> diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list
> index e1d8225128..777ebee788 100644
> --- a/elf/dl-tunables.list
> +++ b/elf/dl-tunables.list
> @@ -91,6 +91,13 @@ glibc {
>         minval: 0
>         security_level: SXID_IGNORE
>       }
> +    use_sbrk {
> +      type: INT_32
> +      minval: 0
> +      maxval: 1
> +      default: 0
> +      security_level: SXID_IGNORE
> +    }
>     }
>     cpu {
>       hwcap_mask {
> diff --git a/include/sys/mman.h b/include/sys/mman.h
> index 503edaae88..d2fc5608c3 100644
> --- a/include/sys/mman.h
> +++ b/include/sys/mman.h
> @@ -22,6 +22,11 @@ extern void *__mremap (void *__addr, size_t __old_len,
>   		       size_t __new_len, int __flags, ...);
>   libc_hidden_proto (__mremap)
>   
> +/* Internal version of mmap() which doesn't attempt to access errno */
> +extern void *__mmap_noerrno (void *addr, size_t len, int prot, int flags,
> +			     int fd, off_t offset, int *err);
> +libc_hidden_proto (__mmap_noerrno)
> +
>   # if IS_IN (rtld)
>   #  include <dl-mman.h>
>   # endif
> diff --git a/manual/tunables.texi b/manual/tunables.texi
> index d72d7a5ec0..46132900e3 100644
> --- a/manual/tunables.texi
> +++ b/manual/tunables.texi
> @@ -227,6 +227,11 @@ pointer, so add 4 on 32-bit systems or 8 on 64-bit systems to the size
>   passed to @code{malloc} for the largest bin size to enable.
>   @end deftp
>   
> +@deftp Tunable glibc.malloc.use_sbrk
> +A value of 1 instructs @theglibc{} to use @code{sbrk()} for memory
> +allocation instead of @code{mmap()}.
> +@end deftp
> +
>   @node Dynamic Linking Tunables
>   @section Dynamic Linking Tunables
>   @cindex dynamic linking tunables
> diff --git a/sysdeps/mach/hurd/dl-sysdep.c b/sysdeps/mach/hurd/dl-sysdep.c
> index 370495710e..40e2919b9d 100644
> --- a/sysdeps/mach/hurd/dl-sysdep.c
> +++ b/sysdeps/mach/hurd/dl-sysdep.c
> @@ -482,9 +482,9 @@ __libc_lseek64 (int fd, off64_t offset, int whence)
>     return offset;
>   }
>   
> -check_no_hidden(__mmap);
> +check_no_hidden(__mmap_noerrno);
>   void *weak_function
> -__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
> +__mmap_noerrno (void *addr, size_t len, int prot, int flags, int fd, off_t offset, int *error)
>   {
>     error_t err;
>     vm_prot_t vmprot;
> @@ -541,10 +541,22 @@ __mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
>       __mach_port_deallocate (__mach_task_self (), memobj_rd);
>   
>     if (err)
> -    return __hurd_fail (err), MAP_FAILED;
> +    *error = err;
>     return (void *) mapaddr;
>   }
>   
> +check_no_hidden(__mmap);
> +void *weak_function
> +__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
> +{
> +  int err = 0;
> +
> +  void *r = __mmap_noerrno (addr, len, prot, flags, fd, offset, &err);
> +  if (err)
> +    return __hurd_fail (err), MAP_FAILED;
> +  return r;
> +}
> +
>   check_no_hidden(__fstat64);
>   int weak_function
>   __fstat64 (int fd, struct stat64 *buf)
> diff --git a/sysdeps/unix/sysv/linux/mmap.c b/sysdeps/unix/sysv/linux/mmap.c
> index 22f276bb14..19eca3fe18 100644
> --- a/sysdeps/unix/sysv/linux/mmap.c
> +++ b/sysdeps/unix/sysv/linux/mmap.c
> @@ -31,20 +31,38 @@
>   # endif
>   
>   void *
> -__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
> +__mmap_noerrno (void *addr, size_t len, int prot, int flags, int fd, off_t offset, int *err)
>   {
>     MMAP_CHECK_PAGE_UNIT ();
>   
>     if (offset & MMAP_OFF_LOW_MASK)
> -    return (void *) INLINE_SYSCALL_ERROR_RETURN_VALUE (EINVAL);
> +    return (void *) -EINVAL;
>   
>   #ifdef __NR_mmap2
> -  return (void *) MMAP_CALL (mmap2, addr, len, prot, flags, fd,
> -			     offset / (uint32_t) MMAP2_PAGE_UNIT);
> +  long int r = MMAP_CALL (mmap2, addr, len, prot, flags, fd,
> +			  offset / (uint32_t) MMAP2_PAGE_UNIT);
>   #else
> -  return (void *) MMAP_CALL (mmap, addr, len, prot, flags, fd,
> -			     MMAP_ADJUST_OFFSET (offset));
> +  long int r = MMAP_CALL (mmap, addr, len, prot, flags, fd,
> +			  MMAP_ADJUST_OFFSET (offset));
>   #endif
> +  if (INTERNAL_SYSCALL_ERROR_P (r))
> +    {
> +      *err = (INTERNAL_SYSCALL_ERRNO (r));
> +      return MAP_FAILED;
> +    }
> +  return (void*) r;
> +}
> +libc_hidden_def (__mmap_noerrno)
> +
> +void *
> +__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
> +{
> +  int error = 0;
> +
> +  void *r = __mmap_noerrno (addr, len, prot, flags, fd, offset, &error);
> +  if (error)
> +    __set_errno(error);

I suppose errno should be cleared if there's no error, so remove the check.

-Topi

> +  return r;
>   }
>   weak_alias (__mmap, mmap)
>   libc_hidden_def (__mmap)
> diff --git a/sysdeps/unix/sysv/linux/mmap64.c b/sysdeps/unix/sysv/linux/mmap64.c
> index 8074deb466..3d6557734b 100644
> --- a/sysdeps/unix/sysv/linux/mmap64.c
> +++ b/sysdeps/unix/sysv/linux/mmap64.c
> @@ -44,25 +44,38 @@
>   #endif
>   
>   void *
> -__mmap64 (void *addr, size_t len, int prot, int flags, int fd, off64_t offset)
> +__mmap64_noerrno (void *addr, size_t len, int prot, int flags, int fd, off64_t offset, int *err)
>   {
>     MMAP_CHECK_PAGE_UNIT ();
>   
>     if (offset & MMAP_OFF_MASK)
> -    return (void *) INLINE_SYSCALL_ERROR_RETURN_VALUE (EINVAL);
> +    return (void *) -EINVAL;
>   
>     MMAP_PREPARE (addr, len, prot, flags, fd, offset);
>   #ifdef __NR_mmap2
> -  return (void *) MMAP_CALL (mmap2, addr, len, prot, flags, fd,
> -			     (off_t) (offset / MMAP2_PAGE_UNIT));
> +  long int r = MMAP_CALL (mmap2, addr, len, prot, flags, fd,
> +			  (off_t) (offset / MMAP2_PAGE_UNIT));
>   #else
> -  return (void *) MMAP_CALL (mmap, addr, len, prot, flags, fd, offset);
> +  long int r = MMAP_CALL (mmap, addr, len, prot, flags, fd, offset);
>   #endif
> +  if (INTERNAL_SYSCALL_ERROR_P (r))
> +    {
> +      *err = INTERNAL_SYSCALL_ERRNO (r);
> +      return MAP_FAILED;
> +    }
> +  return (void *) r;
> +}
> +
> +void *
> +__mmap64 (void *addr, size_t len, int prot, int flags, int fd, off64_t offset)
> +{
> +  return __mmap64_noerrno (addr, len, prot, flags, fd, offset, &errno);
>   }
>   weak_alias (__mmap64, mmap64)
>   libc_hidden_def (__mmap64)
>   
>   #ifdef __OFF_T_MATCHES_OFF64_T
> +weak_alias (__mmap64_noerrno, __mmap_noerrno)
>   weak_alias (__mmap64, mmap)
>   weak_alias (__mmap64, __mmap)
>   libc_hidden_def (__mmap)
> diff --git a/sysdeps/unix/sysv/linux/mmap_internal.h b/sysdeps/unix/sysv/linux/mmap_internal.h
> index d53f0c642a..5386b5eb63 100644
> --- a/sysdeps/unix/sysv/linux/mmap_internal.h
> +++ b/sysdeps/unix/sysv/linux/mmap_internal.h
> @@ -43,7 +43,7 @@ static uint64_t page_unit;
>   /* An architecture may override this.  */
>   #ifndef MMAP_CALL
>   # define MMAP_CALL(__nr, __addr, __len, __prot, __flags, __fd, __offset) \
> -  INLINE_SYSCALL_CALL (__nr, __addr, __len, __prot, __flags, __fd, __offset)
> +  INTERNAL_SYSCALL_CALL (__nr, __addr, __len, __prot, __flags, __fd, __offset)
>   #endif
>   
>   #endif /* MMAP_INTERNAL_LINUX_H  */
> diff --git a/sysdeps/unix/sysv/linux/s390/mmap_internal.h b/sysdeps/unix/sysv/linux/s390/mmap_internal.h
> index 2884f2844b..d2289f311c 100644
> --- a/sysdeps/unix/sysv/linux/s390/mmap_internal.h
> +++ b/sysdeps/unix/sysv/linux/s390/mmap_internal.h
> @@ -24,7 +24,7 @@
>       long int __args[6] = { (long int) (__addr), (long int) (__len),	\
>   			   (long int) (__prot), (long int) (__flags),	\
>   			   (long int) (__fd), (long int) (__offset) };	\
> -    INLINE_SYSCALL_CALL (__nr, __args);					\
> +    INTERNAL_SYSCALL_CALL (__nr, __args);					\
>     })
>   
>   #include_next <mmap_internal.h>
>
diff mbox series

Patch

diff --git a/csu/libc-tls.c b/csu/libc-tls.c
index c3589f0a7d..0cb6cb2e42 100644
--- a/csu/libc-tls.c
+++ b/csu/libc-tls.c
@@ -25,11 +25,18 @@ 
 #include <sys/param.h>
 #include <array_length.h>
 #include <list.h>
+#include <sys/mman.h>
+#include <sysdep.h>
 
 #ifdef SHARED
  #error makefile bug, this file is for static only
 #endif
 
+#if HAVE_TUNABLES
+# define TUNABLE_NAMESPACE malloc
+#endif
+#include <elf/dl-tunables.h>
+
 dtv_t _dl_static_dtv[2 + TLS_SLOTINFO_SURPLUS];
 
 
@@ -135,26 +142,45 @@  __libc_setup_tls (void)
 
   /* We have to set up the TCB block which also (possibly) contains
      'errno'.  Therefore we avoid 'malloc' which might touch 'errno'.
-     Instead we use 'sbrk' which would only uses 'errno' if it fails.
-     In this case we are right away out of memory and the user gets
-     what she/he deserves.  */
+     Instead we use '__mmap_noerrno' (when available) which does not
+     use 'errno', except if instructed by tunable
+     glibc.malloc.use_sbrk to use 'sbrk()' instead. If 'sbrk()' fails,
+     it will access 'errno' with catastrophic results. */
+
+  size_t tlsblock_size;
 #if TLS_TCB_AT_TP
   /* Align the TCB offset to the maximum alignment, as
      _dl_allocate_tls_storage (in elf/dl-tls.c) does using __libc_memalign
      and dl_tls_static_align.  */
   tcb_offset = roundup (memsz + GLRO(dl_tls_static_surplus), max_align);
-  tlsblock = __sbrk (tcb_offset + TLS_INIT_TCB_SIZE + max_align);
+  tlsblock_size = tcb_offset + TLS_INIT_TCB_SIZE + max_align;
 #elif TLS_DTV_AT_TP
   tcb_offset = roundup (TLS_INIT_TCB_SIZE, align ?: 1);
-  tlsblock = __sbrk (tcb_offset + memsz + max_align
-		     + TLS_PRE_TCB_SIZE + GLRO(dl_tls_static_surplus));
-  tlsblock += TLS_PRE_TCB_SIZE;
+  tlsblock_size = tcb_offset + memsz + max_align
+	  + TLS_PRE_TCB_SIZE + GLRO(dl_tls_static_surplus);
 #else
   /* In case a model with a different layout for the TCB and DTV
      is defined add another #elif here and in the following #ifs.  */
 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 #endif
 
+#if HAVE_TUNABLES
+  if (!TUNABLE_GET (use_sbrk, int32_t, NULL))
+    {
+      int error = 0;
+      tlsblock = __mmap_noerrno (NULL, tlsblock_size, PROT_READ | PROT_WRITE,
+				 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, &error);
+      if (error || tlsblock == MAP_FAILED)
+        _startup_fatal ("Cannot allocate TCB");
+    }
+  else
+#endif
+    tlsblock = __sbrk (tlsblock_size);
+
+#if TLS_DTV_AT_TP
+  tlsblock += TLS_PRE_TCB_SIZE;
+#endif
+
   /* Align the TLS block.  */
   tlsblock = (void *) (((uintptr_t) tlsblock + max_align - 1)
 		       & ~(max_align - 1));
diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list
index e1d8225128..777ebee788 100644
--- a/elf/dl-tunables.list
+++ b/elf/dl-tunables.list
@@ -91,6 +91,13 @@  glibc {
       minval: 0
       security_level: SXID_IGNORE
     }
+    use_sbrk {
+      type: INT_32
+      minval: 0
+      maxval: 1
+      default: 0
+      security_level: SXID_IGNORE
+    }
   }
   cpu {
     hwcap_mask {
diff --git a/include/sys/mman.h b/include/sys/mman.h
index 503edaae88..d2fc5608c3 100644
--- a/include/sys/mman.h
+++ b/include/sys/mman.h
@@ -22,6 +22,11 @@  extern void *__mremap (void *__addr, size_t __old_len,
 		       size_t __new_len, int __flags, ...);
 libc_hidden_proto (__mremap)
 
+/* Internal version of mmap() which doesn't attempt to access errno */
+extern void *__mmap_noerrno (void *addr, size_t len, int prot, int flags,
+			     int fd, off_t offset, int *err);
+libc_hidden_proto (__mmap_noerrno)
+
 # if IS_IN (rtld)
 #  include <dl-mman.h>
 # endif
diff --git a/manual/tunables.texi b/manual/tunables.texi
index d72d7a5ec0..46132900e3 100644
--- a/manual/tunables.texi
+++ b/manual/tunables.texi
@@ -227,6 +227,11 @@  pointer, so add 4 on 32-bit systems or 8 on 64-bit systems to the size
 passed to @code{malloc} for the largest bin size to enable.
 @end deftp
 
+@deftp Tunable glibc.malloc.use_sbrk
+A value of 1 instructs @theglibc{} to use @code{sbrk()} for memory
+allocation instead of @code{mmap()}.
+@end deftp
+
 @node Dynamic Linking Tunables
 @section Dynamic Linking Tunables
 @cindex dynamic linking tunables
diff --git a/sysdeps/mach/hurd/dl-sysdep.c b/sysdeps/mach/hurd/dl-sysdep.c
index 370495710e..40e2919b9d 100644
--- a/sysdeps/mach/hurd/dl-sysdep.c
+++ b/sysdeps/mach/hurd/dl-sysdep.c
@@ -482,9 +482,9 @@  __libc_lseek64 (int fd, off64_t offset, int whence)
   return offset;
 }
 
-check_no_hidden(__mmap);
+check_no_hidden(__mmap_noerrno);
 void *weak_function
-__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
+__mmap_noerrno (void *addr, size_t len, int prot, int flags, int fd, off_t offset, int *error)
 {
   error_t err;
   vm_prot_t vmprot;
@@ -541,10 +541,22 @@  __mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
     __mach_port_deallocate (__mach_task_self (), memobj_rd);
 
   if (err)
-    return __hurd_fail (err), MAP_FAILED;
+    *error = err;
   return (void *) mapaddr;
 }
 
+check_no_hidden(__mmap);
+void *weak_function
+__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
+{
+  int err = 0;
+
+  void *r = __mmap_noerrno (addr, len, prot, flags, fd, offset, &err);
+  if (err)
+    return __hurd_fail (err), MAP_FAILED;
+  return r;
+}
+
 check_no_hidden(__fstat64);
 int weak_function
 __fstat64 (int fd, struct stat64 *buf)
diff --git a/sysdeps/unix/sysv/linux/mmap.c b/sysdeps/unix/sysv/linux/mmap.c
index 22f276bb14..19eca3fe18 100644
--- a/sysdeps/unix/sysv/linux/mmap.c
+++ b/sysdeps/unix/sysv/linux/mmap.c
@@ -31,20 +31,38 @@ 
 # endif
 
 void *
-__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
+__mmap_noerrno (void *addr, size_t len, int prot, int flags, int fd, off_t offset, int *err)
 {
   MMAP_CHECK_PAGE_UNIT ();
 
   if (offset & MMAP_OFF_LOW_MASK)
-    return (void *) INLINE_SYSCALL_ERROR_RETURN_VALUE (EINVAL);
+    return (void *) -EINVAL;
 
 #ifdef __NR_mmap2
-  return (void *) MMAP_CALL (mmap2, addr, len, prot, flags, fd,
-			     offset / (uint32_t) MMAP2_PAGE_UNIT);
+  long int r = MMAP_CALL (mmap2, addr, len, prot, flags, fd,
+			  offset / (uint32_t) MMAP2_PAGE_UNIT);
 #else
-  return (void *) MMAP_CALL (mmap, addr, len, prot, flags, fd,
-			     MMAP_ADJUST_OFFSET (offset));
+  long int r = MMAP_CALL (mmap, addr, len, prot, flags, fd,
+			  MMAP_ADJUST_OFFSET (offset));
 #endif
+  if (INTERNAL_SYSCALL_ERROR_P (r))
+    {
+      *err = (INTERNAL_SYSCALL_ERRNO (r));
+      return MAP_FAILED;
+    }
+  return (void*) r;
+}
+libc_hidden_def (__mmap_noerrno)
+
+void *
+__mmap (void *addr, size_t len, int prot, int flags, int fd, off_t offset)
+{
+  int error = 0;
+
+  void *r = __mmap_noerrno (addr, len, prot, flags, fd, offset, &error);
+  if (error)
+    __set_errno(error);
+  return r;
 }
 weak_alias (__mmap, mmap)
 libc_hidden_def (__mmap)
diff --git a/sysdeps/unix/sysv/linux/mmap64.c b/sysdeps/unix/sysv/linux/mmap64.c
index 8074deb466..3d6557734b 100644
--- a/sysdeps/unix/sysv/linux/mmap64.c
+++ b/sysdeps/unix/sysv/linux/mmap64.c
@@ -44,25 +44,38 @@ 
 #endif
 
 void *
-__mmap64 (void *addr, size_t len, int prot, int flags, int fd, off64_t offset)
+__mmap64_noerrno (void *addr, size_t len, int prot, int flags, int fd, off64_t offset, int *err)
 {
   MMAP_CHECK_PAGE_UNIT ();
 
   if (offset & MMAP_OFF_MASK)
-    return (void *) INLINE_SYSCALL_ERROR_RETURN_VALUE (EINVAL);
+    return (void *) -EINVAL;
 
   MMAP_PREPARE (addr, len, prot, flags, fd, offset);
 #ifdef __NR_mmap2
-  return (void *) MMAP_CALL (mmap2, addr, len, prot, flags, fd,
-			     (off_t) (offset / MMAP2_PAGE_UNIT));
+  long int r = MMAP_CALL (mmap2, addr, len, prot, flags, fd,
+			  (off_t) (offset / MMAP2_PAGE_UNIT));
 #else
-  return (void *) MMAP_CALL (mmap, addr, len, prot, flags, fd, offset);
+  long int r = MMAP_CALL (mmap, addr, len, prot, flags, fd, offset);
 #endif
+  if (INTERNAL_SYSCALL_ERROR_P (r))
+    {
+      *err = INTERNAL_SYSCALL_ERRNO (r);
+      return MAP_FAILED;
+    }
+  return (void *) r;
+}
+
+void *
+__mmap64 (void *addr, size_t len, int prot, int flags, int fd, off64_t offset)
+{
+  return __mmap64_noerrno (addr, len, prot, flags, fd, offset, &errno);
 }
 weak_alias (__mmap64, mmap64)
 libc_hidden_def (__mmap64)
 
 #ifdef __OFF_T_MATCHES_OFF64_T
+weak_alias (__mmap64_noerrno, __mmap_noerrno)
 weak_alias (__mmap64, mmap)
 weak_alias (__mmap64, __mmap)
 libc_hidden_def (__mmap)
diff --git a/sysdeps/unix/sysv/linux/mmap_internal.h b/sysdeps/unix/sysv/linux/mmap_internal.h
index d53f0c642a..5386b5eb63 100644
--- a/sysdeps/unix/sysv/linux/mmap_internal.h
+++ b/sysdeps/unix/sysv/linux/mmap_internal.h
@@ -43,7 +43,7 @@  static uint64_t page_unit;
 /* An architecture may override this.  */
 #ifndef MMAP_CALL
 # define MMAP_CALL(__nr, __addr, __len, __prot, __flags, __fd, __offset) \
-  INLINE_SYSCALL_CALL (__nr, __addr, __len, __prot, __flags, __fd, __offset)
+  INTERNAL_SYSCALL_CALL (__nr, __addr, __len, __prot, __flags, __fd, __offset)
 #endif
 
 #endif /* MMAP_INTERNAL_LINUX_H  */
diff --git a/sysdeps/unix/sysv/linux/s390/mmap_internal.h b/sysdeps/unix/sysv/linux/s390/mmap_internal.h
index 2884f2844b..d2289f311c 100644
--- a/sysdeps/unix/sysv/linux/s390/mmap_internal.h
+++ b/sysdeps/unix/sysv/linux/s390/mmap_internal.h
@@ -24,7 +24,7 @@ 
     long int __args[6] = { (long int) (__addr), (long int) (__len),	\
 			   (long int) (__prot), (long int) (__flags),	\
 			   (long int) (__fd), (long int) (__offset) };	\
-    INLINE_SYSCALL_CALL (__nr, __args);					\
+    INTERNAL_SYSCALL_CALL (__nr, __args);					\
   })
 
 #include_next <mmap_internal.h>