[3/4] Optimize i386 syscall inlining
Commit Message
OK for master?
H.J.
---
Define INLINE_SYSCALL_ERROR_RETURN so that i386 can optimize setting
errno by branching to the internal __syscall_error without PLT.
Since GCC 5 and above can properly spill %ebx when needed, we can inline
syscalls with 6 arguments if GCC 5 or above is used to compile glibc.
This patch rewrites INTERNAL_SYSCALL macros and skips __libc_do_syscall
for GCC 5.
For sysdeps/unix/sysv/linux/i386/brk.c, with -O2 -march=i686
-mtune=generic, GCC 5.2 now generates:
<__brk>:
0: push %ebx
1: mov $0x2d,%eax
6: mov 0x8(%esp),%ebx
a: call b <__brk+0xb> b: R_386_PC32 __x86.get_pc_thunk.dx
f: add $0x2,%edx 11: R_386_GOTPC _GLOBAL_OFFSET_TABLE_
15: call *%gs:0x10
1c: mov 0x0(%edx),%edx 1e: R_386_GOT32 __curbrk
22: cmp %eax,%ebx
24: mov %eax,(%edx)
26: ja 30 <__brk+0x30>
28: xor %eax,%eax
2a: pop %ebx
2b: ret
instead of
<__brk>:
0: push %ebx
1: mov 0x8(%esp),%ecx
5: call 6 <__brk+0x6> 6: R_386_PC32 __x86.get_pc_thunk.bx
a: add $0x2,%ebx c: R_386_GOTPC _GLOBAL_OFFSET_TABLE_
10: xchg %ecx,%ebx
12: mov $0x2d,%eax
17: call *%gs:0x10
1e: xchg %ecx,%ebx
20: mov %eax,%edx
22: mov 0x0(%ebx),%eax 24: R_386_GOT32 __curbrk
28: mov %edx,(%eax)
2a: xor %eax,%eax
2c: cmp %edx,%ecx
2e: ja 38 <__brk+0x38>
30: pop %ebx
31: ret
The new one is shorter by 2 instructions.
* sysdeps/unix/sysv/linux/i386/Makefile [$(subdir) == csu]
(sysdep-dl-routines): Add sysdep.
[$(subdir) == nptl] (libpthread-routines): Likewise.
[$(subdir) == rt] (librt-routines): Likewise.
* sysdeps/unix/sysv/linux/i386/brk.c (__brk): Add
INTERNAL_SYSCALL_DECL. Use INLINE_SYSCALL_ERROR_RETURN.
* sysdeps/unix/sysv/linux/i386/clone.S (__clone): Don't check
PIC when branching to SYSCALL_ERROR_LABEL.
* sysdeps/unix/sysv/linux/i386/fxstat.c (__fxstat): Likewise.
* sysdeps/unix/sysv/linux/i386/fxstatat.c (__fxstatat):
Likewise.
* sysdeps/unix/sysv/linux/i386/lockf64.c (lockf64): Likewise.
* sysdeps/unix/sysv/linux/i386/lxstat.c (__lxstat): Likewise.
* sysdeps/unix/sysv/linux/i386/setegid.c (setegid): Likewise.
* sysdeps/unix/sysv/linux/i386/seteuid.c (seteuid): Likewise.
* sysdeps/unix/sysv/linux/i386/sigaction.c (__libc_sigaction):
Likewise.
* sysdeps/unix/sysv/linux/i386/xstat.c (__xstat): Likewise.
* sysdeps/unix/sysv/linux/i386/libc-do-syscall.S
(__libc_do_syscall): Defined only if !__GNUC_PREREQ (5,0).
* sysdeps/unix/sysv/linux/i386/sysdep.S: Removed.
* sysdeps/unix/sysv/linux/i386/sysdep.c: New file.
* sysdeps/unix/sysv/linux/i386/sysdep.h: Define assembler macros
only if !__GNUC_PREREQ (5,0).
(SYSCALL_ERROR_LABEL): Changed to __syscall_error.
(SYSCALL_ERROR_HANDLER): Changed to empty.
(SYSCALL_ERROR_ERRNO): Removed.
(SYSCALL_ERROR_HANDLER_TLS_STORE): Likewise.
(__syscall_error): New prototype.
[IS_IN (libc)] (INLINE_SYSCALL): New macro.
(INLINE_SYSCALL_ERROR_RETURN): Likewise.
(LOADREGS_0): Likewise.
(ASMARGS_0): Likewise.
(LOADREGS_1): Likewise.
(ASMARGS_1): Likewise.
(LOADREGS_2): Likewise.
(ASMARGS_2): Likewise.
(LOADREGS_3): Likewise.
(ASMARGS_3): Likewise.
(LOADREGS_4): Likewise.
(ASMARGS_4): Likewise.
(LOADREGS_5): Likewise.
(ASMARGS_5): Likewise.
(LOADREGS_6): Likewise.
(ASMARGS_6): Likewise.
(INTERNAL_SYSCALL_MAIN_6): Optimize for GCC 5.
(INTERNAL_SYSCALL_MAIN_INLINE): Likewise.
(INTERNAL_SYSCALL_NCS): Likewise.
---
sysdeps/unix/sysv/linux/i386/Makefile | 14 ++
sysdeps/unix/sysv/linux/i386/brk.c | 12 +-
sysdeps/unix/sysv/linux/i386/clone.S | 8 --
sysdeps/unix/sysv/linux/i386/fxstat.c | 10 +-
sysdeps/unix/sysv/linux/i386/fxstatat.c | 9 +-
sysdeps/unix/sysv/linux/i386/libc-do-syscall.S | 3 +
sysdeps/unix/sysv/linux/i386/lockf64.c | 13 +-
sysdeps/unix/sysv/linux/i386/lxstat.c | 10 +-
sysdeps/unix/sysv/linux/i386/setegid.c | 5 +-
sysdeps/unix/sysv/linux/i386/seteuid.c | 5 +-
sysdeps/unix/sysv/linux/i386/sigaction.c | 12 +-
sysdeps/unix/sysv/linux/i386/sysdep.S | 40 ------
sysdeps/unix/sysv/linux/i386/sysdep.c | 30 ++++
sysdeps/unix/sysv/linux/i386/sysdep.h | 192 ++++++++++++++++---------
sysdeps/unix/sysv/linux/i386/xstat.c | 10 +-
15 files changed, 212 insertions(+), 161 deletions(-)
delete mode 100644 sysdeps/unix/sysv/linux/i386/sysdep.S
create mode 100644 sysdeps/unix/sysv/linux/i386/sysdep.c
Comments
On 10-09-2015 10:14, H.J. Lu wrote:
> OK for master?
>
> H.J.
> ---
> Define INLINE_SYSCALL_ERROR_RETURN so that i386 can optimize setting
> errno by branching to the internal __syscall_error without PLT.
>
> Since GCC 5 and above can properly spill %ebx when needed, we can inline
> syscalls with 6 arguments if GCC 5 or above is used to compile glibc.
> This patch rewrites INTERNAL_SYSCALL macros and skips __libc_do_syscall
> for GCC 5.
>
> For sysdeps/unix/sysv/linux/i386/brk.c, with -O2 -march=i686
> -mtune=generic, GCC 5.2 now generates:
You can use this refactoring to cleanup some duplicate code as well,
since i386 brk.c, setegid.c, and seteuid.c can be replaces by generic
implementation.
Also for *stat* implementations i386 could override overflow.h and
redefine stat_overflow with stat64 logic.
For lockf64.c I also think it can be feasible to change default
implementation to accommodate i386 requirements (use flock64
instead of flock).
The only implementation is really i386 specific is sigaction.c,
but I can't tell how much of the assembly hackery it requires
nowadays.
>
> <__brk>:
> 0: push %ebx
> 1: mov $0x2d,%eax
> 6: mov 0x8(%esp),%ebx
> a: call b <__brk+0xb> b: R_386_PC32 __x86.get_pc_thunk.dx
> f: add $0x2,%edx 11: R_386_GOTPC _GLOBAL_OFFSET_TABLE_
> 15: call *%gs:0x10
> 1c: mov 0x0(%edx),%edx 1e: R_386_GOT32 __curbrk
> 22: cmp %eax,%ebx
> 24: mov %eax,(%edx)
> 26: ja 30 <__brk+0x30>
> 28: xor %eax,%eax
> 2a: pop %ebx
> 2b: ret
>
> instead of
>
> <__brk>:
> 0: push %ebx
> 1: mov 0x8(%esp),%ecx
> 5: call 6 <__brk+0x6> 6: R_386_PC32 __x86.get_pc_thunk.bx
> a: add $0x2,%ebx c: R_386_GOTPC _GLOBAL_OFFSET_TABLE_
> 10: xchg %ecx,%ebx
> 12: mov $0x2d,%eax
> 17: call *%gs:0x10
> 1e: xchg %ecx,%ebx
> 20: mov %eax,%edx
> 22: mov 0x0(%ebx),%eax 24: R_386_GOT32 __curbrk
> 28: mov %edx,(%eax)
> 2a: xor %eax,%eax
> 2c: cmp %edx,%ecx
> 2e: ja 38 <__brk+0x38>
> 30: pop %ebx
> 31: ret
>
> The new one is shorter by 2 instructions.
>
> * sysdeps/unix/sysv/linux/i386/Makefile [$(subdir) == csu]
> (sysdep-dl-routines): Add sysdep.
> [$(subdir) == nptl] (libpthread-routines): Likewise.
> [$(subdir) == rt] (librt-routines): Likewise.
> * sysdeps/unix/sysv/linux/i386/brk.c (__brk): Add
> INTERNAL_SYSCALL_DECL. Use INLINE_SYSCALL_ERROR_RETURN.
> * sysdeps/unix/sysv/linux/i386/clone.S (__clone): Don't check
> PIC when branching to SYSCALL_ERROR_LABEL.
> * sysdeps/unix/sysv/linux/i386/fxstat.c (__fxstat): Likewise.
> * sysdeps/unix/sysv/linux/i386/fxstatat.c (__fxstatat):
> Likewise.
> * sysdeps/unix/sysv/linux/i386/lockf64.c (lockf64): Likewise.
> * sysdeps/unix/sysv/linux/i386/lxstat.c (__lxstat): Likewise.
> * sysdeps/unix/sysv/linux/i386/setegid.c (setegid): Likewise.
> * sysdeps/unix/sysv/linux/i386/seteuid.c (seteuid): Likewise.
> * sysdeps/unix/sysv/linux/i386/sigaction.c (__libc_sigaction):
> Likewise.
> * sysdeps/unix/sysv/linux/i386/xstat.c (__xstat): Likewise.
> * sysdeps/unix/sysv/linux/i386/libc-do-syscall.S
> (__libc_do_syscall): Defined only if !__GNUC_PREREQ (5,0).
> * sysdeps/unix/sysv/linux/i386/sysdep.S: Removed.
> * sysdeps/unix/sysv/linux/i386/sysdep.c: New file.
> * sysdeps/unix/sysv/linux/i386/sysdep.h: Define assembler macros
> only if !__GNUC_PREREQ (5,0).
> (SYSCALL_ERROR_LABEL): Changed to __syscall_error.
> (SYSCALL_ERROR_HANDLER): Changed to empty.
> (SYSCALL_ERROR_ERRNO): Removed.
> (SYSCALL_ERROR_HANDLER_TLS_STORE): Likewise.
> (__syscall_error): New prototype.
> [IS_IN (libc)] (INLINE_SYSCALL): New macro.
> (INLINE_SYSCALL_ERROR_RETURN): Likewise.
> (LOADREGS_0): Likewise.
> (ASMARGS_0): Likewise.
> (LOADREGS_1): Likewise.
> (ASMARGS_1): Likewise.
> (LOADREGS_2): Likewise.
> (ASMARGS_2): Likewise.
> (LOADREGS_3): Likewise.
> (ASMARGS_3): Likewise.
> (LOADREGS_4): Likewise.
> (ASMARGS_4): Likewise.
> (LOADREGS_5): Likewise.
> (ASMARGS_5): Likewise.
> (LOADREGS_6): Likewise.
> (ASMARGS_6): Likewise.
> (INTERNAL_SYSCALL_MAIN_6): Optimize for GCC 5.
> (INTERNAL_SYSCALL_MAIN_INLINE): Likewise.
> (INTERNAL_SYSCALL_NCS): Likewise.
> ---
> sysdeps/unix/sysv/linux/i386/Makefile | 14 ++
> sysdeps/unix/sysv/linux/i386/brk.c | 12 +-
> sysdeps/unix/sysv/linux/i386/clone.S | 8 --
> sysdeps/unix/sysv/linux/i386/fxstat.c | 10 +-
> sysdeps/unix/sysv/linux/i386/fxstatat.c | 9 +-
> sysdeps/unix/sysv/linux/i386/libc-do-syscall.S | 3 +
> sysdeps/unix/sysv/linux/i386/lockf64.c | 13 +-
> sysdeps/unix/sysv/linux/i386/lxstat.c | 10 +-
> sysdeps/unix/sysv/linux/i386/setegid.c | 5 +-
> sysdeps/unix/sysv/linux/i386/seteuid.c | 5 +-
> sysdeps/unix/sysv/linux/i386/sigaction.c | 12 +-
> sysdeps/unix/sysv/linux/i386/sysdep.S | 40 ------
> sysdeps/unix/sysv/linux/i386/sysdep.c | 30 ++++
> sysdeps/unix/sysv/linux/i386/sysdep.h | 192 ++++++++++++++++---------
> sysdeps/unix/sysv/linux/i386/xstat.c | 10 +-
> 15 files changed, 212 insertions(+), 161 deletions(-)
> delete mode 100644 sysdeps/unix/sysv/linux/i386/sysdep.S
> create mode 100644 sysdeps/unix/sysv/linux/i386/sysdep.c
>
> diff --git a/sysdeps/unix/sysv/linux/i386/Makefile b/sysdeps/unix/sysv/linux/i386/Makefile
> index 80da593..e10d133 100644
> --- a/sysdeps/unix/sysv/linux/i386/Makefile
> +++ b/sysdeps/unix/sysv/linux/i386/Makefile
> @@ -27,3 +27,17 @@ endif
> ifeq ($(subdir),stdlib)
> gen-as-const-headers += ucontext_i.sym
> endif
> +
> +ifeq ($(subdir),csu)
> +sysdep-dl-routines += sysdep
> +endif
> +
> +ifeq ($(subdir),nptl)
> +# pull in __syscall_error routine
> +libpthread-routines += sysdep
> +endif
> +
> +ifeq ($(subdir),rt)
> +# pull in __syscall_error routine
> +librt-routines += sysdep
> +endif
> diff --git a/sysdeps/unix/sysv/linux/i386/brk.c b/sysdeps/unix/sysv/linux/i386/brk.c
> index 5b9a0ce..2d2daa5 100644
> --- a/sysdeps/unix/sysv/linux/i386/brk.c
> +++ b/sysdeps/unix/sysv/linux/i386/brk.c
> @@ -31,19 +31,11 @@ weak_alias (__curbrk, ___brk_addr)
> int
> __brk (void *addr)
> {
> - void *newbrk;
> -
> INTERNAL_SYSCALL_DECL (err);
> - newbrk = (void *) INTERNAL_SYSCALL (brk, err, 1, addr);
> -
> + void *newbrk = (void *) INTERNAL_SYSCALL (brk, err, 1, addr);
> __curbrk = newbrk;
> -
> if (newbrk < addr)
> - {
> - __set_errno (ENOMEM);
> - return -1;
> - }
> -
> + return INLINE_SYSCALL_ERROR_RETURN (ENOMEM);
> return 0;
> }
> weak_alias (__brk, brk)
> diff --git a/sysdeps/unix/sysv/linux/i386/clone.S b/sysdeps/unix/sysv/linux/i386/clone.S
> index 243dbfe..2aafb3a 100644
> --- a/sysdeps/unix/sysv/linux/i386/clone.S
> +++ b/sysdeps/unix/sysv/linux/i386/clone.S
> @@ -47,19 +47,11 @@ ENTRY (__clone)
> /* Sanity check arguments. */
> movl $-EINVAL,%eax
> movl FUNC(%esp),%ecx /* no NULL function pointers */
> -#ifdef PIC
> - jecxz SYSCALL_ERROR_LABEL
> -#else
> testl %ecx,%ecx
> jz SYSCALL_ERROR_LABEL
> -#endif
> movl STACK(%esp),%ecx /* no NULL stack pointers */
> -#ifdef PIC
> - jecxz SYSCALL_ERROR_LABEL
> -#else
> testl %ecx,%ecx
> jz SYSCALL_ERROR_LABEL
> -#endif
>
> /* Insert the argument onto the new stack. Make sure the new
> thread is started with an alignment of (mod 16). */
> diff --git a/sysdeps/unix/sysv/linux/i386/fxstat.c b/sysdeps/unix/sysv/linux/i386/fxstat.c
> index 2f7a8fe..3919e0c 100644
> --- a/sysdeps/unix/sysv/linux/i386/fxstat.c
> +++ b/sysdeps/unix/sysv/linux/i386/fxstat.c
> @@ -42,10 +42,12 @@ __fxstat (int vers, int fd, struct stat *buf)
> {
> struct stat64 buf64;
>
> - result = INLINE_SYSCALL (fstat64, 2, fd, &buf64);
> - if (result == 0)
> - result = __xstat32_conv (vers, &buf64, buf);
> - return result;
> + INTERNAL_SYSCALL_DECL (err);
> + result = INTERNAL_SYSCALL (fstat64, err, 2, fd, &buf64);
> + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
> + return INLINE_SYSCALL_ERROR_RETURN (-result);
> + else
> + return __xstat32_conv (vers, &buf64, buf);
> }
> }
>
> diff --git a/sysdeps/unix/sysv/linux/i386/fxstatat.c b/sysdeps/unix/sysv/linux/i386/fxstatat.c
> index 6f3c251..c4be04d 100644
> --- a/sysdeps/unix/sysv/linux/i386/fxstatat.c
> +++ b/sysdeps/unix/sysv/linux/i386/fxstatat.c
> @@ -42,13 +42,10 @@ __fxstatat (int vers, int fd, const char *file, struct stat *st, int flag)
> struct stat64 st64;
>
> result = INTERNAL_SYSCALL (fstatat64, err, 4, fd, file, &st64, flag);
> - if (!__builtin_expect (INTERNAL_SYSCALL_ERROR_P (result, err), 1))
> - return __xstat32_conv (vers, &st64, st);
> + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
> + return INLINE_SYSCALL_ERROR_RETURN (-result);
> else
> - {
> - __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
> - return -1;
> - }
> + return __xstat32_conv (vers, &st64, st);
> }
> libc_hidden_def (__fxstatat)
> #ifdef XSTAT_IS_XSTAT64
> diff --git a/sysdeps/unix/sysv/linux/i386/libc-do-syscall.S b/sysdeps/unix/sysv/linux/i386/libc-do-syscall.S
> index af5c6f0..cdef3d5 100644
> --- a/sysdeps/unix/sysv/linux/i386/libc-do-syscall.S
> +++ b/sysdeps/unix/sysv/linux/i386/libc-do-syscall.S
> @@ -18,6 +18,8 @@
>
> #include <sysdep.h>
>
> +#if !__GNUC_PREREQ (5,0)
> +
> /* %eax, %ecx, %edx and %esi contain the values expected by the kernel.
> %edi points to a structure with the values of %ebx, %edi and %ebp. */
>
> @@ -48,3 +50,4 @@ ENTRY (__libc_do_syscall)
> cfi_restore (ebx)
> ret
> END (__libc_do_syscall)
> +#endif
> diff --git a/sysdeps/unix/sysv/linux/i386/lockf64.c b/sysdeps/unix/sysv/linux/i386/lockf64.c
> index 61fcf22..ae0735c 100644
> --- a/sysdeps/unix/sysv/linux/i386/lockf64.c
> +++ b/sysdeps/unix/sysv/linux/i386/lockf64.c
> @@ -29,6 +29,7 @@ lockf64 (int fd, int cmd, off64_t len64)
> {
> struct flock64 fl64;
> int cmd64;
> + int result;
>
> memset ((char *) &fl64, '\0', sizeof (fl64));
> fl64.l_whence = SEEK_CUR;
> @@ -41,12 +42,13 @@ lockf64 (int fd, int cmd, off64_t len64)
> /* Test the lock: return 0 if FD is unlocked or locked by this process;
> return -1, set errno to EACCES, if another process holds the lock. */
> fl64.l_type = F_RDLCK;
> - if (INLINE_SYSCALL (fcntl64, 3, fd, F_GETLK64, &fl64) < 0)
> - return -1;
> + INTERNAL_SYSCALL_DECL (err);
> + result = INTERNAL_SYSCALL (fcntl64, err, 3, fd, F_GETLK64, &fl64);
> + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
> + return INLINE_SYSCALL_ERROR_RETURN (-result);
> if (fl64.l_type == F_UNLCK || fl64.l_pid == __getpid ())
> return 0;
> - __set_errno (EACCES);
> - return -1;
> + return INLINE_SYSCALL_ERROR_RETURN (EACCES);
> case F_ULOCK:
> fl64.l_type = F_UNLCK;
> cmd64 = F_SETLK64;
> @@ -61,8 +63,7 @@ lockf64 (int fd, int cmd, off64_t len64)
> break;
>
> default:
> - __set_errno (EINVAL);
> - return -1;
> + return INLINE_SYSCALL_ERROR_RETURN (EINVAL);
> }
> return INLINE_SYSCALL (fcntl64, 3, fd, cmd64, &fl64);
> }
> diff --git a/sysdeps/unix/sysv/linux/i386/lxstat.c b/sysdeps/unix/sysv/linux/i386/lxstat.c
> index 0891127..759d240 100644
> --- a/sysdeps/unix/sysv/linux/i386/lxstat.c
> +++ b/sysdeps/unix/sysv/linux/i386/lxstat.c
> @@ -43,10 +43,12 @@ __lxstat (int vers, const char *name, struct stat *buf)
> {
> struct stat64 buf64;
>
> - result = INLINE_SYSCALL (lstat64, 2, name, &buf64);
> - if (result == 0)
> - result = __xstat32_conv (vers, &buf64, buf);
> - return result;
> + INTERNAL_SYSCALL_DECL (err);
> + result = INTERNAL_SYSCALL (lstat64, err, 2, name, &buf64);
> + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
> + return INLINE_SYSCALL_ERROR_RETURN (-result);
> + else
> + return __xstat32_conv (vers, &buf64, buf);
> }
> }
>
> diff --git a/sysdeps/unix/sysv/linux/i386/setegid.c b/sysdeps/unix/sysv/linux/i386/setegid.c
> index 8c39784..851513e 100644
> --- a/sysdeps/unix/sysv/linux/i386/setegid.c
> +++ b/sysdeps/unix/sysv/linux/i386/setegid.c
> @@ -27,10 +27,7 @@ setegid (gid)
> int result;
>
> if (gid == (gid_t) ~0)
> - {
> - __set_errno (EINVAL);
> - return -1;
> - }
> + INLINE_SYSCALL_ERROR_RETURN (EINVAL);
>
> result = INLINE_SETXID_SYSCALL (setresgid32, 3, -1, gid, -1);
>
> diff --git a/sysdeps/unix/sysv/linux/i386/seteuid.c b/sysdeps/unix/sysv/linux/i386/seteuid.c
> index d6a7a27..bebca30 100644
> --- a/sysdeps/unix/sysv/linux/i386/seteuid.c
> +++ b/sysdeps/unix/sysv/linux/i386/seteuid.c
> @@ -26,10 +26,7 @@ seteuid (uid_t uid)
> int result;
>
> if (uid == (uid_t) ~0)
> - {
> - __set_errno (EINVAL);
> - return -1;
> - }
> + INLINE_SYSCALL_ERROR_RETURN (EINVAL);
>
> result = INLINE_SETXID_SYSCALL (setresuid32, 3, -1, uid, -1);
>
> diff --git a/sysdeps/unix/sysv/linux/i386/sigaction.c b/sysdeps/unix/sysv/linux/i386/sigaction.c
> index b20a9b9..00df354 100644
> --- a/sysdeps/unix/sysv/linux/i386/sigaction.c
> +++ b/sysdeps/unix/sysv/linux/i386/sigaction.c
> @@ -69,11 +69,13 @@ __libc_sigaction (int sig, const struct sigaction *act, struct sigaction *oact)
>
> /* XXX The size argument hopefully will have to be changed to the
> real size of the user-level sigset_t. */
> - result = INLINE_SYSCALL (rt_sigaction, 4,
> - sig, act ? &kact : NULL,
> - oact ? &koact : NULL, _NSIG / 8);
> -
> - if (oact && result >= 0)
> + INTERNAL_SYSCALL_DECL (err);
> + result = INTERNAL_SYSCALL (rt_sigaction, err, 4,
> + sig, act ? &kact : NULL,
> + oact ? &koact : NULL, _NSIG / 8);
> + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
> + return INLINE_SYSCALL_ERROR_RETURN (-result);
> + else if (oact && result >= 0)
> {
> oact->sa_handler = koact.k_sa_handler;
> memcpy (&oact->sa_mask, &koact.sa_mask, sizeof (sigset_t));
> diff --git a/sysdeps/unix/sysv/linux/i386/sysdep.S b/sysdeps/unix/sysv/linux/i386/sysdep.S
> deleted file mode 100644
> index 4e6c262..0000000
> --- a/sysdeps/unix/sysv/linux/i386/sysdep.S
> +++ /dev/null
> @@ -1,40 +0,0 @@
> -/* Copyright (C) 1995-2015 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#include <sysdep.h>
> -
> -/* The following code is only used in the shared library when we
> - compile the reentrant version. Otherwise each system call defines
> - each own version. */
> -
> -#ifndef PIC
> -
> -/* The syscall stubs jump here when they detect an error.
> - The code for Linux is almost identical to the canonical Unix/i386
> - code, except that the error number in %eax is negated. */
> -
> -#undef CALL_MCOUNT
> -#define CALL_MCOUNT /* Don't insert the profiling call, it clobbers %eax. */
> -
> - .text
> -ENTRY (__syscall_error)
> - negl %eax
> -
> -#define __syscall_error __syscall_error_1
> -#include <sysdeps/unix/i386/sysdep.S>
> -
> -#endif /* !PIC */
> diff --git a/sysdeps/unix/sysv/linux/i386/sysdep.c b/sysdeps/unix/sysv/linux/i386/sysdep.c
> new file mode 100644
> index 0000000..141105e
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/i386/sysdep.c
> @@ -0,0 +1,30 @@
> +/* Copyright (C) 2015 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <errno.h>
> +#include <sysdep.h>
> +
> +/* This routine is jumped to by all the syscall handlers, to stash
> + an error number into errno. ERROR is the negative error number
> + returned from the x86 kernel. */
> +int
> +__attribute__ ((__regparm__ (1)))
> +__syscall_error (int error)
> +{
> + __set_errno (-error);
> + return -1;
> +}
> diff --git a/sysdeps/unix/sysv/linux/i386/sysdep.h b/sysdeps/unix/sysv/linux/i386/sysdep.h
> index aac3e7b..1515fa6 100644
> --- a/sysdeps/unix/sysv/linux/i386/sysdep.h
> +++ b/sysdeps/unix/sysv/linux/i386/sysdep.h
> @@ -56,11 +56,7 @@
>
> /* We don't want the label for the error handle to be global when we define
> it here. */
> -#ifdef PIC
> -# define SYSCALL_ERROR_LABEL 0f
> -#else
> -# define SYSCALL_ERROR_LABEL syscall_error
> -#endif
> +#define SYSCALL_ERROR_LABEL __syscall_error
>
> #undef PSEUDO
> #define PSEUDO(name, syscall_name, args) \
> @@ -101,55 +97,7 @@
>
> #define ret_ERRVAL ret
>
> -#ifndef PIC
> -# define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.S is used. */
> -#else
> -
> -# if RTLD_PRIVATE_ERRNO
> -# define SYSCALL_ERROR_HANDLER \
> -0:SETUP_PIC_REG(cx); \
> - addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
> - negl %eax; \
> - movl %eax, rtld_errno@GOTOFF(%ecx); \
> - orl $-1, %eax; \
> - ret;
> -
> -# elif defined _LIBC_REENTRANT
> -
> -# if IS_IN (libc)
> -# define SYSCALL_ERROR_ERRNO __libc_errno
> -# else
> -# define SYSCALL_ERROR_ERRNO errno
> -# endif
> -# define SYSCALL_ERROR_HANDLER \
> -0:SETUP_PIC_REG (cx); \
> - addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
> - movl SYSCALL_ERROR_ERRNO@GOTNTPOFF(%ecx), %ecx; \
> - negl %eax; \
> - SYSCALL_ERROR_HANDLER_TLS_STORE (%eax, %ecx); \
> - orl $-1, %eax; \
> - ret;
> -# ifndef NO_TLS_DIRECT_SEG_REFS
> -# define SYSCALL_ERROR_HANDLER_TLS_STORE(src, destoff) \
> - movl src, %gs:(destoff)
> -# else
> -# define SYSCALL_ERROR_HANDLER_TLS_STORE(src, destoff) \
> - addl %gs:0, destoff; \
> - movl src, (destoff)
> -# endif
> -# else
> -/* Store (- %eax) into errno through the GOT. */
> -# define SYSCALL_ERROR_HANDLER \
> -0:SETUP_PIC_REG(cx); \
> - addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
> - negl %eax; \
> - movl errno@GOT(%ecx), %ecx; \
> - movl %eax, (%ecx); \
> - orl $-1, %eax; \
> - ret;
> -# endif /* _LIBC_REENTRANT */
> -#endif /* PIC */
> -
> +#define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.c is used. */
>
> /* The original calling convention for system calls on Linux/i386 is
> to use int $0x80. */
> @@ -276,6 +224,10 @@
>
> #else /* !__ASSEMBLER__ */
>
> +extern int __syscall_error (int)
> + attribute_hidden __attribute__ ((__regparm__ (1)));
> +
> +#if !__GNUC_PREREQ (5,0)
> /* We need some help from the assembler to generate optimal code. We
> define some macros here which later will be used. */
> asm (".L__X'%ebx = 1\n\t"
> @@ -315,11 +267,20 @@ struct libc_do_syscall_args
> {
> int ebx, edi, ebp;
> };
> +#endif
>
> /* Define a macro which expands inline into the wrapper code for a system
> call. */
> #undef INLINE_SYSCALL
> -#define INLINE_SYSCALL(name, nr, args...) \
> +#if IS_IN (libc)
> +# define INLINE_SYSCALL(name, nr, args...) \
> + ({ \
> + unsigned int resultvar = INTERNAL_SYSCALL (name, , nr, args); \
> + __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (resultvar, )) \
> + ? __syscall_error (-INTERNAL_SYSCALL_ERRNO (resultvar, )) \
> + : (int) resultvar; })
> +#else
> +# define INLINE_SYSCALL(name, nr, args...) \
> ({ \
> unsigned int resultvar = INTERNAL_SYSCALL (name, , nr, args); \
> if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (resultvar, ))) \
> @@ -328,6 +289,14 @@ struct libc_do_syscall_args
> resultvar = 0xffffffff; \
> } \
> (int) resultvar; })
> +#endif
> +
> +/* Set error number and return -1. Return the internal function,
> + __syscall_error, which sets errno from the negative error number
> + and returns -1, to avoid PIC. */
> +#undef INLINE_SYSCALL_ERROR_RETURN
> +#define INLINE_SYSCALL_ERROR_RETURN(resultvar) \
> + __syscall_error (-(resultvar))
>
> /* List of system calls which are supported as vsyscalls. */
> # define HAVE_CLOCK_GETTIME_VSYSCALL 1
> @@ -355,8 +324,12 @@ struct libc_do_syscall_args
> INTERNAL_SYSCALL_MAIN_INLINE(name, err, 5, args)
> /* Each object using 6-argument inline syscalls must include a
> definition of __libc_do_syscall. */
> -#define INTERNAL_SYSCALL_MAIN_6(name, err, arg1, arg2, arg3, \
> - arg4, arg5, arg6) \
> +#if __GNUC_PREREQ (5,0)
> +# define INTERNAL_SYSCALL_MAIN_6(name, err, args...) \
> + INTERNAL_SYSCALL_MAIN_INLINE(name, err, 6, args)
> +#else /* GCC 5 */
> +# define INTERNAL_SYSCALL_MAIN_6(name, err, arg1, arg2, arg3, \
> + arg4, arg5, arg6) \
> struct libc_do_syscall_args _xv = \
> { \
> (int) (arg1), \
> @@ -369,14 +342,52 @@ struct libc_do_syscall_args
> : "=a" (resultvar) \
> : "i" (__NR_##name), "c" (arg2), "d" (arg3), "S" (arg4), "D" (&_xv) \
> : "memory", "cc")
> +#endif /* GCC 5 */
> #define INTERNAL_SYSCALL(name, err, nr, args...) \
> ({ \
> register unsigned int resultvar; \
> INTERNAL_SYSCALL_MAIN_##nr (name, err, args); \
> (int) resultvar; })
> #ifdef I386_USE_SYSENTER
> -# ifdef SHARED
> -# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> +# if __GNUC_PREREQ (5,0)
> +# ifdef SHARED
> +# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> + LOADREGS_##nr(args) \
> + asm volatile ( \
> + "call *%%gs:%P2" \
> + : "=a" (resultvar) \
> + : "a" (__NR_##name), "i" (offsetof (tcbhead_t, sysinfo)) \
> + ASMARGS_##nr(args) : "memory", "cc")
> +# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> + ({ \
> + register unsigned int resultvar; \
> + LOADREGS_##nr(args) \
> + asm volatile ( \
> + "call *%%gs:%P2" \
> + : "=a" (resultvar) \
> + : "a" (name), "i" (offsetof (tcbhead_t, sysinfo)) \
> + ASMARGS_##nr(args) : "memory", "cc"); \
> + (int) resultvar; })
> +# else
> +# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> + LOADREGS_##nr(args) \
> + asm volatile ( \
> + "call *_dl_sysinfo" \
> + : "=a" (resultvar) \
> + : "a" (__NR_##name) ASMARGS_##nr(args) : "memory", "cc")
> +# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> + ({ \
> + register unsigned int resultvar; \
> + LOADREGS_##nr(args) \
> + asm volatile ( \
> + "call *_dl_sysinfo" \
> + : "=a" (resultvar) \
> + : "a" (name) ASMARGS_##nr(args) : "memory", "cc"); \
> + (int) resultvar; })
> +# endif
> +# else /* GCC 5 */
> +# ifdef SHARED
> +# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> EXTRAVAR_##nr \
> asm volatile ( \
> LOADARGS_##nr \
> @@ -386,7 +397,7 @@ struct libc_do_syscall_args
> : "=a" (resultvar) \
> : "i" (__NR_##name), "i" (offsetof (tcbhead_t, sysinfo)) \
> ASMFMT_##nr(args) : "memory", "cc")
> -# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> +# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> ({ \
> register unsigned int resultvar; \
> EXTRAVAR_##nr \
> @@ -398,8 +409,8 @@ struct libc_do_syscall_args
> : "0" (name), "i" (offsetof (tcbhead_t, sysinfo)) \
> ASMFMT_##nr(args) : "memory", "cc"); \
> (int) resultvar; })
> -# else
> -# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> +# else
> +# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> EXTRAVAR_##nr \
> asm volatile ( \
> LOADARGS_##nr \
> @@ -408,7 +419,7 @@ struct libc_do_syscall_args
> RESTOREARGS_##nr \
> : "=a" (resultvar) \
> : "i" (__NR_##name) ASMFMT_##nr(args) : "memory", "cc")
> -# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> +# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> ({ \
> register unsigned int resultvar; \
> EXTRAVAR_##nr \
> @@ -419,9 +430,27 @@ struct libc_do_syscall_args
> : "=a" (resultvar) \
> : "0" (name) ASMFMT_##nr(args) : "memory", "cc"); \
> (int) resultvar; })
> -# endif
> +# endif
> +# endif /* GCC 5 */
> #else
> -# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> +# if __GNUC_PREREQ (5,0)
> +# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> + LOADREGS_##nr(args) \
> + asm volatile ( \
> + "int $0x80" \
> + : "=a" (resultvar) \
> + : "a" (__NR_##name) ASMARGS_##nr(args) : "memory", "cc")
> +# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> + ({ \
> + register unsigned int resultvar; \
> + LOADREGS_##nr(args) \
> + asm volatile ( \
> + "int $0x80" \
> + : "=a" (resultvar) \
> + : "a" (name) ASMARGS_##nr(args) : "memory", "cc"); \
> + (int) resultvar; })
> +# else /* GCC 5 */
> +# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
> EXTRAVAR_##nr \
> asm volatile ( \
> LOADARGS_##nr \
> @@ -430,7 +459,7 @@ struct libc_do_syscall_args
> RESTOREARGS_##nr \
> : "=a" (resultvar) \
> : "i" (__NR_##name) ASMFMT_##nr(args) : "memory", "cc")
> -# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> +# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
> ({ \
> register unsigned int resultvar; \
> EXTRAVAR_##nr \
> @@ -441,6 +470,7 @@ struct libc_do_syscall_args
> : "=a" (resultvar) \
> : "0" (name) ASMFMT_##nr(args) : "memory", "cc"); \
> (int) resultvar; })
> +# endif /* GCC 5 */
> #endif
>
> #undef INTERNAL_SYSCALL_DECL
> @@ -505,6 +535,36 @@ struct libc_do_syscall_args
> # define RESTOREARGS_5
> #endif
>
> +#if __GNUC_PREREQ (5,0)
> +# define LOADREGS_0()
> +# define ASMARGS_0()
> +# define LOADREGS_1(arg1) \
> + LOADREGS_0 ()
> +# define ASMARGS_1(arg1) \
> + ASMARGS_0 (), "b" ((unsigned int) (arg1))
> +# define LOADREGS_2(arg1, arg2) \
> + LOADREGS_1 (arg1)
> +# define ASMARGS_2(arg1, arg2) \
> + ASMARGS_1 (arg1), "c" ((unsigned int) (arg2))
> +# define LOADREGS_3(arg1, arg2, arg3) \
> + LOADREGS_2 (arg1, arg2)
> +# define ASMARGS_3(arg1, arg2, arg3) \
> + ASMARGS_2 (arg1, arg2), "d" ((unsigned int) (arg3))
> +# define LOADREGS_4(arg1, arg2, arg3, arg4) \
> + LOADREGS_3 (arg1, arg2, arg3)
> +# define ASMARGS_4(arg1, arg2, arg3, arg4) \
> + ASMARGS_3 (arg1, arg2, arg3), "S" ((unsigned int) (arg4))
> +# define LOADREGS_5(arg1, arg2, arg3, arg4, arg5) \
> + LOADREGS_4 (arg1, arg2, arg3, arg4)
> +# define ASMARGS_5(arg1, arg2, arg3, arg4, arg5) \
> + ASMARGS_4 (arg1, arg2, arg3, arg4), "D" ((unsigned int) (arg5))
> +# define LOADREGS_6(arg1, arg2, arg3, arg4, arg5, arg6) \
> + register unsigned int _a6 asm ("ebp") = (unsigned int) (arg6); \
> + LOADREGS_5 (arg1, arg2, arg3, arg4, arg5)
> +# define ASMARGS_6(arg1, arg2, arg3, arg4, arg5, arg6) \
> + ASMARGS_5 (arg1, arg2, arg3, arg4, arg5), "r" (_a6)
> +#endif /* GCC 5 */
> +
> #define ASMFMT_0()
> #ifdef __PIC__
> # define ASMFMT_1(arg1) \
> diff --git a/sysdeps/unix/sysv/linux/i386/xstat.c b/sysdeps/unix/sysv/linux/i386/xstat.c
> index 2424434..4dce122 100644
> --- a/sysdeps/unix/sysv/linux/i386/xstat.c
> +++ b/sysdeps/unix/sysv/linux/i386/xstat.c
> @@ -43,10 +43,12 @@ __xstat (int vers, const char *name, struct stat *buf)
> {
> struct stat64 buf64;
>
> - result = INLINE_SYSCALL (stat64, 2, name, &buf64);
> - if (result == 0)
> - result = __xstat32_conv (vers, &buf64, buf);
> - return result;
> + INTERNAL_SYSCALL_DECL (err);
> + result = INTERNAL_SYSCALL (stat64, err, 2, name, &buf64);
> + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
> + return INLINE_SYSCALL_ERROR_RETURN (-result);
> + else
> + return __xstat32_conv (vers, &buf64, buf);
> }
> }
> hidden_def (__xstat)
>
On Thu, Sep 10, 2015 at 10:23 AM, Adhemerval Zanella
<adhemerval.zanella@linaro.org> wrote:
>
>
> On 10-09-2015 10:14, H.J. Lu wrote:
>> OK for master?
>>
>> H.J.
>> ---
>> Define INLINE_SYSCALL_ERROR_RETURN so that i386 can optimize setting
>> errno by branching to the internal __syscall_error without PLT.
>>
>> Since GCC 5 and above can properly spill %ebx when needed, we can inline
>> syscalls with 6 arguments if GCC 5 or above is used to compile glibc.
>> This patch rewrites INTERNAL_SYSCALL macros and skips __libc_do_syscall
>> for GCC 5.
>>
>> For sysdeps/unix/sysv/linux/i386/brk.c, with -O2 -march=i686
>> -mtune=generic, GCC 5.2 now generates:
>
> You can use this refactoring to cleanup some duplicate code as well,
> since i386 brk.c, setegid.c, and seteuid.c can be replaces by generic
> implementation.
>
> Also for *stat* implementations i386 could override overflow.h and
> redefine stat_overflow with stat64 logic.
>
> For lockf64.c I also think it can be feasible to change default
> implementation to accommodate i386 requirements (use flock64
> instead of flock).
>
> The only implementation is really i386 specific is sigaction.c,
> but I can't tell how much of the assembly hackery it requires
> nowadays.
>
I'd like to keep my changes as mechanical as possible. These
cleanups belong to separate patches after my patches are
checked-in.
@@ -27,3 +27,17 @@ endif
ifeq ($(subdir),stdlib)
gen-as-const-headers += ucontext_i.sym
endif
+
+ifeq ($(subdir),csu)
+sysdep-dl-routines += sysdep
+endif
+
+ifeq ($(subdir),nptl)
+# pull in __syscall_error routine
+libpthread-routines += sysdep
+endif
+
+ifeq ($(subdir),rt)
+# pull in __syscall_error routine
+librt-routines += sysdep
+endif
@@ -31,19 +31,11 @@ weak_alias (__curbrk, ___brk_addr)
int
__brk (void *addr)
{
- void *newbrk;
-
INTERNAL_SYSCALL_DECL (err);
- newbrk = (void *) INTERNAL_SYSCALL (brk, err, 1, addr);
-
+ void *newbrk = (void *) INTERNAL_SYSCALL (brk, err, 1, addr);
__curbrk = newbrk;
-
if (newbrk < addr)
- {
- __set_errno (ENOMEM);
- return -1;
- }
-
+ return INLINE_SYSCALL_ERROR_RETURN (ENOMEM);
return 0;
}
weak_alias (__brk, brk)
@@ -47,19 +47,11 @@ ENTRY (__clone)
/* Sanity check arguments. */
movl $-EINVAL,%eax
movl FUNC(%esp),%ecx /* no NULL function pointers */
-#ifdef PIC
- jecxz SYSCALL_ERROR_LABEL
-#else
testl %ecx,%ecx
jz SYSCALL_ERROR_LABEL
-#endif
movl STACK(%esp),%ecx /* no NULL stack pointers */
-#ifdef PIC
- jecxz SYSCALL_ERROR_LABEL
-#else
testl %ecx,%ecx
jz SYSCALL_ERROR_LABEL
-#endif
/* Insert the argument onto the new stack. Make sure the new
thread is started with an alignment of (mod 16). */
@@ -42,10 +42,12 @@ __fxstat (int vers, int fd, struct stat *buf)
{
struct stat64 buf64;
- result = INLINE_SYSCALL (fstat64, 2, fd, &buf64);
- if (result == 0)
- result = __xstat32_conv (vers, &buf64, buf);
- return result;
+ INTERNAL_SYSCALL_DECL (err);
+ result = INTERNAL_SYSCALL (fstat64, err, 2, fd, &buf64);
+ if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
+ return INLINE_SYSCALL_ERROR_RETURN (-result);
+ else
+ return __xstat32_conv (vers, &buf64, buf);
}
}
@@ -42,13 +42,10 @@ __fxstatat (int vers, int fd, const char *file, struct stat *st, int flag)
struct stat64 st64;
result = INTERNAL_SYSCALL (fstatat64, err, 4, fd, file, &st64, flag);
- if (!__builtin_expect (INTERNAL_SYSCALL_ERROR_P (result, err), 1))
- return __xstat32_conv (vers, &st64, st);
+ if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
+ return INLINE_SYSCALL_ERROR_RETURN (-result);
else
- {
- __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
- return -1;
- }
+ return __xstat32_conv (vers, &st64, st);
}
libc_hidden_def (__fxstatat)
#ifdef XSTAT_IS_XSTAT64
@@ -18,6 +18,8 @@
#include <sysdep.h>
+#if !__GNUC_PREREQ (5,0)
+
/* %eax, %ecx, %edx and %esi contain the values expected by the kernel.
%edi points to a structure with the values of %ebx, %edi and %ebp. */
@@ -48,3 +50,4 @@ ENTRY (__libc_do_syscall)
cfi_restore (ebx)
ret
END (__libc_do_syscall)
+#endif
@@ -29,6 +29,7 @@ lockf64 (int fd, int cmd, off64_t len64)
{
struct flock64 fl64;
int cmd64;
+ int result;
memset ((char *) &fl64, '\0', sizeof (fl64));
fl64.l_whence = SEEK_CUR;
@@ -41,12 +42,13 @@ lockf64 (int fd, int cmd, off64_t len64)
/* Test the lock: return 0 if FD is unlocked or locked by this process;
return -1, set errno to EACCES, if another process holds the lock. */
fl64.l_type = F_RDLCK;
- if (INLINE_SYSCALL (fcntl64, 3, fd, F_GETLK64, &fl64) < 0)
- return -1;
+ INTERNAL_SYSCALL_DECL (err);
+ result = INTERNAL_SYSCALL (fcntl64, err, 3, fd, F_GETLK64, &fl64);
+ if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
+ return INLINE_SYSCALL_ERROR_RETURN (-result);
if (fl64.l_type == F_UNLCK || fl64.l_pid == __getpid ())
return 0;
- __set_errno (EACCES);
- return -1;
+ return INLINE_SYSCALL_ERROR_RETURN (EACCES);
case F_ULOCK:
fl64.l_type = F_UNLCK;
cmd64 = F_SETLK64;
@@ -61,8 +63,7 @@ lockf64 (int fd, int cmd, off64_t len64)
break;
default:
- __set_errno (EINVAL);
- return -1;
+ return INLINE_SYSCALL_ERROR_RETURN (EINVAL);
}
return INLINE_SYSCALL (fcntl64, 3, fd, cmd64, &fl64);
}
@@ -43,10 +43,12 @@ __lxstat (int vers, const char *name, struct stat *buf)
{
struct stat64 buf64;
- result = INLINE_SYSCALL (lstat64, 2, name, &buf64);
- if (result == 0)
- result = __xstat32_conv (vers, &buf64, buf);
- return result;
+ INTERNAL_SYSCALL_DECL (err);
+ result = INTERNAL_SYSCALL (lstat64, err, 2, name, &buf64);
+ if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
+ return INLINE_SYSCALL_ERROR_RETURN (-result);
+ else
+ return __xstat32_conv (vers, &buf64, buf);
}
}
@@ -27,10 +27,7 @@ setegid (gid)
int result;
if (gid == (gid_t) ~0)
- {
- __set_errno (EINVAL);
- return -1;
- }
+ INLINE_SYSCALL_ERROR_RETURN (EINVAL);
result = INLINE_SETXID_SYSCALL (setresgid32, 3, -1, gid, -1);
@@ -26,10 +26,7 @@ seteuid (uid_t uid)
int result;
if (uid == (uid_t) ~0)
- {
- __set_errno (EINVAL);
- return -1;
- }
+ INLINE_SYSCALL_ERROR_RETURN (EINVAL);
result = INLINE_SETXID_SYSCALL (setresuid32, 3, -1, uid, -1);
@@ -69,11 +69,13 @@ __libc_sigaction (int sig, const struct sigaction *act, struct sigaction *oact)
/* XXX The size argument hopefully will have to be changed to the
real size of the user-level sigset_t. */
- result = INLINE_SYSCALL (rt_sigaction, 4,
- sig, act ? &kact : NULL,
- oact ? &koact : NULL, _NSIG / 8);
-
- if (oact && result >= 0)
+ INTERNAL_SYSCALL_DECL (err);
+ result = INTERNAL_SYSCALL (rt_sigaction, err, 4,
+ sig, act ? &kact : NULL,
+ oact ? &koact : NULL, _NSIG / 8);
+ if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
+ return INLINE_SYSCALL_ERROR_RETURN (-result);
+ else if (oact && result >= 0)
{
oact->sa_handler = koact.k_sa_handler;
memcpy (&oact->sa_mask, &koact.sa_mask, sizeof (sigset_t));
deleted file mode 100644
@@ -1,40 +0,0 @@
-/* Copyright (C) 1995-2015 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-
-/* The following code is only used in the shared library when we
- compile the reentrant version. Otherwise each system call defines
- each own version. */
-
-#ifndef PIC
-
-/* The syscall stubs jump here when they detect an error.
- The code for Linux is almost identical to the canonical Unix/i386
- code, except that the error number in %eax is negated. */
-
-#undef CALL_MCOUNT
-#define CALL_MCOUNT /* Don't insert the profiling call, it clobbers %eax. */
-
- .text
-ENTRY (__syscall_error)
- negl %eax
-
-#define __syscall_error __syscall_error_1
-#include <sysdeps/unix/i386/sysdep.S>
-
-#endif /* !PIC */
new file mode 100644
@@ -0,0 +1,30 @@
+/* Copyright (C) 2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <errno.h>
+#include <sysdep.h>
+
+/* This routine is jumped to by all the syscall handlers, to stash
+ an error number into errno. ERROR is the negative error number
+ returned from the x86 kernel. */
+int
+__attribute__ ((__regparm__ (1)))
+__syscall_error (int error)
+{
+ __set_errno (-error);
+ return -1;
+}
@@ -56,11 +56,7 @@
/* We don't want the label for the error handle to be global when we define
it here. */
-#ifdef PIC
-# define SYSCALL_ERROR_LABEL 0f
-#else
-# define SYSCALL_ERROR_LABEL syscall_error
-#endif
+#define SYSCALL_ERROR_LABEL __syscall_error
#undef PSEUDO
#define PSEUDO(name, syscall_name, args) \
@@ -101,55 +97,7 @@
#define ret_ERRVAL ret
-#ifndef PIC
-# define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.S is used. */
-#else
-
-# if RTLD_PRIVATE_ERRNO
-# define SYSCALL_ERROR_HANDLER \
-0:SETUP_PIC_REG(cx); \
- addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
- negl %eax; \
- movl %eax, rtld_errno@GOTOFF(%ecx); \
- orl $-1, %eax; \
- ret;
-
-# elif defined _LIBC_REENTRANT
-
-# if IS_IN (libc)
-# define SYSCALL_ERROR_ERRNO __libc_errno
-# else
-# define SYSCALL_ERROR_ERRNO errno
-# endif
-# define SYSCALL_ERROR_HANDLER \
-0:SETUP_PIC_REG (cx); \
- addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
- movl SYSCALL_ERROR_ERRNO@GOTNTPOFF(%ecx), %ecx; \
- negl %eax; \
- SYSCALL_ERROR_HANDLER_TLS_STORE (%eax, %ecx); \
- orl $-1, %eax; \
- ret;
-# ifndef NO_TLS_DIRECT_SEG_REFS
-# define SYSCALL_ERROR_HANDLER_TLS_STORE(src, destoff) \
- movl src, %gs:(destoff)
-# else
-# define SYSCALL_ERROR_HANDLER_TLS_STORE(src, destoff) \
- addl %gs:0, destoff; \
- movl src, (destoff)
-# endif
-# else
-/* Store (- %eax) into errno through the GOT. */
-# define SYSCALL_ERROR_HANDLER \
-0:SETUP_PIC_REG(cx); \
- addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
- negl %eax; \
- movl errno@GOT(%ecx), %ecx; \
- movl %eax, (%ecx); \
- orl $-1, %eax; \
- ret;
-# endif /* _LIBC_REENTRANT */
-#endif /* PIC */
-
+#define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.c is used. */
/* The original calling convention for system calls on Linux/i386 is
to use int $0x80. */
@@ -276,6 +224,10 @@
#else /* !__ASSEMBLER__ */
+extern int __syscall_error (int)
+ attribute_hidden __attribute__ ((__regparm__ (1)));
+
+#if !__GNUC_PREREQ (5,0)
/* We need some help from the assembler to generate optimal code. We
define some macros here which later will be used. */
asm (".L__X'%ebx = 1\n\t"
@@ -315,11 +267,20 @@ struct libc_do_syscall_args
{
int ebx, edi, ebp;
};
+#endif
/* Define a macro which expands inline into the wrapper code for a system
call. */
#undef INLINE_SYSCALL
-#define INLINE_SYSCALL(name, nr, args...) \
+#if IS_IN (libc)
+# define INLINE_SYSCALL(name, nr, args...) \
+ ({ \
+ unsigned int resultvar = INTERNAL_SYSCALL (name, , nr, args); \
+ __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (resultvar, )) \
+ ? __syscall_error (-INTERNAL_SYSCALL_ERRNO (resultvar, )) \
+ : (int) resultvar; })
+#else
+# define INLINE_SYSCALL(name, nr, args...) \
({ \
unsigned int resultvar = INTERNAL_SYSCALL (name, , nr, args); \
if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (resultvar, ))) \
@@ -328,6 +289,14 @@ struct libc_do_syscall_args
resultvar = 0xffffffff; \
} \
(int) resultvar; })
+#endif
+
+/* Set error number and return -1. Return the internal function,
+ __syscall_error, which sets errno from the negative error number
+ and returns -1, to avoid PIC. */
+#undef INLINE_SYSCALL_ERROR_RETURN
+#define INLINE_SYSCALL_ERROR_RETURN(resultvar) \
+ __syscall_error (-(resultvar))
/* List of system calls which are supported as vsyscalls. */
# define HAVE_CLOCK_GETTIME_VSYSCALL 1
@@ -355,8 +324,12 @@ struct libc_do_syscall_args
INTERNAL_SYSCALL_MAIN_INLINE(name, err, 5, args)
/* Each object using 6-argument inline syscalls must include a
definition of __libc_do_syscall. */
-#define INTERNAL_SYSCALL_MAIN_6(name, err, arg1, arg2, arg3, \
- arg4, arg5, arg6) \
+#if __GNUC_PREREQ (5,0)
+# define INTERNAL_SYSCALL_MAIN_6(name, err, args...) \
+ INTERNAL_SYSCALL_MAIN_INLINE(name, err, 6, args)
+#else /* GCC 5 */
+# define INTERNAL_SYSCALL_MAIN_6(name, err, arg1, arg2, arg3, \
+ arg4, arg5, arg6) \
struct libc_do_syscall_args _xv = \
{ \
(int) (arg1), \
@@ -369,14 +342,52 @@ struct libc_do_syscall_args
: "=a" (resultvar) \
: "i" (__NR_##name), "c" (arg2), "d" (arg3), "S" (arg4), "D" (&_xv) \
: "memory", "cc")
+#endif /* GCC 5 */
#define INTERNAL_SYSCALL(name, err, nr, args...) \
({ \
register unsigned int resultvar; \
INTERNAL_SYSCALL_MAIN_##nr (name, err, args); \
(int) resultvar; })
#ifdef I386_USE_SYSENTER
-# ifdef SHARED
-# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
+# if __GNUC_PREREQ (5,0)
+# ifdef SHARED
+# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
+ LOADREGS_##nr(args) \
+ asm volatile ( \
+ "call *%%gs:%P2" \
+ : "=a" (resultvar) \
+ : "a" (__NR_##name), "i" (offsetof (tcbhead_t, sysinfo)) \
+ ASMARGS_##nr(args) : "memory", "cc")
+# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
+ ({ \
+ register unsigned int resultvar; \
+ LOADREGS_##nr(args) \
+ asm volatile ( \
+ "call *%%gs:%P2" \
+ : "=a" (resultvar) \
+ : "a" (name), "i" (offsetof (tcbhead_t, sysinfo)) \
+ ASMARGS_##nr(args) : "memory", "cc"); \
+ (int) resultvar; })
+# else
+# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
+ LOADREGS_##nr(args) \
+ asm volatile ( \
+ "call *_dl_sysinfo" \
+ : "=a" (resultvar) \
+ : "a" (__NR_##name) ASMARGS_##nr(args) : "memory", "cc")
+# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
+ ({ \
+ register unsigned int resultvar; \
+ LOADREGS_##nr(args) \
+ asm volatile ( \
+ "call *_dl_sysinfo" \
+ : "=a" (resultvar) \
+ : "a" (name) ASMARGS_##nr(args) : "memory", "cc"); \
+ (int) resultvar; })
+# endif
+# else /* GCC 5 */
+# ifdef SHARED
+# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
EXTRAVAR_##nr \
asm volatile ( \
LOADARGS_##nr \
@@ -386,7 +397,7 @@ struct libc_do_syscall_args
: "=a" (resultvar) \
: "i" (__NR_##name), "i" (offsetof (tcbhead_t, sysinfo)) \
ASMFMT_##nr(args) : "memory", "cc")
-# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
+# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
({ \
register unsigned int resultvar; \
EXTRAVAR_##nr \
@@ -398,8 +409,8 @@ struct libc_do_syscall_args
: "0" (name), "i" (offsetof (tcbhead_t, sysinfo)) \
ASMFMT_##nr(args) : "memory", "cc"); \
(int) resultvar; })
-# else
-# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
+# else
+# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
EXTRAVAR_##nr \
asm volatile ( \
LOADARGS_##nr \
@@ -408,7 +419,7 @@ struct libc_do_syscall_args
RESTOREARGS_##nr \
: "=a" (resultvar) \
: "i" (__NR_##name) ASMFMT_##nr(args) : "memory", "cc")
-# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
+# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
({ \
register unsigned int resultvar; \
EXTRAVAR_##nr \
@@ -419,9 +430,27 @@ struct libc_do_syscall_args
: "=a" (resultvar) \
: "0" (name) ASMFMT_##nr(args) : "memory", "cc"); \
(int) resultvar; })
-# endif
+# endif
+# endif /* GCC 5 */
#else
-# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
+# if __GNUC_PREREQ (5,0)
+# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
+ LOADREGS_##nr(args) \
+ asm volatile ( \
+ "int $0x80" \
+ : "=a" (resultvar) \
+ : "a" (__NR_##name) ASMARGS_##nr(args) : "memory", "cc")
+# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
+ ({ \
+ register unsigned int resultvar; \
+ LOADREGS_##nr(args) \
+ asm volatile ( \
+ "int $0x80" \
+ : "=a" (resultvar) \
+ : "a" (name) ASMARGS_##nr(args) : "memory", "cc"); \
+ (int) resultvar; })
+# else /* GCC 5 */
+# define INTERNAL_SYSCALL_MAIN_INLINE(name, err, nr, args...) \
EXTRAVAR_##nr \
asm volatile ( \
LOADARGS_##nr \
@@ -430,7 +459,7 @@ struct libc_do_syscall_args
RESTOREARGS_##nr \
: "=a" (resultvar) \
: "i" (__NR_##name) ASMFMT_##nr(args) : "memory", "cc")
-# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
+# define INTERNAL_SYSCALL_NCS(name, err, nr, args...) \
({ \
register unsigned int resultvar; \
EXTRAVAR_##nr \
@@ -441,6 +470,7 @@ struct libc_do_syscall_args
: "=a" (resultvar) \
: "0" (name) ASMFMT_##nr(args) : "memory", "cc"); \
(int) resultvar; })
+# endif /* GCC 5 */
#endif
#undef INTERNAL_SYSCALL_DECL
@@ -505,6 +535,36 @@ struct libc_do_syscall_args
# define RESTOREARGS_5
#endif
+#if __GNUC_PREREQ (5,0)
+# define LOADREGS_0()
+# define ASMARGS_0()
+# define LOADREGS_1(arg1) \
+ LOADREGS_0 ()
+# define ASMARGS_1(arg1) \
+ ASMARGS_0 (), "b" ((unsigned int) (arg1))
+# define LOADREGS_2(arg1, arg2) \
+ LOADREGS_1 (arg1)
+# define ASMARGS_2(arg1, arg2) \
+ ASMARGS_1 (arg1), "c" ((unsigned int) (arg2))
+# define LOADREGS_3(arg1, arg2, arg3) \
+ LOADREGS_2 (arg1, arg2)
+# define ASMARGS_3(arg1, arg2, arg3) \
+ ASMARGS_2 (arg1, arg2), "d" ((unsigned int) (arg3))
+# define LOADREGS_4(arg1, arg2, arg3, arg4) \
+ LOADREGS_3 (arg1, arg2, arg3)
+# define ASMARGS_4(arg1, arg2, arg3, arg4) \
+ ASMARGS_3 (arg1, arg2, arg3), "S" ((unsigned int) (arg4))
+# define LOADREGS_5(arg1, arg2, arg3, arg4, arg5) \
+ LOADREGS_4 (arg1, arg2, arg3, arg4)
+# define ASMARGS_5(arg1, arg2, arg3, arg4, arg5) \
+ ASMARGS_4 (arg1, arg2, arg3, arg4), "D" ((unsigned int) (arg5))
+# define LOADREGS_6(arg1, arg2, arg3, arg4, arg5, arg6) \
+ register unsigned int _a6 asm ("ebp") = (unsigned int) (arg6); \
+ LOADREGS_5 (arg1, arg2, arg3, arg4, arg5)
+# define ASMARGS_6(arg1, arg2, arg3, arg4, arg5, arg6) \
+ ASMARGS_5 (arg1, arg2, arg3, arg4, arg5), "r" (_a6)
+#endif /* GCC 5 */
+
#define ASMFMT_0()
#ifdef __PIC__
# define ASMFMT_1(arg1) \
@@ -43,10 +43,12 @@ __xstat (int vers, const char *name, struct stat *buf)
{
struct stat64 buf64;
- result = INLINE_SYSCALL (stat64, 2, name, &buf64);
- if (result == 0)
- result = __xstat32_conv (vers, &buf64, buf);
- return result;
+ INTERNAL_SYSCALL_DECL (err);
+ result = INTERNAL_SYSCALL (stat64, err, 2, name, &buf64);
+ if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, )))
+ return INLINE_SYSCALL_ERROR_RETURN (-result);
+ else
+ return __xstat32_conv (vers, &buf64, buf);
}
}
hidden_def (__xstat)