@@ -27,199 +27,155 @@
*
*/
-/* Offsets for data table __svml_slog2_data_internal
- */
-#define MinNorm 0
-#define MaxNorm 32
-#define iBrkValue 64
-#define iOffExpoMask 96
-#define One 128
-#define sPoly 160
+#define LOCAL_DATA_NAME __svml_slog2_data_internal
+#include "svml_s_common_avx2_rodata_offsets.h"
+/* Offsets for data table __svml_slog2_data_internal. */
+#define _Coeff_9 0
+#define _Coeff_8 32
+#define _Coeff_7 64
+#define _Coeff_6 96
+#define _Coeff_5 128
+#define _Coeff_4 160
+#define _Coeff_3 192
+#define _Coeff_2 224
+#define _Coeff_1 256
#include <sysdep.h>
.section .text.avx2, "ax", @progbits
ENTRY(_ZGVdN8v_log2f_avx2)
- pushq %rbp
- cfi_def_cfa_offset(16)
- movq %rsp, %rbp
- cfi_def_cfa(6, 16)
- cfi_offset(6, -16)
- andq $-32, %rsp
- subq $96, %rsp
+ /* reduction: compute r, n. */
+ vmovups COMMON_DATA(_IBrkValue)(%rip), %ymm4
+
- /* reduction: compute r, n */
- vmovups iBrkValue+__svml_slog2_data_internal(%rip), %ymm4
- vmovups sPoly+64+__svml_slog2_data_internal(%rip), %ymm9
- vmovups sPoly+128+__svml_slog2_data_internal(%rip), %ymm10
- vmovups sPoly+192+__svml_slog2_data_internal(%rip), %ymm12
vpsubd %ymm4, %ymm0, %ymm1
- vcmplt_oqps MinNorm+__svml_slog2_data_internal(%rip), %ymm0, %ymm5
- vcmpnle_uqps MaxNorm+__svml_slog2_data_internal(%rip), %ymm0, %ymm6
- vpand iOffExpoMask+__svml_slog2_data_internal(%rip), %ymm1, %ymm3
+ vmovups COMMON_DATA(_NotiOffExpoMask)(%rip), %ymm2
+ vpsubd %ymm2, %ymm0, %ymm5
+ vmovups COMMON_DATA(_ILoRange)(%rip), %ymm6
+ vpcmpgtd %ymm5, %ymm6, %ymm6
+ vpandn %ymm1, %ymm2, %ymm3
vpsrad $23, %ymm1, %ymm2
- vmovups sPoly+__svml_slog2_data_internal(%rip), %ymm1
- vpaddd %ymm4, %ymm3, %ymm8
- vcvtdq2ps %ymm2, %ymm14
- vsubps One+__svml_slog2_data_internal(%rip), %ymm8, %ymm13
- vfmadd213ps sPoly+32+__svml_slog2_data_internal(%rip), %ymm13, %ymm1
- vfmadd213ps sPoly+96+__svml_slog2_data_internal(%rip), %ymm13, %ymm9
- vmulps %ymm13, %ymm13, %ymm11
- vfmadd213ps sPoly+160+__svml_slog2_data_internal(%rip), %ymm13, %ymm10
- vfmadd213ps sPoly+224+__svml_slog2_data_internal(%rip), %ymm13, %ymm12
- vfmadd213ps %ymm9, %ymm11, %ymm1
- vfmadd213ps %ymm10, %ymm11, %ymm1
- vfmadd213ps %ymm12, %ymm11, %ymm1
- vfmadd213ps sPoly+256+__svml_slog2_data_internal(%rip), %ymm13, %ymm1
- vorps %ymm6, %ymm5, %ymm7
-
- /* combine and get argument value range mask */
- vmovmskps %ymm7, %edx
- vfmadd213ps %ymm14, %ymm13, %ymm1
+ vmovups LOCAL_DATA(_Coeff_9)(%rip), %ymm1
+ vpaddd %ymm4, %ymm3, %ymm3
+
+ vsubps COMMON_DATA(_OneF)(%rip), %ymm3, %ymm3
+ vfmadd213ps LOCAL_DATA(_Coeff_8)(%rip), %ymm3, %ymm1
+ vmovups LOCAL_DATA(_Coeff_7)(%rip), %ymm4
+ vfmadd213ps LOCAL_DATA(_Coeff_6)(%rip), %ymm3, %ymm4
+ vmulps %ymm3, %ymm3, %ymm5
+ vmovups LOCAL_DATA(_Coeff_5)(%rip), %ymm7
+ vfmadd213ps LOCAL_DATA(_Coeff_4)(%rip), %ymm3, %ymm7
+ vmovmskps %ymm6, %edx
+
+ vmovups LOCAL_DATA(_Coeff_3)(%rip), %ymm6
+ vfmadd213ps LOCAL_DATA(_Coeff_2)(%rip), %ymm3, %ymm6
+ vfmadd213ps %ymm4, %ymm5, %ymm1
+ vfmadd213ps %ymm7, %ymm5, %ymm1
+ vfmadd213ps %ymm6, %ymm5, %ymm1
+ vfmadd213ps LOCAL_DATA(_Coeff_1)(%rip), %ymm3, %ymm1
+ vcvtdq2ps %ymm2, %ymm2
+
+ /* combine and get argument value range mask. */
+
+ vfmadd213ps %ymm2, %ymm3, %ymm1
testl %edx, %edx
- /* Go to special inputs processing branch */
+ /* Go to special inputs processing branch. */
jne L(SPECIAL_VALUES_BRANCH)
- # LOE rbx r12 r13 r14 r15 edx ymm0 ymm1
- /* Restore registers
- * and exit the function
- */
-L(EXIT):
vmovaps %ymm1, %ymm0
- movq %rbp, %rsp
- popq %rbp
- cfi_def_cfa(7, 8)
- cfi_restore(6)
ret
- cfi_def_cfa(6, 16)
- cfi_offset(6, -16)
+ /* Cold case. edx has 1s where there was a special value that
+ needs to be handled by a atanhf call. Optimize for code size
+ more so than speed here. */
+L(SPECIAL_VALUES_BRANCH):
- /* Branch to process
- * special inputs
- */
+ /* Use r13 to save/restore the stack. This allows us to use rbp
+ as callee save register saving code size. */
+ pushq %r13
+ cfi_adjust_cfa_offset (8)
+ cfi_offset (r13, -16)
+ /* Need to callee save registers to preserve state across tanhf
+ calls. */
+ pushq %rbx
+ cfi_adjust_cfa_offset (8)
+ cfi_offset (rbx, -24)
+ pushq %rbp
+ cfi_adjust_cfa_offset (8)
+ cfi_offset (rbp, -32)
+ movq %rsp, %r13
+ cfi_def_cfa_register (r13)
-L(SPECIAL_VALUES_BRANCH):
- vmovups %ymm0, 32(%rsp)
- vmovups %ymm1, 64(%rsp)
- # LOE rbx r12 r13 r14 r15 edx ymm1
+ /* Align stack and make room for 2x ymm vectors. */
+ andq $-32, %rsp
+ addq $-64, %rsp
- xorl %eax, %eax
- # LOE rbx r12 r13 r14 r15 eax edx
+ /* Save all already computed inputs. */
+ vmovups %ymm1, (%rsp)
+ /* Save original input (ymm0 unchanged up to this point). */
+ vmovups %ymm0, 32(%rsp)
vzeroupper
- movq %r12, 16(%rsp)
- /* DW_CFA_expression: r12 (r12) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -80; DW_OP_plus) */
- .cfi_escape 0x10, 0x0c, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xb0, 0xff, 0xff, 0xff, 0x22
- movl %eax, %r12d
- movq %r13, 8(%rsp)
- /* DW_CFA_expression: r13 (r13) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -88; DW_OP_plus) */
- .cfi_escape 0x10, 0x0d, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa8, 0xff, 0xff, 0xff, 0x22
- movl %edx, %r13d
- movq %r14, (%rsp)
- /* DW_CFA_expression: r14 (r14) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -96; DW_OP_plus) */
- .cfi_escape 0x10, 0x0e, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa0, 0xff, 0xff, 0xff, 0x22
- # LOE rbx r15 r12d r13d
-
- /* Range mask
- * bits check
- */
-
-L(RANGEMASK_CHECK):
- btl %r12d, %r13d
-
- /* Call scalar math function */
- jc L(SCALAR_MATH_CALL)
- # LOE rbx r15 r12d r13d
-
- /* Special inputs
- * processing loop
- */
+ /* edx has 1s where there was a special value that needs to be
+ handled by a atanhf call. */
+ movl %edx, %ebx
L(SPECIAL_VALUES_LOOP):
- incl %r12d
- cmpl $8, %r12d
-
- /* Check bits in range mask */
- jl L(RANGEMASK_CHECK)
- # LOE rbx r15 r12d r13d
-
- movq 16(%rsp), %r12
- cfi_restore(12)
- movq 8(%rsp), %r13
- cfi_restore(13)
- movq (%rsp), %r14
- cfi_restore(14)
- vmovups 64(%rsp), %ymm1
-
- /* Go to exit */
- jmp L(EXIT)
- /* DW_CFA_expression: r12 (r12) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -80; DW_OP_plus) */
- .cfi_escape 0x10, 0x0c, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xb0, 0xff, 0xff, 0xff, 0x22
- /* DW_CFA_expression: r13 (r13) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -88; DW_OP_plus) */
- .cfi_escape 0x10, 0x0d, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa8, 0xff, 0xff, 0xff, 0x22
- /* DW_CFA_expression: r14 (r14) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -96; DW_OP_plus) */
- .cfi_escape 0x10, 0x0e, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa0, 0xff, 0xff, 0xff, 0x22
- # LOE rbx r12 r13 r14 r15 ymm1
-
- /* Scalar math fucntion call
- * to process special input
- */
-
-L(SCALAR_MATH_CALL):
- movl %r12d, %r14d
- vmovss 32(%rsp, %r14, 4), %xmm0
+
+ /* use rbp as index for special value that is saved across calls
+ to atanhf. We technically don't need a callee save register
+ here as offset to rsp is always [0, 28] so we can restore
+ rsp by realigning to 64. Essentially the tradeoff is 1 extra
+ save/restore vs 2 extra instructions in the loop. Realigning
+ also costs more code size. */
+ xorl %ebp, %ebp
+ tzcntl %ebx, %ebp
+
+ /* Scalar math fucntion call to process special input. */
+ vmovss 32(%rsp, %rbp, 4), %xmm0
call log2f@PLT
- # LOE rbx r14 r15 r12d r13d xmm0
+ /* No good way to avoid the store-forwarding fault this will
+ cause on return. `lfence` avoids the SF fault but at greater
+ cost as it serialized stack/callee save restoration. */
+ vmovss %xmm0, (%rsp, %rbp, 4)
+
+ blsrl %ebx, %ebx
+ jnz L(SPECIAL_VALUES_LOOP)
+
- vmovss %xmm0, 64(%rsp, %r14, 4)
- /* Process special inputs in loop */
- jmp L(SPECIAL_VALUES_LOOP)
- # LOE rbx r15 r12d r13d
+ /* All results have been written to (%rsp). */
+ vmovups (%rsp), %ymm0
+ /* Restore rsp. */
+ movq %r13, %rsp
+ cfi_def_cfa_register (rsp)
+ /* Restore callee save registers. */
+ popq %rbp
+ cfi_adjust_cfa_offset (-8)
+ cfi_restore (rbp)
+ popq %rbx
+ cfi_adjust_cfa_offset (-8)
+ cfi_restore (rbp)
+ popq %r13
+ cfi_adjust_cfa_offset (-8)
+ cfi_restore (r13)
+ ret
END(_ZGVdN8v_log2f_avx2)
- .section .rodata, "a"
+ .section .rodata.avx2, "a"
.align 32
-#ifdef __svml_slog2_data_internal_typedef
-typedef unsigned int VUINT32;
-typedef struct {
- __declspec(align(32)) VUINT32 MinNorm[8][1];
- __declspec(align(32)) VUINT32 MaxNorm[8][1];
- __declspec(align(32)) VUINT32 iBrkValue[8][1];
- __declspec(align(32)) VUINT32 iOffExpoMask[8][1];
- __declspec(align(32)) VUINT32 One[8][1];
- __declspec(align(32)) VUINT32 sPoly[9][8][1];
-} __svml_slog2_data_internal;
-#endif
-__svml_slog2_data_internal:
- /* MinNorm */
- .long 0x00800000, 0x00800000, 0x00800000, 0x00800000, 0x00800000, 0x00800000, 0x00800000, 0x00800000
- /* MaxNorm */
- .align 32
- .long 0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff
- /* iBrkValue = SP 2/3 */
- .align 32
- .long 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab
- /* iOffExpoMask = SP significand mask */
- .align 32
- .long 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff
- /* sOne = SP 1.0 */
- .align 32
- .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000
- /* spoly[9] */
- .align 32
- .long 0x3e554012, 0x3e554012, 0x3e554012, 0x3e554012, 0x3e554012, 0x3e554012, 0x3e554012, 0x3e554012 /* coeff9 */
- .long 0xbe638E14, 0xbe638E14, 0xbe638E14, 0xbe638E14, 0xbe638E14, 0xbe638E14, 0xbe638E14, 0xbe638E14 /* coeff8 */
- .long 0x3e4D660B, 0x3e4D660B, 0x3e4D660B, 0x3e4D660B, 0x3e4D660B, 0x3e4D660B, 0x3e4D660B, 0x3e4D660B /* coeff7 */
- .long 0xbe727824, 0xbe727824, 0xbe727824, 0xbe727824, 0xbe727824, 0xbe727824, 0xbe727824, 0xbe727824 /* coeff6 */
- .long 0x3e93DD07, 0x3e93DD07, 0x3e93DD07, 0x3e93DD07, 0x3e93DD07, 0x3e93DD07, 0x3e93DD07, 0x3e93DD07 /* coeff5 */
- .long 0xbeB8B969, 0xbeB8B969, 0xbeB8B969, 0xbeB8B969, 0xbeB8B969, 0xbeB8B969, 0xbeB8B969, 0xbeB8B969 /* coeff4 */
- .long 0x3eF637C0, 0x3eF637C0, 0x3eF637C0, 0x3eF637C0, 0x3eF637C0, 0x3eF637C0, 0x3eF637C0, 0x3eF637C0 /* coeff3 */
- .long 0xbf38AA2B, 0xbf38AA2B, 0xbf38AA2B, 0xbf38AA2B, 0xbf38AA2B, 0xbf38AA2B, 0xbf38AA2B, 0xbf38AA2B /* coeff2 */
- .long 0x3fB8AA3B, 0x3fB8AA3B, 0x3fB8AA3B, 0x3fB8AA3B, 0x3fB8AA3B, 0x3fB8AA3B, 0x3fB8AA3B, 0x3fB8AA3B /* coeff1 */
- .align 32
- .type __svml_slog2_data_internal, @object
- .size __svml_slog2_data_internal, .-__svml_slog2_data_internal
+LOCAL_DATA_NAME:
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_9, 0x3e554012)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_8, 0xbe638e14)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_7, 0x3e4d660b)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_6, 0xbe727824)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_5, 0x3e93dd07)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_4, 0xbeb8b969)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_3, 0x3ef637c0)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_2, 0xbf38aa2b)
+ DATA_VEC (LOCAL_DATA_NAME, _Coeff_1, 0x3fb8aa3b)
+
+ .type LOCAL_DATA_NAME, @object
+ .size LOCAL_DATA_NAME, .-LOCAL_DATA_NAME