diff mbox series

x86: Set rep_movsb_threshold to 2112 on processors with FSRM

Message ID 20210430182442.3612464-1-hjl.tools@gmail.com
State Superseded
Headers
Series x86: Set rep_movsb_threshold to 2112 on processors with FSRM |

Commit Message

H.J. Lu April 30, 2021, 6:24 p.m. UTC 
  The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
that REP MOVSB became faster after 2112 bytes:

                                    Vector + REP MOVSB  REP MOVSB
length=2112, align1=0, align2=0:        24.20             24.40
length=2112, align1=1, align2=0:        26.07             23.13
length=2112, align1=0, align2=1:        27.18             28.13
length=2112, align1=1, align2=1:        26.23             25.16
length=2176, align1=0, align2=0:        23.18             22.52
length=2176, align1=2, align2=0:        25.45             22.52
length=2176, align1=0, align2=2:        27.14             27.82
length=2176, align1=2, align2=2:        22.73             25.56
length=2240, align1=0, align2=0:        24.62             24.25
length=2240, align1=3, align2=0:        29.77             27.15
length=2240, align1=0, align2=3:        35.55             29.93
length=2240, align1=3, align2=3:        34.49             25.15
length=2304, align1=0, align2=0:        34.75             26.64
length=2304, align1=4, align2=0:        32.09             22.63
length=2304, align1=0, align2=4:        28.43             31.24

Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
fast short REP MOVSB (FSRM).

	* sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
	rep_movsb_threshold to 2112 on processors with fast short REP
	MOVSB (FSRM).
---
 sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

Comments

Noah Goldstein April 30, 2021, 10:59 p.m. UTC | #1 
On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>
> The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
> that REP MOVSB became faster after 2112 bytes:
>
>                                     Vector + REP MOVSB  REP MOVSB
> length=2112, align1=0, align2=0:        24.20             24.40
> length=2112, align1=1, align2=0:        26.07             23.13
> length=2112, align1=0, align2=1:        27.18             28.13
> length=2112, align1=1, align2=1:        26.23             25.16
> length=2176, align1=0, align2=0:        23.18             22.52
> length=2176, align1=2, align2=0:        25.45             22.52
> length=2176, align1=0, align2=2:        27.14             27.82
> length=2176, align1=2, align2=2:        22.73             25.56
> length=2240, align1=0, align2=0:        24.62             24.25
> length=2240, align1=3, align2=0:        29.77             27.15
> length=2240, align1=0, align2=3:        35.55             29.93
> length=2240, align1=3, align2=3:        34.49             25.15
> length=2304, align1=0, align2=0:        34.75             26.64
> length=2304, align1=4, align2=0:        32.09             22.63
> length=2304, align1=0, align2=4:        28.43             31.24
>

Do you know what is happening at:
length=2304, align1=0, align2=4:        28.43             31.24

Seems align2 > align1 gets worse for larger sizes as well.

I.e from my Icelake (1 run):
                                                              w/o
Patch      w/ Patch
length=3648, align1=0, align2=25:        72.91             83.99
length=3712, align1=0, align2=26:        72.52             84.92
length=3776, align1=0, align2=27:        76.16             84.02
length=3840, align1=0, align2=28:        75.44             90.13
length=3904, align1=0, align2=29:        81.62             84.43
length=3968, align1=0, align2=30:        76.82             93.39
length=4032, align1=0, align2=31:        80.01             89.89
length=4096, align1=0, align2=32:        72.89             97.50

> Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
> fast short REP MOVSB (FSRM).
>
>         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
>         rep_movsb_threshold to 2112 on processors with fast short REP
>         MOVSB (FSRM).
> ---
>  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--
>  1 file changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
> index d9944250fc..3f04fb5019 100644
> --- a/sysdeps/x86/dl-cacheinfo.h
> +++ b/sysdeps/x86/dl-cacheinfo.h
> @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
>    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
>        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
>      {
> -      rep_movsb_threshold = 2048 * (64 / 16);
> +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> +       rep_movsb_threshold = 2112;
> +      else
> +       rep_movsb_threshold = 2048 * (64 / 16);
>  #if HAVE_TUNABLES
>        minimum_rep_movsb_threshold = 64 * 8;
>  #endif
> @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
>    else if (CPU_FEATURE_PREFERRED_P (cpu_features,
>                                     AVX_Fast_Unaligned_Load))
>      {
> -      rep_movsb_threshold = 2048 * (32 / 16);
> +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> +       rep_movsb_threshold = 2112;
> +      else
> +       rep_movsb_threshold = 2048 * (32 / 16);
>  #if HAVE_TUNABLES
>        minimum_rep_movsb_threshold = 32 * 8;
>  #endif
> --
> 2.31.1
>
H.J. Lu April 30, 2021, 11:50 p.m. UTC | #2 
On Fri, Apr 30, 2021 at 3:59 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha
> <libc-alpha@sourceware.org> wrote:
> >
> > The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
> > that REP MOVSB became faster after 2112 bytes:
> >
> >                                     Vector + REP MOVSB  REP MOVSB
> > length=2112, align1=0, align2=0:        24.20             24.40
> > length=2112, align1=1, align2=0:        26.07             23.13
> > length=2112, align1=0, align2=1:        27.18             28.13
> > length=2112, align1=1, align2=1:        26.23             25.16
> > length=2176, align1=0, align2=0:        23.18             22.52
> > length=2176, align1=2, align2=0:        25.45             22.52
> > length=2176, align1=0, align2=2:        27.14             27.82
> > length=2176, align1=2, align2=2:        22.73             25.56
> > length=2240, align1=0, align2=0:        24.62             24.25
> > length=2240, align1=3, align2=0:        29.77             27.15
> > length=2240, align1=0, align2=3:        35.55             29.93
> > length=2240, align1=3, align2=3:        34.49             25.15
> > length=2304, align1=0, align2=0:        34.75             26.64
> > length=2304, align1=4, align2=0:        32.09             22.63
> > length=2304, align1=0, align2=4:        28.43             31.24
> >
>
> Do you know what is happening at:
> length=2304, align1=0, align2=4:        28.43             31.24
>
> Seems align2 > align1 gets worse for larger sizes as well.
>
> I.e from my Icelake (1 run):
>                                                               w/o
> Patch      w/ Patch
> length=3648, align1=0, align2=25:        72.91             83.99
> length=3712, align1=0, align2=26:        72.52             84.92
> length=3776, align1=0, align2=27:        76.16             84.02
> length=3840, align1=0, align2=28:        75.44             90.13
> length=3904, align1=0, align2=29:        81.62             84.43
> length=3968, align1=0, align2=30:        76.82             93.39
> length=4032, align1=0, align2=31:        80.01             89.89
> length=4096, align1=0, align2=32:        72.89             97.50

On Intel Core i7-1165G7 (Tiger Lake), I got

                                                                Vector
       REP MOVSB
      length=2112, align1=0, align2=1:        45.21        43.69
     length=2176, align1=0, align2=2:        45.20        45.82
     length=2240, align1=0, align2=3:        53.46        51.41
     length=2304, align1=0, align2=4:        47.65        48.28
     length=2368, align1=0, align2=5:        50.10        48.27
     length=2432, align1=0, align2=6:        50.10        50.70
     length=2496, align1=0, align2=7:        52.54        50.71
     length=2560, align1=0, align2=8:        52.83        53.15
     length=2624, align1=0, align2=9:        64.98        53.15
    length=2688, align1=0, align2=10:        54.98        55.59
    length=2752, align1=0, align2=11:        57.43        55.59
    length=2816, align1=0, align2=12:        57.87        58.03
    length=2880, align1=0, align2=13:        61.97        58.03
    length=2944, align1=0, align2=14:        71.97        60.79
    length=3008, align1=0, align2=15:        65.02        60.47
    length=3072, align1=0, align2=16:        65.39        63.23
    length=3136, align1=0, align2=17:        72.54        62.92
    length=3200, align1=0, align2=18:        69.91        65.39
    length=3264, align1=0, align2=19:        73.38        65.39
    length=3328, align1=0, align2=20:        73.50        67.81
    length=3392, align1=0, align2=21:        76.81        67.81
    length=3456, align1=0, align2=22:        76.88        70.79
    length=3520, align1=0, align2=23:        80.30        71.07
    length=3584, align1=0, align2=24:        80.25        72.69
    length=3648, align1=0, align2=25:        84.58        73.01
    length=3712, align1=0, align2=26:        85.02        75.13
    length=3776, align1=0, align2=27:        87.51        75.14
    length=3840, align1=0, align2=28:        87.20        77.67
    length=3904, align1=0, align2=29:        89.82        77.99
    length=3968, align1=0, align2=30:       102.59        80.45
    length=4032, align1=0, align2=31:        92.38        80.60
    length=4096, align1=0, align2=32:        85.69        84.69

REP MOVSB is a little bit faster.

> > Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
> > fast short REP MOVSB (FSRM).
> >
> >         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
> >         rep_movsb_threshold to 2112 on processors with fast short REP
> >         MOVSB (FSRM).
> > ---
> >  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--
> >  1 file changed, 8 insertions(+), 2 deletions(-)
> >
> > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
> > index d9944250fc..3f04fb5019 100644
> > --- a/sysdeps/x86/dl-cacheinfo.h
> > +++ b/sysdeps/x86/dl-cacheinfo.h
> > @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
> >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
> >      {
> > -      rep_movsb_threshold = 2048 * (64 / 16);
> > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> > +       rep_movsb_threshold = 2112;
> > +      else
> > +       rep_movsb_threshold = 2048 * (64 / 16);
> >  #if HAVE_TUNABLES
> >        minimum_rep_movsb_threshold = 64 * 8;
> >  #endif
> > @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,
> >                                     AVX_Fast_Unaligned_Load))
> >      {
> > -      rep_movsb_threshold = 2048 * (32 / 16);
> > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> > +       rep_movsb_threshold = 2112;
> > +      else
> > +       rep_movsb_threshold = 2048 * (32 / 16);
> >  #if HAVE_TUNABLES
> >        minimum_rep_movsb_threshold = 32 * 8;
> >  #endif
> > --
> > 2.31.1
> >
Noah Goldstein May 1, 2021, 12:17 a.m. UTC | #3 
On Fri, Apr 30, 2021 at 4:51 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>
> On Fri, Apr 30, 2021 at 3:59 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> >
> > On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha
> > <libc-alpha@sourceware.org> wrote:
> > >
> > > The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
> > > that REP MOVSB became faster after 2112 bytes:
> > >
> > >                                     Vector + REP MOVSB  REP MOVSB
> > > length=2112, align1=0, align2=0:        24.20             24.40
> > > length=2112, align1=1, align2=0:        26.07             23.13
> > > length=2112, align1=0, align2=1:        27.18             28.13
> > > length=2112, align1=1, align2=1:        26.23             25.16
> > > length=2176, align1=0, align2=0:        23.18             22.52
> > > length=2176, align1=2, align2=0:        25.45             22.52
> > > length=2176, align1=0, align2=2:        27.14             27.82
> > > length=2176, align1=2, align2=2:        22.73             25.56
> > > length=2240, align1=0, align2=0:        24.62             24.25
> > > length=2240, align1=3, align2=0:        29.77             27.15
> > > length=2240, align1=0, align2=3:        35.55             29.93
> > > length=2240, align1=3, align2=3:        34.49             25.15
> > > length=2304, align1=0, align2=0:        34.75             26.64
> > > length=2304, align1=4, align2=0:        32.09             22.63
> > > length=2304, align1=0, align2=4:        28.43             31.24
> > >
> >
> > Do you know what is happening at:
> > length=2304, align1=0, align2=4:        28.43             31.24
> >
> > Seems align2 > align1 gets worse for larger sizes as well.
> >
> > I.e from my Icelake (1 run):
> >                                                               w/o
> > Patch      w/ Patch
> > length=3648, align1=0, align2=25:        72.91             83.99
> > length=3712, align1=0, align2=26:        72.52             84.92
> > length=3776, align1=0, align2=27:        76.16             84.02
> > length=3840, align1=0, align2=28:        75.44             90.13
> > length=3904, align1=0, align2=29:        81.62             84.43
> > length=3968, align1=0, align2=30:        76.82             93.39
> > length=4032, align1=0, align2=31:        80.01             89.89
> > length=4096, align1=0, align2=32:        72.89             97.50
>
> On Intel Core i7-1165G7 (Tiger Lake), I got

Where the original numbers icl or tgl? But was just wondering
if the align2 != 0, align1 = 0 case on Icelake warranted more investigation.
That being said the change is a clear winner on Icelake as well
for any other alignment schema.

>
>                                                                 Vector
>        REP MOVSB
>       length=2112, align1=0, align2=1:        45.21        43.69
>      length=2176, align1=0, align2=2:        45.20        45.82
>      length=2240, align1=0, align2=3:        53.46        51.41
>      length=2304, align1=0, align2=4:        47.65        48.28
>      length=2368, align1=0, align2=5:        50.10        48.27
>      length=2432, align1=0, align2=6:        50.10        50.70
>      length=2496, align1=0, align2=7:        52.54        50.71
>      length=2560, align1=0, align2=8:        52.83        53.15
>      length=2624, align1=0, align2=9:        64.98        53.15
>     length=2688, align1=0, align2=10:        54.98        55.59
>     length=2752, align1=0, align2=11:        57.43        55.59
>     length=2816, align1=0, align2=12:        57.87        58.03
>     length=2880, align1=0, align2=13:        61.97        58.03
>     length=2944, align1=0, align2=14:        71.97        60.79
>     length=3008, align1=0, align2=15:        65.02        60.47
>     length=3072, align1=0, align2=16:        65.39        63.23
>     length=3136, align1=0, align2=17:        72.54        62.92
>     length=3200, align1=0, align2=18:        69.91        65.39
>     length=3264, align1=0, align2=19:        73.38        65.39
>     length=3328, align1=0, align2=20:        73.50        67.81
>     length=3392, align1=0, align2=21:        76.81        67.81
>     length=3456, align1=0, align2=22:        76.88        70.79
>     length=3520, align1=0, align2=23:        80.30        71.07
>     length=3584, align1=0, align2=24:        80.25        72.69
>     length=3648, align1=0, align2=25:        84.58        73.01
>     length=3712, align1=0, align2=26:        85.02        75.13
>     length=3776, align1=0, align2=27:        87.51        75.14
>     length=3840, align1=0, align2=28:        87.20        77.67
>     length=3904, align1=0, align2=29:        89.82        77.99
>     length=3968, align1=0, align2=30:       102.59        80.45
>     length=4032, align1=0, align2=31:        92.38        80.60
>     length=4096, align1=0, align2=32:        85.69        84.69
>
> REP MOVSB is a little bit faster.
>
> > > Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
> > > fast short REP MOVSB (FSRM).
> > >
> > >         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
> > >         rep_movsb_threshold to 2112 on processors with fast short REP
> > >         MOVSB (FSRM).
> > > ---
> > >  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--
> > >  1 file changed, 8 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
> > > index d9944250fc..3f04fb5019 100644
> > > --- a/sysdeps/x86/dl-cacheinfo.h
> > > +++ b/sysdeps/x86/dl-cacheinfo.h
> > > @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> > >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
> > >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
> > >      {
> > > -      rep_movsb_threshold = 2048 * (64 / 16);
> > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> > > +       rep_movsb_threshold = 2112;
> > > +      else
> > > +       rep_movsb_threshold = 2048 * (64 / 16);
> > >  #if HAVE_TUNABLES
> > >        minimum_rep_movsb_threshold = 64 * 8;
> > >  #endif
> > > @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> > >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,
> > >                                     AVX_Fast_Unaligned_Load))
> > >      {
> > > -      rep_movsb_threshold = 2048 * (32 / 16);
> > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> > > +       rep_movsb_threshold = 2112;
> > > +      else
> > > +       rep_movsb_threshold = 2048 * (32 / 16);
> > >  #if HAVE_TUNABLES
> > >        minimum_rep_movsb_threshold = 32 * 8;
> > >  #endif
> > > --
> > > 2.31.1
> > >
>
>
>
> --
> H.J.
H.J. Lu May 1, 2021, 1:26 a.m. UTC | #4 
On Fri, Apr 30, 2021 at 5:17 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> On Fri, Apr 30, 2021 at 4:51 PM H.J. Lu <hjl.tools@gmail.com> wrote:
> >
> > On Fri, Apr 30, 2021 at 3:59 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> > >
> > > On Fri, Apr 30, 2021 at 4:43 PM H.J. Lu via Libc-alpha
> > > <libc-alpha@sourceware.org> wrote:
> > > >
> > > > The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
> > > > that REP MOVSB became faster after 2112 bytes:
> > > >
> > > >                                     Vector + REP MOVSB  REP MOVSB
> > > > length=2112, align1=0, align2=0:        24.20             24.40
> > > > length=2112, align1=1, align2=0:        26.07             23.13
> > > > length=2112, align1=0, align2=1:        27.18             28.13
> > > > length=2112, align1=1, align2=1:        26.23             25.16
> > > > length=2176, align1=0, align2=0:        23.18             22.52
> > > > length=2176, align1=2, align2=0:        25.45             22.52
> > > > length=2176, align1=0, align2=2:        27.14             27.82
> > > > length=2176, align1=2, align2=2:        22.73             25.56
> > > > length=2240, align1=0, align2=0:        24.62             24.25
> > > > length=2240, align1=3, align2=0:        29.77             27.15
> > > > length=2240, align1=0, align2=3:        35.55             29.93
> > > > length=2240, align1=3, align2=3:        34.49             25.15
> > > > length=2304, align1=0, align2=0:        34.75             26.64
> > > > length=2304, align1=4, align2=0:        32.09             22.63
> > > > length=2304, align1=0, align2=4:        28.43             31.24
> > > >
> > >
> > > Do you know what is happening at:
> > > length=2304, align1=0, align2=4:        28.43             31.24
> > >
> > > Seems align2 > align1 gets worse for larger sizes as well.
> > >
> > > I.e from my Icelake (1 run):
> > >                                                               w/o
> > > Patch      w/ Patch
> > > length=3648, align1=0, align2=25:        72.91             83.99
> > > length=3712, align1=0, align2=26:        72.52             84.92
> > > length=3776, align1=0, align2=27:        76.16             84.02
> > > length=3840, align1=0, align2=28:        75.44             90.13
> > > length=3904, align1=0, align2=29:        81.62             84.43
> > > length=3968, align1=0, align2=30:        76.82             93.39
> > > length=4032, align1=0, align2=31:        80.01             89.89
> > > length=4096, align1=0, align2=32:        72.89             97.50
> >
> > On Intel Core i7-1165G7 (Tiger Lake), I got
>
> Where the original numbers icl or tgl? But was just wondering

Here are numbers on Core i7-1065G7 (Ice Lake):

                                    __memcpy_evex_unaligned_erms __memcpy_erms
     length=2048, align1=0, align2=0:        21.51        22.52
     length=2048, align1=0, align2=0:        22.11        21.50
     length=2048, align1=0, align2=0:        21.96        22.10
     length=2048, align1=0, align2=0:        22.58        21.51
     length=2112, align1=0, align2=0:        24.20        24.40
     length=2112, align1=1, align2=0:        26.07        23.13
     length=2112, align1=0, align2=1:        27.18        28.13
     length=2112, align1=1, align2=1:        26.23        25.16
     length=2176, align1=0, align2=0:        23.18        22.52
     length=2176, align1=2, align2=0:        25.45        22.52
     length=2176, align1=0, align2=2:        27.14        35.82
     length=2176, align1=2, align2=2:        22.73        25.56
     length=2240, align1=0, align2=0:        24.62        24.25
     length=2240, align1=3, align2=0:        29.77        27.15
     length=2240, align1=0, align2=3:        35.55        29.93
     length=2240, align1=3, align2=3:        34.49        25.15
     length=2304, align1=0, align2=0:        34.75        26.64
     length=2304, align1=4, align2=0:        32.09        22.63
     length=2304, align1=0, align2=4:        28.43        31.24
     length=2304, align1=4, align2=4:        23.95        25.56
     length=2368, align1=0, align2=0:        25.57        24.34
     length=2368, align1=5, align2=0:        30.57        23.94
     length=2368, align1=0, align2=5:        31.29        31.24
     length=2368, align1=5, align2=5:        25.57        26.69
     length=2432, align1=0, align2=0:        25.16        23.77
     length=2432, align1=6, align2=0:        29.27        23.13
     length=2432, align1=0, align2=6:        30.40        33.77
     length=2432, align1=6, align2=6:        25.16        26.37
     length=2496, align1=0, align2=0:        26.85        24.34
     length=2496, align1=7, align2=0:        32.04        24.75
     length=2496, align1=0, align2=7:        32.70        33.06
     length=2496, align1=7, align2=7:        26.79        25.97
     length=2560, align1=0, align2=0:        61.84        23.53
     length=2560, align1=8, align2=0:        30.74        23.53
     length=2560, align1=0, align2=8:        33.35        35.44
     length=2560, align1=8, align2=8:        27.94        27.52
     length=2624, align1=0, align2=0:        27.60        24.75
     length=2624, align1=9, align2=0:        33.28        25.16
     length=2624, align1=0, align2=9:        34.14        34.48
     length=2624, align1=9, align2=9:        28.00        26.78
     length=2688, align1=0, align2=0:        38.94        24.61
    length=2688, align1=10, align2=0:        31.23        23.94
    length=2688, align1=0, align2=10:        33.89        37.11
   length=2688, align1=10, align2=10:        28.40        27.18
     length=2752, align1=0, align2=0:        36.17        25.56
    length=2752, align1=11, align2=0:        35.88        25.57
    length=2752, align1=0, align2=11:        35.40        36.11
   length=2752, align1=11, align2=11:        28.82        27.94
     length=2816, align1=0, align2=0:        33.78        24.54
    length=2816, align1=12, align2=0:        33.75        25.96
    length=2816, align1=0, align2=12:        34.24        37.73
   length=2816, align1=12, align2=12:        29.22        27.59
     length=2880, align1=0, align2=0:        39.76        25.97
    length=2880, align1=13, align2=0:        41.26        26.66
    length=2880, align1=0, align2=13:        40.72        37.73
   length=2880, align1=13, align2=13:        35.13        27.59
     length=2944, align1=0, align2=0:        32.09        24.75
    length=2944, align1=14, align2=0:        40.27        26.78
    length=2944, align1=0, align2=14:        45.59        39.55
   length=2944, align1=14, align2=14:        40.94        28.00
     length=3008, align1=0, align2=0:        42.17        25.97
    length=3008, align1=15, align2=0:        41.81        27.07
    length=3008, align1=0, align2=15:        42.67        39.35
   length=3008, align1=15, align2=15:        42.18        28.00
     length=3072, align1=0, align2=0:        34.71        25.16
    length=3072, align1=16, align2=0:        43.31        26.37
    length=3072, align1=0, align2=16:        43.41        40.98
   length=3072, align1=16, align2=16:        36.62        27.99
     length=3136, align1=0, align2=0:        45.75        26.78
    length=3136, align1=17, align2=0:        43.29        27.59
    length=3136, align1=0, align2=17:        44.44        41.16
   length=3136, align1=17, align2=17:        34.99        28.59
     length=3200, align1=0, align2=0:        43.12        25.16
    length=3200, align1=18, align2=0:        43.04        27.59
    length=3200, align1=0, align2=18:        45.08        42.59
   length=3200, align1=18, align2=18:        36.00        28.81
     length=3264, align1=0, align2=0:        43.01        27.48
    length=3264, align1=19, align2=0:        47.72        28.61
    length=3264, align1=0, align2=19:        47.00        42.60
   length=3264, align1=19, align2=19:        44.44        28.81
     length=3328, align1=0, align2=0:        35.50        25.75
    length=3328, align1=20, align2=0:        44.84        27.19
    length=3328, align1=0, align2=20:        46.38        44.41
   length=3328, align1=20, align2=20:        33.68        28.81
     length=3392, align1=0, align2=0:        43.82        27.18
    length=3392, align1=21, align2=0:        45.91        28.81
    length=3392, align1=0, align2=21:        47.61        43.06
   length=3392, align1=21, align2=21:        43.82        29.21
     length=3456, align1=0, align2=0:        43.83        26.64
    length=3456, align1=22, align2=0:        44.77        29.49
    length=3456, align1=0, align2=22:        47.57        46.01
   length=3456, align1=22, align2=22:        44.11        29.62
     length=3520, align1=0, align2=0:        45.04        27.99
    length=3520, align1=23, align2=0:        46.59        30.55
    length=3520, align1=0, align2=23:        50.58        44.80
   length=3520, align1=23, align2=23:        43.85        29.21
     length=3584, align1=0, align2=0:        45.04        26.88
    length=3584, align1=24, align2=0:        47.22        29.21
    length=3584, align1=0, align2=24:        50.12        47.47
   length=3584, align1=24, align2=24:        45.04        29.80
     length=3648, align1=0, align2=0:        47.13        28.40
    length=3648, align1=25, align2=0:        48.78        30.94
    length=3648, align1=0, align2=25:        49.25        61.42
   length=3648, align1=25, align2=25:        45.85        30.02
     length=3712, align1=0, align2=0:        47.24        27.19
    length=3712, align1=26, align2=0:        48.41        30.79
    length=3712, align1=0, align2=26:        49.65        51.57
   length=3712, align1=26, align2=26:        48.76        30.33
     length=3776, align1=0, align2=0:        47.35        34.61
    length=3776, align1=27, align2=0:        49.16        31.99
    length=3776, align1=0, align2=27:        52.06        49.38
   length=3776, align1=27, align2=27:        47.36        30.43
     length=3840, align1=0, align2=0:        49.70        28.35
    length=3840, align1=28, align2=0:        50.29        31.81
    length=3840, align1=0, align2=28:        51.15        50.71
   length=3840, align1=28, align2=28:        47.07        31.65
     length=3904, align1=0, align2=0:        48.28        29.21
    length=3904, align1=29, align2=0:        51.96        32.99
    length=3904, align1=0, align2=29:        53.24        50.71
   length=3904, align1=29, align2=29:        47.88        31.64
     length=3968, align1=0, align2=0:        47.89        29.19
    length=3968, align1=30, align2=0:        51.52        32.57
    length=3968, align1=0, align2=30:        54.03        52.33
   length=3968, align1=30, align2=30:        46.62        33.15
     length=4032, align1=0, align2=0:        49.77        29.62
    length=4032, align1=31, align2=0:        53.62        34.27
    length=4032, align1=0, align2=31:        56.44        52.33
   length=4032, align1=31, align2=31:        48.94        33.16
     length=4096, align1=0, align2=0:        49.37        29.21
    length=4096, align1=32, align2=0:        50.46        33.00
    length=4096, align1=0, align2=32:        49.31        55.14
   length=4096, align1=32, align2=32:        49.28        33.05

> if the align2 != 0, align1 = 0 case on Icelake warranted more investigation.
> That being said the change is a clear winner on Icelake as well
> for any other alignment schema.

True, REP MOVSB is more efficient.

> >
> >                                                                 Vector
> >        REP MOVSB
> >       length=2112, align1=0, align2=1:        45.21        43.69
> >      length=2176, align1=0, align2=2:        45.20        45.82
> >      length=2240, align1=0, align2=3:        53.46        51.41
> >      length=2304, align1=0, align2=4:        47.65        48.28
> >      length=2368, align1=0, align2=5:        50.10        48.27
> >      length=2432, align1=0, align2=6:        50.10        50.70
> >      length=2496, align1=0, align2=7:        52.54        50.71
> >      length=2560, align1=0, align2=8:        52.83        53.15
> >      length=2624, align1=0, align2=9:        64.98        53.15
> >     length=2688, align1=0, align2=10:        54.98        55.59
> >     length=2752, align1=0, align2=11:        57.43        55.59
> >     length=2816, align1=0, align2=12:        57.87        58.03
> >     length=2880, align1=0, align2=13:        61.97        58.03
> >     length=2944, align1=0, align2=14:        71.97        60.79
> >     length=3008, align1=0, align2=15:        65.02        60.47
> >     length=3072, align1=0, align2=16:        65.39        63.23
> >     length=3136, align1=0, align2=17:        72.54        62.92
> >     length=3200, align1=0, align2=18:        69.91        65.39
> >     length=3264, align1=0, align2=19:        73.38        65.39
> >     length=3328, align1=0, align2=20:        73.50        67.81
> >     length=3392, align1=0, align2=21:        76.81        67.81
> >     length=3456, align1=0, align2=22:        76.88        70.79
> >     length=3520, align1=0, align2=23:        80.30        71.07
> >     length=3584, align1=0, align2=24:        80.25        72.69
> >     length=3648, align1=0, align2=25:        84.58        73.01
> >     length=3712, align1=0, align2=26:        85.02        75.13
> >     length=3776, align1=0, align2=27:        87.51        75.14
> >     length=3840, align1=0, align2=28:        87.20        77.67
> >     length=3904, align1=0, align2=29:        89.82        77.99
> >     length=3968, align1=0, align2=30:       102.59        80.45
> >     length=4032, align1=0, align2=31:        92.38        80.60
> >     length=4096, align1=0, align2=32:        85.69        84.69
> >
> > REP MOVSB is a little bit faster.
> >
> > > > Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
> > > > fast short REP MOVSB (FSRM).
> > > >
> > > >         * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
> > > >         rep_movsb_threshold to 2112 on processors with fast short REP
> > > >         MOVSB (FSRM).
> > > > ---
> > > >  sysdeps/x86/dl-cacheinfo.h | 10 ++++++++--
> > > >  1 file changed, 8 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
> > > > index d9944250fc..3f04fb5019 100644
> > > > --- a/sysdeps/x86/dl-cacheinfo.h
> > > > +++ b/sysdeps/x86/dl-cacheinfo.h
> > > > @@ -871,7 +871,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> > > >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
> > > >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
> > > >      {
> > > > -      rep_movsb_threshold = 2048 * (64 / 16);
> > > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> > > > +       rep_movsb_threshold = 2112;
> > > > +      else
> > > > +       rep_movsb_threshold = 2048 * (64 / 16);
> > > >  #if HAVE_TUNABLES
> > > >        minimum_rep_movsb_threshold = 64 * 8;
> > > >  #endif
> > > > @@ -879,7 +882,10 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> > > >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,
> > > >                                     AVX_Fast_Unaligned_Load))
> > > >      {
> > > > -      rep_movsb_threshold = 2048 * (32 / 16);
> > > > +      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
> > > > +       rep_movsb_threshold = 2112;
> > > > +      else
> > > > +       rep_movsb_threshold = 2048 * (32 / 16);
> > > >  #if HAVE_TUNABLES
> > > >        minimum_rep_movsb_threshold = 32 * 8;
> > > >  #endif
> > > > --
> > > > 2.31.1
> > > >
> >
> >
> >
> > --
> > H.J.
diff mbox series

Patch

diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
index d9944250fc..3f04fb5019 100644
--- a/sysdeps/x86/dl-cacheinfo.h
+++ b/sysdeps/x86/dl-cacheinfo.h
@@ -871,7 +871,10 @@  dl_init_cacheinfo (struct cpu_features *cpu_features)
   if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
       && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
     {
-      rep_movsb_threshold = 2048 * (64 / 16);
+      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
+	rep_movsb_threshold = 2112;
+      else
+	rep_movsb_threshold = 2048 * (64 / 16);
 #if HAVE_TUNABLES
       minimum_rep_movsb_threshold = 64 * 8;
 #endif
@@ -879,7 +882,10 @@  dl_init_cacheinfo (struct cpu_features *cpu_features)
   else if (CPU_FEATURE_PREFERRED_P (cpu_features,
 				    AVX_Fast_Unaligned_Load))
     {
-      rep_movsb_threshold = 2048 * (32 / 16);
+      if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
+	rep_movsb_threshold = 2112;
+      else
+	rep_movsb_threshold = 2048 * (32 / 16);
 #if HAVE_TUNABLES
       minimum_rep_movsb_threshold = 32 * 8;
 #endif