diff mbox series

[3/3] resolv: Handle transaction ID collisions in parallel queries (bug 26600)

Message ID 87lfhgqx9u.fsf@oldenburg2.str.redhat.com
State Committed
Headers show
Series [1/3] support: Provide a way to reorder responses within the DNS test server | expand

Commit Message

Florian Weimer Sept. 11, 2020, 1:05 p.m. UTC
If the transaction IDs are equal, the old check attributed both
responses to the first query, not recognizing the second response.
This fixes bug 26600.

(Tested on x86_64-linux-gnu.  I verified that the referral path for
RCODE 0 responses is taken.  I also linked tst-resolv-txnid-collision.o
against an ABI-compatible static glibc build without the fix, to verify
that the test reproduces the bug.)

---
 resolv/Makefile                     |   7 +
 resolv/res_send.c                   |  40 ++--
 resolv/tst-resolv-txnid-collision.c | 329 ++++++++++++++++++++++++++++
 3 files changed, 356 insertions(+), 20 deletions(-)
 create mode 100644 resolv/tst-resolv-txnid-collision.c

Comments

Florian Weimer Oct. 13, 2020, 10:14 a.m. UTC | #1
* Florian Weimer via Libc-alpha:

> If the transaction IDs are equal, the old check attributed both
> responses to the first query, not recognizing the second response.
> This fixes bug 26600.
>
> (Tested on x86_64-linux-gnu.  I verified that the referral path for
> RCODE 0 responses is taken.  I also linked tst-resolv-txnid-collision.o
> against an ABI-compatible static glibc build without the fix, to verify
> that the test reproduces the bug.)

Ping.  These patches need review:

  <https://sourceware.org/pipermail/libc-alpha/2020-September/117545.html>
  <https://sourceware.org/pipermail/libc-alpha/2020-September/117546.html>
  <https://sourceware.org/pipermail/libc-alpha/2020-September/117547.html>

Apparently, users encounter this bug quite frequently on s390x with
certain applications.

Thanks,
Florian
Siddhesh Poyarekar Oct. 13, 2020, 6:18 p.m. UTC | #2
On 9/11/20 6:35 PM, Florian Weimer via Libc-alpha wrote:
> If the transaction IDs are equal, the old check attributed both
> responses to the first query, not recognizing the second response.
> This fixes bug 26600.
> 
> (Tested on x86_64-linux-gnu.  I verified that the referral path for
> RCODE 0 responses is taken.  I also linked tst-resolv-txnid-collision.o
> against an ABI-compatible static glibc build without the fix, to verify
> that the test reproduces the bug.)

This is fine.  Transaction IDs ending up being equal is not unsafe in
itself since it happens at random and does not necessarily improve
chances of crafting a malicious response to the second query.

> ---
>  resolv/Makefile                     |   7 +
>  resolv/res_send.c                   |  40 ++--
>  resolv/tst-resolv-txnid-collision.c | 329 ++++++++++++++++++++++++++++
>  3 files changed, 356 insertions(+), 20 deletions(-)
>  create mode 100644 resolv/tst-resolv-txnid-collision.c
> 
> diff --git a/resolv/Makefile b/resolv/Makefile
> index b61c0c3e0c..dbd8f8bf4f 100644
> --- a/resolv/Makefile
> +++ b/resolv/Makefile
> @@ -61,6 +61,11 @@ tests += \
>    tst-resolv-search \
>    tst-resolv-trailing \
>  
> +# This test calls __res_context_send directly, which is not exported
> +# from libresolv.
> +tests-internal += tst-resolv-txnid-collision
> +tests-static += tst-resolv-txnid-collision
> +
>  # These tests need libdl.
>  ifeq (yes,$(build-shared))
>  tests += \
> @@ -191,6 +196,8 @@ $(objpfx)tst-resolv-search: $(objpfx)libresolv.so $(shared-thread-library)
>  $(objpfx)tst-resolv-trailing: $(objpfx)libresolv.so $(shared-thread-library)
>  $(objpfx)tst-resolv-threads: \
>    $(libdl) $(objpfx)libresolv.so $(shared-thread-library)
> +$(objpfx)tst-resolv-txnid-collision: $(objpfx)libresolv.a \
> +  $(static-thread-library)
>  $(objpfx)tst-resolv-canonname: \
>    $(libdl) $(objpfx)libresolv.so $(shared-thread-library)
>  $(objpfx)tst-resolv-trustad: $(objpfx)libresolv.so $(shared-thread-library)
> diff --git a/resolv/res_send.c b/resolv/res_send.c
> index 7e5fec6646..70e5066031 100644
> --- a/resolv/res_send.c
> +++ b/resolv/res_send.c
> @@ -1342,15 +1342,6 @@ send_dg(res_state statp,
>  			*terrno = EMSGSIZE;
>  			return close_and_return_error (statp, resplen2);
>  		}
> -		if ((recvresp1 || hp->id != anhp->id)
> -		    && (recvresp2 || hp2->id != anhp->id)) {
> -			/*
> -			 * response from old query, ignore it.
> -			 * XXX - potential security hazard could
> -			 *	 be detected here.
> -			 */
> -			goto wait;
> -		}
>  
>  		/* Paranoia check.  Due to the connected UDP socket,
>  		   the kernel has already filtered invalid addresses
> @@ -1360,15 +1351,24 @@ send_dg(res_state statp,
>  
>  		/* Check for the correct header layout and a matching
>  		   question.  */
> -		if ((recvresp1 || !res_queriesmatch(buf, buf + buflen,
> -						       *thisansp,
> -						       *thisansp
> -						       + *thisanssizp))
> -		    && (recvresp2 || !res_queriesmatch(buf2, buf2 + buflen2,
> -						       *thisansp,
> -						       *thisansp
> -						       + *thisanssizp)))
> -		  goto wait;
> +		int matching_query = 0; /* Default to no matching query.  */
> +		if (!recvresp1
> +		    && anhp->id == hp->id
> +		    && res_queriesmatch (buf, buf + buflen,
> +					 *thisansp, *thisansp + *thisanssizp))
> +		  matching_query = 1;
> +		if (!recvresp2
> +		    && anhp->id == hp2->id
> +		    && res_queriesmatch (buf2, buf2 + buflen2,
> +					 *thisansp, *thisansp + *thisanssizp))
> +		  matching_query = 2;
> +		if (matching_query == 0)
> +		  /* Spurious UDP packet.  Drop it and continue
> +		     waiting.  */
> +		  {
> +		    need_recompute = 1;
> +		    goto wait;
> +		  }

Checking against the first and second query independently.  OK.

>  
>  		if (anhp->rcode == SERVFAIL ||
>  		    anhp->rcode == NOTIMP ||
> @@ -1383,7 +1383,7 @@ send_dg(res_state statp,
>  			    /* No data from the first reply.  */
>  			    resplen = 0;
>  			    /* We are waiting for a possible second reply.  */
> -			    if (hp->id == anhp->id)
> +			    if (matching_query == 1)
>  			      recvresp1 = 1;
>  			    else
>  			      recvresp2 = 1;
> @@ -1414,7 +1414,7 @@ send_dg(res_state statp,
>  			return (1);
>  		}
>  		/* Mark which reply we received.  */
> -		if (recvresp1 == 0 && hp->id == anhp->id)
> +		if (matching_query == 1)
>  			recvresp1 = 1;
>  		else
>  			recvresp2 = 1;

Using MATCHING_QUERY to check which query was responded to.  OK.

> diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c
> new file mode 100644
> index 0000000000..611d37362f
> --- /dev/null
> +++ b/resolv/tst-resolv-txnid-collision.c
> @@ -0,0 +1,329 @@
> +/* Test parallel queries with transaction ID collisions.

Test is also OK.

Looks good to me.

Thanks,
Siddhesh
Stefan Liebler Oct. 14, 2020, 4:32 p.m. UTC | #3
On 10/13/20 8:18 PM, Siddhesh Poyarekar wrote:
> On 9/11/20 6:35 PM, Florian Weimer via Libc-alpha wrote:
...
> 
> Test is also OK.
> 
> Looks good to me.
> 
> Thanks,
> Siddhesh
> 
Hi Florian,

I've recognized a test-fail for the new test
resolv/tst-resolv-txnid-collision (of course on s390x):
tst-resolv-txnid-collision.c:166: numeric comparison failure
   left: 0 (0x0); from: ctx->server_index
  right: 1 (0x1); from: previous_server_index
error: tst-resolv-txnid-collision.c:167: not true: previous_qtype != qtype
error: 2 test failures
(See attached tst-resolv-txnid-collision_20201014_174129_4983.log)

So far, I've just run the test resolv/tst-resolv-txnid-collision with
--direct --verbose in a loop and collected the output (see attachements).


It also happens that the test segfaults.
Here is one backtrace (also see attached
tst-resolv-txnid-collision_20201014_174533_4318.log):
(gdb) bt full
#0  0x0000000001001c28 in resolv_response_context_free (ctx=0x0) at
resolv_response_context_free.c:25
No locals.
#1  0x00000000010017cc in response (ctx=0x3fffcf7cab8, b=0x3ffec000b60,
qname=0x3fffcf7cb72 "reorder-1-0.rcode-2.example.net",
    qclass=<optimized out>, qtype=<optimized out>) at
tst-resolv-txnid-collision.c:204
        parsed = {rcode = 2, reorder = {true, false}}
#2  0x000000000100319a in server_thread_udp_process_one
(obj=obj@entry=0x10eb3d0, server_index=server_index@entry=1) at
resolv_test.c:677
        query =
"\331\032\001\000\000\001\000\000\000\000\000\000\vreorder-1-0\arcode-2\aexample\003net\000\000\001\000\001",
'\000' <repeats 462 times>
        peer = {ss_family = 2, __ss_padding =
"\240\251\177\000\000\001", '\000' <repeats 111 times>, __ss_align = 0}
        peerlen = 16
        length = <optimized out>
        qinfo = {qname = "reorder-1-0.rcode-2.example.net", '\000'
<repeats 993 times>, qclass = 1, qtype = 1, edns = {active = false,
            extended_rcode = 0 '\000', version = 0 '\000', flags = 0,
payload_size = 0}}
        ctx = {test = 0x10eb3d0, client_address = 0x3fffcf7caf0,
client_address_length = 16,
          query_buffer = 0x3fffcf7cf80 "\331\032\001", query_length =
49, server_index = 1, tcp = false, edns = {active = false,
            extended_rcode = 0 '\000', version = 0 '\000', flags = 0,
payload_size = 0}}
        b = 0x3ffec000b60
#3  0x00000000010032da in server_thread_udp (obj=0x10eb3d0,
server_index=<optimized out>) at resolv_test.c:734
No locals.
#4  0x0000000001001c74 in thread_callback_wrapper (arg=0x10eb800) at
resolv_test.c:606
        closure = 0x10eb800
#5  0x000000000100c93c in start_thread (arg=0x3fffcf7d870) at
pthread_create.c:463
        ret = <optimized out>
        start = <optimized out>
        pd = 0x3fffcf7d870
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {{__gregs =
{4397995645040, 4393751543808, 4398046505551, 4397995644112,
                    4398046505550, 4398046505552, 4397995645040,
4398046507848, 56514505740802731, 56510891581063335}, __fpregs = {
                    8392704, 0, 4397987254272, 0, 4397995645232, 0,
4398046507237, 2929168734320}}}, mask_was_saved = 0}}, priv = {
            pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup =
0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x000000000104df90 in thread_start () at
../sysdeps/unix/sysv/linux/s390/s390-64/clone.S:65
No locals.


=> ctx (or previous_query in #1) is NULL. It also happened once at a
different location. In both cases, previous_query inside response
function was NULL.
If I remember correctly the other location was at line 187 while
accessing previous_query:
        struct resolv_response_builder *btmp
          = resolv_response_builder_allocate (previous_query->query_buffer,
previous_query->query_length);

I hope, this information helps. Otherwise, please let me know which
further information is required.

Bye,
Stefan
Florian Weimer Oct. 14, 2020, 5:57 p.m. UTC | #4
* Stefan Liebler via Libc-alpha:

> On 10/13/20 8:18 PM, Siddhesh Poyarekar wrote:
>> On 9/11/20 6:35 PM, Florian Weimer via Libc-alpha wrote:
> ...
>> 
>> Test is also OK.
>> 
>> Looks good to me.
>> 
>> Thanks,
>> Siddhesh
>> 
> Hi Florian,
>
> I've recognized a test-fail for the new test
> resolv/tst-resolv-txnid-collision (of course on s390x):
> tst-resolv-txnid-collision.c:166: numeric comparison failure
>    left: 0 (0x0); from: ctx->server_index
>   right: 1 (0x1); from: previous_server_index
> error: tst-resolv-txnid-collision.c:167: not true: previous_qtype != qtype
> error: 2 test failures
> (See attached tst-resolv-txnid-collision_20201014_174129_4983.log)
>
> So far, I've just run the test resolv/tst-resolv-txnid-collision with
> --direct --verbose in a loop and collected the output (see attachements).

Well, this is embarrassing.  I will have to make the test more tolerant
of packet reordering.  Thanks for reporting this, I can (somewhat
rarely) reproduce this, on an s390x system (under high load, I think).

> It also happens that the test segfaults.
> Here is one backtrace (also see attached
> tst-resolv-txnid-collision_20201014_174533_4318.log):
> (gdb) bt full
> #0  0x0000000001001c28 in resolv_response_context_free (ctx=0x0) at
> resolv_response_context_free.c:25
> No locals.
> #1  0x00000000010017cc in response (ctx=0x3fffcf7cab8, b=0x3ffec000b60,
> qname=0x3fffcf7cb72 "reorder-1-0.rcode-2.example.net",
>     qclass=<optimized out>, qtype=<optimized out>) at
> tst-resolv-txnid-collision.c:204
>         parsed = {rcode = 2, reorder = {true, false}}

This should fix the crash at least, it's a concurrency bug in the test:

diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c
index 611d3736..db9a52b6 100644
--- a/resolv/tst-resolv-txnid-collision.c
+++ b/resolv/tst-resolv-txnid-collision.c
@@ -309,6 +309,7 @@ do_test (void)
     ((struct resolv_redirect_config)
      {
        .response_callback = response,
+       .single_thread_udp = true,
      });
 
   for (int rcode = 0; rcode <= 5; ++rcode)

I doubt it will fix the other test failure, but I'm going to run the
test over night with this test.

Thanks,
Florian
Florian Weimer Oct. 15, 2020, 10:35 a.m. UTC | #5
* Florian Weimer:

> This should fix the crash at least, it's a concurrency bug in the test:
>
> diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c
> index 611d3736..db9a52b6 100644
> --- a/resolv/tst-resolv-txnid-collision.c
> +++ b/resolv/tst-resolv-txnid-collision.c
> @@ -309,6 +309,7 @@ do_test (void)
>      ((struct resolv_redirect_config)
>       {
>         .response_callback = response,
> +       .single_thread_udp = true,
>       });
>  
>    for (int rcode = 0; rcode <= 5; ++rcode)
>
> I doubt it will fix the other test failure, but I'm going to run the
> test over night with this test.

My tests over night looked good, and the above change is clearly needed,
so I have pushed that as commit b8b53b338f6da91e86d115a39da860cefac736ad.

Thanks,
Florian
Stefan Liebler Oct. 15, 2020, 3:02 p.m. UTC | #6
On 10/14/20 7:57 PM, Florian Weimer wrote:
> * Stefan Liebler via Libc-alpha:
> 
>> On 10/13/20 8:18 PM, Siddhesh Poyarekar wrote:
>>> On 9/11/20 6:35 PM, Florian Weimer via Libc-alpha wrote:
>> ...
>>>
>>> Test is also OK.
>>>
>>> Looks good to me.
>>>
>>> Thanks,
>>> Siddhesh
>>>
>> Hi Florian,
>>
>> I've recognized a test-fail for the new test
>> resolv/tst-resolv-txnid-collision (of course on s390x):
>> tst-resolv-txnid-collision.c:166: numeric comparison failure
>>    left: 0 (0x0); from: ctx->server_index
>>   right: 1 (0x1); from: previous_server_index
>> error: tst-resolv-txnid-collision.c:167: not true: previous_qtype != qtype
>> error: 2 test failures
>> (See attached tst-resolv-txnid-collision_20201014_174129_4983.log)
>>
>> So far, I've just run the test resolv/tst-resolv-txnid-collision with
>> --direct --verbose in a loop and collected the output (see attachements).
> 
> Well, this is embarrassing.  I will have to make the test more tolerant
> of packet reordering.  Thanks for reporting this, I can (somewhat
> rarely) reproduce this, on an s390x system (under high load, I think).
> 
>> It also happens that the test segfaults.
>> Here is one backtrace (also see attached
>> tst-resolv-txnid-collision_20201014_174533_4318.log):
>> (gdb) bt full
>> #0  0x0000000001001c28 in resolv_response_context_free (ctx=0x0) at
>> resolv_response_context_free.c:25
>> No locals.
>> #1  0x00000000010017cc in response (ctx=0x3fffcf7cab8, b=0x3ffec000b60,
>> qname=0x3fffcf7cb72 "reorder-1-0.rcode-2.example.net",
>>     qclass=<optimized out>, qtype=<optimized out>) at
>> tst-resolv-txnid-collision.c:204
>>         parsed = {rcode = 2, reorder = {true, false}}
> 
> This should fix the crash at least, it's a concurrency bug in the test:
> 
> diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c
> index 611d3736..db9a52b6 100644
> --- a/resolv/tst-resolv-txnid-collision.c
> +++ b/resolv/tst-resolv-txnid-collision.c
> @@ -309,6 +309,7 @@ do_test (void)
>      ((struct resolv_redirect_config)
>       {
>         .response_callback = response,
> +       .single_thread_udp = true,
>       });
>  
>    for (int rcode = 0; rcode <= 5; ++rcode)
> 
> I doubt it will fix the other test failure, but I'm going to run the
> test over night with this test.
> 
> Thanks,
> Florian
> 

As information, my system was used by multiple users and it uses shared
CPUs between other LPARs. But it was definitely not loaded 100% for the
whole time.

I've also run this test (without single_thread_udp=true) in a loop on an
intel kvm-guest and also recognized test-failures and the segfaults.

With single_thread_udp=true, I also don't see segfaults and so far no
other test-failures (both on s390x and the intel kvm-guest). I'll let
you know if I observe ones in the future.

Thanks,
Stefan
diff mbox series

Patch

diff --git a/resolv/Makefile b/resolv/Makefile
index b61c0c3e0c..dbd8f8bf4f 100644
--- a/resolv/Makefile
+++ b/resolv/Makefile
@@ -61,6 +61,11 @@  tests += \
   tst-resolv-search \
   tst-resolv-trailing \
 
+# This test calls __res_context_send directly, which is not exported
+# from libresolv.
+tests-internal += tst-resolv-txnid-collision
+tests-static += tst-resolv-txnid-collision
+
 # These tests need libdl.
 ifeq (yes,$(build-shared))
 tests += \
@@ -191,6 +196,8 @@  $(objpfx)tst-resolv-search: $(objpfx)libresolv.so $(shared-thread-library)
 $(objpfx)tst-resolv-trailing: $(objpfx)libresolv.so $(shared-thread-library)
 $(objpfx)tst-resolv-threads: \
   $(libdl) $(objpfx)libresolv.so $(shared-thread-library)
+$(objpfx)tst-resolv-txnid-collision: $(objpfx)libresolv.a \
+  $(static-thread-library)
 $(objpfx)tst-resolv-canonname: \
   $(libdl) $(objpfx)libresolv.so $(shared-thread-library)
 $(objpfx)tst-resolv-trustad: $(objpfx)libresolv.so $(shared-thread-library)
diff --git a/resolv/res_send.c b/resolv/res_send.c
index 7e5fec6646..70e5066031 100644
--- a/resolv/res_send.c
+++ b/resolv/res_send.c
@@ -1342,15 +1342,6 @@  send_dg(res_state statp,
 			*terrno = EMSGSIZE;
 			return close_and_return_error (statp, resplen2);
 		}
-		if ((recvresp1 || hp->id != anhp->id)
-		    && (recvresp2 || hp2->id != anhp->id)) {
-			/*
-			 * response from old query, ignore it.
-			 * XXX - potential security hazard could
-			 *	 be detected here.
-			 */
-			goto wait;
-		}
 
 		/* Paranoia check.  Due to the connected UDP socket,
 		   the kernel has already filtered invalid addresses
@@ -1360,15 +1351,24 @@  send_dg(res_state statp,
 
 		/* Check for the correct header layout and a matching
 		   question.  */
-		if ((recvresp1 || !res_queriesmatch(buf, buf + buflen,
-						       *thisansp,
-						       *thisansp
-						       + *thisanssizp))
-		    && (recvresp2 || !res_queriesmatch(buf2, buf2 + buflen2,
-						       *thisansp,
-						       *thisansp
-						       + *thisanssizp)))
-		  goto wait;
+		int matching_query = 0; /* Default to no matching query.  */
+		if (!recvresp1
+		    && anhp->id == hp->id
+		    && res_queriesmatch (buf, buf + buflen,
+					 *thisansp, *thisansp + *thisanssizp))
+		  matching_query = 1;
+		if (!recvresp2
+		    && anhp->id == hp2->id
+		    && res_queriesmatch (buf2, buf2 + buflen2,
+					 *thisansp, *thisansp + *thisanssizp))
+		  matching_query = 2;
+		if (matching_query == 0)
+		  /* Spurious UDP packet.  Drop it and continue
+		     waiting.  */
+		  {
+		    need_recompute = 1;
+		    goto wait;
+		  }
 
 		if (anhp->rcode == SERVFAIL ||
 		    anhp->rcode == NOTIMP ||
@@ -1383,7 +1383,7 @@  send_dg(res_state statp,
 			    /* No data from the first reply.  */
 			    resplen = 0;
 			    /* We are waiting for a possible second reply.  */
-			    if (hp->id == anhp->id)
+			    if (matching_query == 1)
 			      recvresp1 = 1;
 			    else
 			      recvresp2 = 1;
@@ -1414,7 +1414,7 @@  send_dg(res_state statp,
 			return (1);
 		}
 		/* Mark which reply we received.  */
-		if (recvresp1 == 0 && hp->id == anhp->id)
+		if (matching_query == 1)
 			recvresp1 = 1;
 		else
 			recvresp2 = 1;
diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c
new file mode 100644
index 0000000000..611d37362f
--- /dev/null
+++ b/resolv/tst-resolv-txnid-collision.c
@@ -0,0 +1,329 @@ 
+/* Test parallel queries with transaction ID collisions.
+   Copyright (C) 2020 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <arpa/nameser.h>
+#include <array_length.h>
+#include <resolv-internal.h>
+#include <resolv_context.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+#include <support/check.h>
+#include <support/check_nss.h>
+#include <support/resolv_test.h>
+#include <support/support.h>
+#include <support/test-driver.h>
+
+/* Result of parsing a DNS question name.
+
+   A question name has the form reorder-N-M-rcode-C.example.net, where
+   N and M are either 0 and 1, corresponding to the reorder member,
+   and C is a number that will be stored in the rcode field.
+
+   Also see parse_qname below.  */
+struct parsed_qname
+{
+  /* The DNS response code requested from the first server.  The
+     second server always responds with RCODE zero.  */
+  int rcode;
+
+  /* Indicates whether to perform reordering in the responses from the
+     respective server.  */
+  bool reorder[2];
+};
+
+/* Fills *PARSED based on QNAME.  */
+static void
+parse_qname (struct parsed_qname *parsed, const char *qname)
+{
+  int reorder0;
+  int reorder1;
+  int rcode;
+  char *suffix;
+  if (sscanf (qname, "reorder-%d-%d.rcode-%d.%ms",
+              &reorder0, &reorder1, &rcode, &suffix) == 4)
+    {
+      if (reorder0 != 0)
+        TEST_COMPARE (reorder0, 1);
+      if (reorder1 != 0)
+        TEST_COMPARE (reorder1, 1);
+      TEST_VERIFY (rcode >= 0 && rcode <= 15);
+      TEST_COMPARE_STRING (suffix, "example.net");
+      free (suffix);
+
+      parsed->rcode = rcode;
+      parsed->reorder[0] = reorder0;
+      parsed->reorder[1] = reorder1;
+    }
+  else
+    FAIL_EXIT1 ("unexpected query: %s", qname);
+}
+
+/* Used to construct a response. The first server responds with an
+   error, the second server succeeds.  */
+static void
+build_response (const struct resolv_response_context *ctx,
+                struct resolv_response_builder *b,
+                const char *qname, uint16_t qclass, uint16_t qtype)
+{
+  struct parsed_qname parsed;
+  parse_qname (&parsed, qname);
+
+  switch (ctx->server_index)
+    {
+    case 0:
+      {
+        struct resolv_response_flags flags = { 0 };
+        if (parsed.rcode == 0)
+          /* Simulate a delegation in case a NODATA (RCODE zero)
+             response is requested.  */
+          flags.clear_ra = true;
+        else
+          flags.rcode = parsed.rcode;
+
+        resolv_response_init (b, flags);
+        resolv_response_add_question (b, qname, qclass, qtype);
+      }
+      break;
+
+    case 1:
+      {
+        struct resolv_response_flags flags = { 0, };
+        resolv_response_init (b, flags);
+        resolv_response_add_question (b, qname, qclass, qtype);
+
+        resolv_response_section (b, ns_s_an);
+        resolv_response_open_record (b, qname, qclass, qtype, 0);
+        if (qtype == T_A)
+          {
+            char ipv4[4] = { 192, 0, 2, 1 };
+            resolv_response_add_data (b, &ipv4, sizeof (ipv4));
+          }
+        else
+          {
+            char ipv6[16]
+              = { 0x20, 0x01, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+            resolv_response_add_data (b, &ipv6, sizeof (ipv6));
+          }
+        resolv_response_close_record (b);
+      }
+      break;
+    }
+}
+
+/* Used to reorder responses.  */
+struct resolv_response_context *previous_query;
+
+/* Used to keep track of the queries received.  */
+static int previous_server_index = -1;
+static uint16_t previous_qtype;
+
+/* For each server, buffer the first query and then send both answers
+   to the second query, reordered if requested.  */
+static void
+response (const struct resolv_response_context *ctx,
+          struct resolv_response_builder *b,
+          const char *qname, uint16_t qclass, uint16_t qtype)
+{
+  TEST_VERIFY (qtype == T_A || qtype == T_AAAA);
+  if (ctx->server_index != 0)
+    TEST_COMPARE (ctx->server_index, 1);
+
+  struct parsed_qname parsed;
+  parse_qname (&parsed, qname);
+
+  if (previous_query == NULL)
+    {
+      /* No buffered query.  Record this query and do not send a
+         response.  */
+      TEST_COMPARE (previous_qtype, 0);
+      previous_query = resolv_response_context_duplicate (ctx);
+      previous_qtype = qtype;
+      resolv_response_drop (b);
+      previous_server_index = ctx->server_index;
+
+      if (test_verbose)
+        printf ("info: buffering first query for: %s\n", qname);
+    }
+  else
+    {
+      TEST_VERIFY (previous_query != 0);
+      TEST_COMPARE (ctx->server_index, previous_server_index);
+      TEST_VERIFY (previous_qtype != qtype); /* Not a duplicate.  */
+
+      /* If reordering, send a response for this query explicitly, and
+         then skip the implicit send.  */
+      if (parsed.reorder[ctx->server_index])
+        {
+          if (test_verbose)
+            printf ("info: sending reordered second response for: %s\n",
+                    qname);
+          build_response (ctx, b, qname, qclass, qtype);
+          resolv_response_send_udp (ctx, b);
+          resolv_response_drop (b);
+        }
+
+      /* Build a response for the previous query and send it, thus
+         reordering the two responses.  */
+      {
+        if (test_verbose)
+          printf ("info: sending first response for: %s\n", qname);
+        struct resolv_response_builder *btmp
+          = resolv_response_builder_allocate (previous_query->query_buffer,
+                                              previous_query->query_length);
+        build_response (ctx, btmp, qname, qclass, previous_qtype);
+        resolv_response_send_udp (ctx, btmp);
+        resolv_response_builder_free (btmp);
+      }
+
+      /* If not reordering, send the reply as usual.  */
+      if (!parsed.reorder[ctx->server_index])
+        {
+          if (test_verbose)
+            printf ("info: sending non-reordered second response for: %s\n",
+                    qname);
+          build_response (ctx, b, qname, qclass, qtype);
+        }
+
+      /* Unbuffer the response and prepare for the next query.  */
+      resolv_response_context_free (previous_query);
+      previous_query = NULL;
+      previous_qtype = 0;
+      previous_server_index = -1;
+    }
+}
+
+/* Runs a query for QNAME and checks for the expected reply.  See
+   struct parsed_qname for the expected format for QNAME.  */
+static void
+test_qname (const char *qname, int rcode)
+{
+  struct resolv_context *ctx = __resolv_context_get ();
+  TEST_VERIFY_EXIT (ctx != NULL);
+
+  unsigned char q1[512];
+  int q1len = res_mkquery (QUERY, qname, C_IN, T_A, NULL, 0, NULL,
+                           q1, sizeof (q1));
+  TEST_VERIFY_EXIT (q1len > 12);
+
+  unsigned char q2[512];
+  int q2len = res_mkquery (QUERY, qname, C_IN, T_AAAA, NULL, 0, NULL,
+                           q2, sizeof (q2));
+  TEST_VERIFY_EXIT (q2len > 12);
+
+  /* Produce a transaction ID collision.  */
+  memcpy (q2, q1, 2);
+
+  unsigned char ans1[512];
+  unsigned char *ans1p = ans1;
+  unsigned char *ans2p = NULL;
+  int nans2p = 0;
+  int resplen2 = 0;
+  int ans2p_malloced = 0;
+
+  /* Perform a parallel A/AAAA query.  */
+  int resplen1 = __res_context_send (ctx, q1, q1len, q2, q2len,
+                                     ans1, sizeof (ans1), &ans1p,
+                                     &ans2p, &nans2p,
+                                     &resplen2, &ans2p_malloced);
+
+  TEST_VERIFY (resplen1 > 12);
+  TEST_VERIFY (resplen2 > 12);
+  if (resplen1 <= 12 || resplen2 <= 12)
+    return;
+
+  if (rcode == 1 || rcode == 3)
+    {
+      /* Format Error and Name Error responses does not trigger
+         switching to the next server.  */
+      TEST_COMPARE (ans1p[3] & 0x0f, rcode);
+      TEST_COMPARE (ans2p[3] & 0x0f, rcode);
+      return;
+    }
+
+  /* The response should be successful.  */
+  TEST_COMPARE (ans1p[3] & 0x0f, 0);
+  TEST_COMPARE (ans2p[3] & 0x0f, 0);
+
+  /* Due to bug 19691, the answer may not be in the slot matching the
+     query.  Assume that the AAAA response is the longer one.  */
+  unsigned char *a_answer;
+  int a_answer_length;
+  unsigned char *aaaa_answer;
+  int aaaa_answer_length;
+  if (resplen2 > resplen1)
+    {
+      a_answer = ans1p;
+      a_answer_length = resplen1;
+      aaaa_answer = ans2p;
+      aaaa_answer_length = resplen2;
+    }
+  else
+    {
+      a_answer = ans2p;
+      a_answer_length = resplen2;
+      aaaa_answer = ans1p;
+      aaaa_answer_length = resplen1;
+    }
+
+  {
+    char *expected = xasprintf ("name: %s\n"
+                                "address: 192.0.2.1\n",
+                                qname);
+    check_dns_packet (qname, a_answer, a_answer_length, expected);
+    free (expected);
+  }
+  {
+    char *expected = xasprintf ("name: %s\n"
+                                "address: 2001:db8::1\n",
+                                qname);
+    check_dns_packet (qname, aaaa_answer, aaaa_answer_length, expected);
+    free (expected);
+  }
+
+  if (ans2p_malloced)
+    free (ans2p);
+
+  __resolv_context_put (ctx);
+}
+
+static int
+do_test (void)
+{
+  struct resolv_test *aux = resolv_test_start
+    ((struct resolv_redirect_config)
+     {
+       .response_callback = response,
+     });
+
+  for (int rcode = 0; rcode <= 5; ++rcode)
+    for (int do_reorder_0 = 0; do_reorder_0 < 2; ++do_reorder_0)
+      for (int do_reorder_1 = 0; do_reorder_1 < 2; ++do_reorder_1)
+        {
+          char *qname = xasprintf ("reorder-%d-%d.rcode-%d.example.net",
+                                   do_reorder_0, do_reorder_1, rcode);
+          test_qname (qname, rcode);
+          free (qname);
+        }
+
+  resolv_test_end (aux);
+
+  return 0;
+}
+
+#include <support/test-driver.c>