diff --git a/bfd/dwarf2.c b/bfd/dwarf2.c
index 151ed0237ca..e93a331c5b4 100644
--- a/bfd/dwarf2.c
+++ b/bfd/dwarf2.c
@@ -292,12 +292,6 @@ struct dwarf2_debug
   /* Per-file stuff.  */
   struct dwarf2_debug_file f, alt;
 
-  /* Pointer to the original bfd for which debug was loaded.  This is what
-     we use to compare and so check that the cached debug data is still
-     valid - it saves having to possibly dereference the gnu_debuglink each
-     time.  */
-  bfd *orig_bfd;
-
   /* If the most recent call to bfd_find_nearest_line was given an
      address in an inlined function, preserve a pointer into the
      calling chain for subsequent calls to bfd_find_inliner_info to
@@ -315,6 +309,9 @@ struct dwarf2_debug
   /* Array of sections with adjusted VMA.  */
   struct adjusted_section *adjusted_sections;
 
+  /* Used to validate the cached debug data.  */
+  unsigned int orig_bfd_id;
+
   /* Number of times find_line is called.  This is used in
      the heuristic for enabling the info hash tables.  */
   int info_hash_count;
@@ -5401,7 +5398,7 @@ _bfd_dwarf2_slurp_debug_info (bfd *abfd, bfd *debug_bfd,
 
   if (stash != NULL)
     {
-      if (stash->orig_bfd == abfd
+      if (stash->orig_bfd_id == abfd->id
 	  && section_vma_same (abfd, stash))
 	{
 	  /* Check that we did previously find some debug information
@@ -5425,7 +5422,7 @@ _bfd_dwarf2_slurp_debug_info (bfd *abfd, bfd *debug_bfd,
 	return false;
       *pinfo = stash;
     }
-  stash->orig_bfd = abfd;
+  stash->orig_bfd_id = abfd->id;
   stash->debug_sections = debug_sections;
   stash->f.syms = symbols;
   if (!save_section_vma (abfd, stash))