diff --git a/gas/config/rx-parse.y b/gas/config/rx-parse.y
index a8bead28d21..fa632f90d00 100644
--- a/gas/config/rx-parse.y
+++ b/gas/config/rx-parse.y
@@ -1934,32 +1934,21 @@ displacement (expressionS exp, int msize)
   valueT val;
   int vshift = 0;
 
-  if (exp.X_op == O_symbol
-      && exp.X_md)
+  if ((exp.X_op == O_symbol && exp.X_md == BFD_RELOC_GPREL16) ||
+      (exp.X_op == O_subtract))
     {
-      switch (exp.X_md)
+      switch (msize)
 	{
-	case BFD_RELOC_GPREL16:
-	  switch (msize)
-	    {
-	    case BSIZE:
-	      exp.X_md = BFD_RELOC_RX_GPRELB;
-	      break;
-	    case WSIZE:
-	      exp.X_md = BFD_RELOC_RX_GPRELW;
-	      break;
-	    case LSIZE:
-	      exp.X_md = BFD_RELOC_RX_GPRELL;
-	      break;
-	    }
-	  O2 (exp);
-	  return 2;
+	case BSIZE:
+	  exp.X_md = BFD_RELOC_RX_GPRELB;
+	  break;
+	case WSIZE:
+	  exp.X_md = BFD_RELOC_RX_GPRELW;
+	  break;
+	case LSIZE:
+	  exp.X_md = BFD_RELOC_RX_GPRELL;
+	  break;
 	}
-    }
-
-  if (exp.X_op == O_subtract)
-    {
-      exp.X_md = BFD_RELOC_RX_DIFF;
       O2 (exp);
       return 2;
     }
diff --git a/gas/testsuite/gas/rx/mov.d b/gas/testsuite/gas/rx/mov.d
index 276b6d9d898..4c5d2389746 100644
--- a/gas/testsuite/gas/rx/mov.d
+++ b/gas/testsuite/gas/rx/mov.d
@@ -528,3 +528,6 @@ Disassembly of section .*:
  [0-9a-f]+:	fd 2e 0f                      	mov.l	\[-r0\], r15
  [0-9a-f]+:	fd 2e f0                      	mov.l	\[-r15\], r0
  [0-9a-f]+:	fd 2e ff                      	mov.l	\[-r15\], r15
+ [0-9a-f]+:	ce 00 04 00                   	mov.b	4\[r0\], r0
+ [0-9a-f]+:	de 00 02 00                   	mov.w	4\[r0\], r0
+ [0-9a-f]+:	ee 00 01 00                   	mov.l	4\[r0\], r0
diff --git a/gas/testsuite/gas/rx/mov.sm b/gas/testsuite/gas/rx/mov.sm
index 8bebbf5908a..03bec4a7bc1 100644
--- a/gas/testsuite/gas/rx/mov.sm
+++ b/gas/testsuite/gas/rx/mov.sm
@@ -31,3 +31,10 @@
 	mov{bwl}	[{reg}+],{reg}
 	mov{bwl}	{reg},[-{reg}]
 	mov{bwl}	[-{reg}],{reg}
+
+1:	mov.B		2f - 1b[r0], r0
+2:
+1:	mov.W		2f - 1b[r0], r0
+2:
+1:	mov.L		2f - 1b[r0], r0
+2: