diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 5fca59b28f2..96744c2f13c 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -548,6 +548,7 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid)
 	char	   *linkloc = palloc(OIDCHARS + OIDCHARS + 1);
 	char	   *location_with_version_dir = palloc(strlen(location) + 1 +
 								   strlen(TABLESPACE_VERSION_DIRECTORY) + 1);
+	struct stat st;
 
 	sprintf(linkloc, "pg_tblspc/%u", tablespaceoid);
 	sprintf(location_with_version_dir, "%s/%s", location,
@@ -574,8 +575,6 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid)
 
 	if (InRecovery)
 	{
-		struct stat st;
-
 		/*
 		 * Our theory for replaying a CREATE is to forcibly drop the target
 		 * subdirectory if present, and then recreate it. This may be more
@@ -609,14 +608,32 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid)
 							location_with_version_dir)));
 	}
 
-	/* Remove old symlink in recovery, in case it points to the wrong place */
+	/*
+	 * In recovery, remove old symlink, in case it points to the wrong place.
+	 *
+	 * On Windows, junction points act like directories so we must be able to
+	 * apply rmdir; in general it seems best to make this code work like the
+	 * symlink removal code in destroy_tablespace_directories, except that
+	 * failure to remove is always an ERROR.
+	 */
 	if (InRecovery)
 	{
-		if (unlink(linkloc) < 0 && errno != ENOENT)
-			ereport(ERROR,
-					(errcode_for_file_access(),
-					 errmsg("could not remove symbolic link \"%s\": %m",
-							linkloc)));
+		if (lstat(linkloc, &st) == 0 && S_ISDIR(st.st_mode))
+		{
+			if (rmdir(linkloc) < 0)
+				ereport(ERROR,
+						(errcode_for_file_access(),
+						 errmsg("could not remove directory \"%s\": %m",
+								linkloc)));
+		}
+		else
+		{
+			if (unlink(linkloc) < 0 && errno != ENOENT)
+				ereport(ERROR,
+						(errcode_for_file_access(),
+						 errmsg("could not remove symbolic link \"%s\": %m",
+								linkloc)));
+		}
 	}
 
 	/*