The `recoverLocallyUpToGlobalCheckpoint` step of peer recovery may take an extended period of time, but today we do not disable the recovery monitor during this step so if it takes more than 30 minutes (by default) then the recovery will time out, fail, and retry repeatedly. We should disable the recovery monitor during this step.
server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
.append(", took [")
                    .append(timeValueMillis(recoveryResponse.phase2Time))
                    .append("]")
                    .append("\n");
                logger.trace("{}", sb);
            } else {
                logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), request.sourceNode(), recoveryTime);
            }
        }
        @Override
        public void handleException(TransportException e) {
            if (logger.isTraceEnabled()) {
                logger.trace(
                    () -> format("[%s][%s] Got exception on recovery", request.shardId().getIndex().getName(), request.shardId().id()),
                    e
                );
            }
            Throwable cause = ExceptionsHelper.unwrapCause(e);
            if (transportService.lifecycleState() != Lifecycle.State.STARTED) {
                
                onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "node is shutting down", cause), false);
                return;
            }
            if (cause instanceof CancellableThreads.ExecutionCancelledException) {
                
                onGoingRecoveries.failRecovery(
                    recoveryId,
                    new RecoveryFailedException(request, "source has canceled the recovery", cause),
                    false
                );