The `recoverLocallyUpToGlobalCheckpoint` step of peer recovery may take an extended period of time, but today we do not disable the recovery monitor during this step so if it takes more than 30 minutes (by default) then the recovery will time out, fail, and retry repeatedly. We should disable the recovery monitor during this step.
server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
if (recoveryRef == null) {
            logger.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
            return;
        }
        final RecoveryTarget recoveryTarget = recoveryRef.target();
        assert recoveryTarget.sourceNode() != null : "cannot do a recovery without a source node";
        final RecoveryState recoveryState = recoveryTarget.state();
        final RecoveryState.Timer timer = recoveryState.getTimer();
        final IndexShard indexShard = recoveryTarget.indexShard();
        final var failureHandler = ActionListener.notifyOnce(ActionListener.runBefore(ActionListener.noop().delegateResponse((l, e) -> {
            
            logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
            onGoingRecoveries.failRecovery(
                recoveryId,
                new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e),
                true
            );
        }), recoveryRef::close));
        if (indexShard.routingEntry().isPromotableToPrimary() == false) {
            assert preExistingRequest == null;
            assert indexShard.indexSettings().getIndexMetadata().isSearchableSnapshot() == false;
            try {
                indexShard.preRecovery(failureHandler.map(v -> {
                    logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
                    indexShard.prepareForIndexRecovery();
                    
                    recoveryState.setStage(RecoveryState.Stage.VERIFY_INDEX);
                    recoveryState.setStage(RecoveryState.Stage.TRANSLOG);
                    indexShard.openEngineAndSkipTranslogRecovery();
                    recoveryState.getIndex().setFileDetailsComplete();