We limit the number of concurrent blob downloads per node and per recovery, but if a user configures too many concurrent recoveries they may run out of per-node download permits which yields the following message:

> Unable to acquire permit to use snapshot files during recovery, this recovery will recover index files from the source node. Ensure snapshot files can be used during recovery by setting [indices.recovery.max_concurrent_snapshot_file_downloads] to be no greater than [25]

The `[25]` here is wrong, the 25 permits are shared across all inbound recoveries so we should divide the suggested value by `cluster.routing.allocation.node_concurrent_incoming_recoveries`. In extreme cases the user may have configured so many concurrent recoveries that the only reasonable limit is zero, and in that case we should suggest setting `indices.recovery.use_snapshots: false` instead.


server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
*/
    private static Setting<Double> operatorFactorSetting(String key, double defaultValue) {
        return new Setting<>(key, Double.toString(defaultValue), s -> Setting.parseDouble(s, 0d, 1d, key), v -> {
            if (v == 0d) {
                throw new IllegalArgumentException("Failed to validate value [" + v + "] for factor setting [" + key + "] must be > [0]");
            }
        }, Property.NodeScope, Property.OperatorDynamic);
    }
    private static Setting<Double> operatorFactorSetting(String key) {
        return new Setting<>(key, NODE_BANDWIDTH_RECOVERY_OPERATOR_FACTOR_SETTING, s -> Setting.parseDouble(s, 0d, 1d, key), v -> {
            if (v == 0d) {
                throw new IllegalArgumentException("Failed to validate value [" + v + "] for factor setting [" + key + "] must be > [0]");
            }
        }, Property.NodeScope, Property.OperatorDynamic);
    }
    /**
     * User-defined factors have a value in (0.0, 1.0] and fall back to a corresponding operator factor setting.
     */
    private static Setting<Double> factorSetting(String key, Setting<Double> operatorFallback) {
        return new Setting<>(key, operatorFallback, s -> Setting.parseDouble(s, 0d, 1d, key), v -> {
            if (v == 0d) {
                throw new IllegalArgumentException("Failed to validate value [" + v + "] for factor setting [" + key + "] must be > [0]");
            }
        }, Property.NodeScope, Property.Dynamic);
    }
    static final ByteSizeValue DEFAULT_MAX_BYTES_PER_SEC = new ByteSizeValue(40L, ByteSizeUnit.MB);
    public static final Setting<ByteSizeValue> INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting(
        "indices.recovery.max_bytes_per_sec",
        s -> {
            final List<DiscoveryNodeRole> roles = NodeRoleSettings.NODE_ROLES_SETTING.get(s);