<!-- If you need to report a security issue please visit https://github.com/apache/dubbo/security/policy -->

- [ ] I have searched the [issues](https://github.com/apache/dubbo/issues) of this repository and believe that this is not a duplicate.

ReferenceConfig throw IllegalStateException when check failed, but consumerModel is still registered.
private static final Logger logger = LoggerFactory.getLogger(RegistryDirectory.class);
    private final ConsumerConfigurationListener consumerConfigurationListener;
    private ReferenceConfigurationListener referenceConfigurationListener;
    /**
     * Map<url, Invoker> cache service url to invoker mapping.
     * The initial value is null and the midway may be assigned to null, please use the local variable reference
     */
    protected volatile Map<URL, Invoker<T>> urlInvokerMap;
    /**
     * The initial value is null and the midway may be assigned to null, please use the local variable reference
     */
    protected volatile Set<URL> cachedInvokerUrls;
    private final ApplicationModel applicationModel;
    public RegistryDirectory(Class<T> serviceType, URL url) {
        super(serviceType, url);
        applicationModel = getApplicationModel(url.getScopeModel());
        consumerConfigurationListener = getConsumerConfigurationListener(url.getOrDefaultModuleModel());
    }
    @Override
    public void subscribe(URL url) {
        super.subscribe(url);