The problem was originally reported in https://github.com/bitcoin/bitcoin/issues/22450#issuecomment-880629375. This is a dedicated issue to track it independently because it is a different problem than the one reported in #22450 OP/description.

```
fuzz: addrman.cpp:149: void CAddrMan::SwapRandom(unsigned int, unsigned int): Assertion `nRndPos1 < vRandom.size() && nRndPos2 < vRandom.size()' failed.
```

To reproduce:
```sh
FUZZ=addrman_deserialize ./src/test/fuzz/fuzz /tmp/crash-ef7d4a4f940b8fdb4999f6a92969260cf8e1405f.log
```
[crash-ef7d4a4f940b8fdb4999f6a92969260cf8e1405f.log](https://github.com/bitcoin/bitcoin/files/6830072/crash-ef7d4a4f940b8fdb4999f6a92969260cf8e1405f.log)

{
        LOCK(cs);
        return CAddrMan::Find(addr, pnId);
    }
    CAddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr)
    {
        LOCK(cs);
        return CAddrMan::Create(addr, addrSource, pnId);
    }
    void Delete(int nId)
    {
        LOCK(cs);
        CAddrMan::Delete(nId);
    }
    
    std::pair<int, int> GetBucketAndEntry(const CAddress& addr)
    {
        LOCK(cs);
        int nId = mapAddr[addr];
        for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) {
            for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) {
                if (nId == vvNew[bucket][entry]) {
                    return std::pair<int, int>(bucket, entry);
                }
            }
        }
        return std::pair<int, int>(-1, -1);
    }
    
    void SimConnFail(const CService& addr)