I noticed my node was not syncing, and through some debugging on #bitcoin-core-dev it seems like it was caused by running out of disk space which left the node in a desynced state, with the valid chain marked as invalid.

After freeing up some space and doing `reconsiderblock` it fixed it.

Comments from @sipa:

> Ugh. That is bad. Out of disk space should not result in database corruption.

> Database errors propagating up and being interpreted as (permanent) block invalidity was one of the contributing factors to the BDB/LevelDB fork in the 0.7/0.8 transition. 

logs:
```
2022-09-05T00:28:55Z UpdateTip: new best=000000000000000000087e337539b369b19570a88c493725020266387d22dac0 height=752167 version=0x2ce8e004 log2_work=93.70770
7 tx=761139817 date='2022-09-01T15:00:06Z' progress=0.998887 cache=146.8MiB(1101860txo)
2022-09-05T00:28:55Z Fatal LevelDB error: IO error: /home/jb55/.bitcoin/chainstate/2963623.ldb: No space left on device
2022-09-05T00:28:55Z You can use -debug=leveldb to get more complete diagnostic messages
2022-09-05T00:28:55Z *** System error while flushing: Fatal LevelDB error: IO error: /home/jb55/.bitcoin/chainstate/2963623.ldb: No space left on device
2022-09-05T00:28:55Z Error: A fatal internal error occurred, see debug.log for details
2022-09-05T00:28:56Z ERROR: ProcessNewBlock: ActivateBestChain failed (System error while flushing: Fatal LevelDB error: IO error: /home/jb55/.bitcoin/chains
tate/2963623.ldb: No space left on device)
2022-09-05T00:28:56Z ERROR: ConnectBlock: Consensus::CheckTxInputs: 221fa678c5c9953d6cd17e584f05c12ab10ba0f2fc8e8131e266f3f0e9819848, bad-txns-inputs-missing
orspent, CheckTxInputs: inputs missing/spent
2022-09-05T00:28:56Z InvalidChainFound: invalid block=000000000000000000079ba062298aa7cef1888f870c23e303a361c19b097ff8  height=752168  log2_work=93.707719  date=2022-09-01T15:13:42Z
2022-09-05T00:28:56Z InvalidChainFound:  current best=000000000000000000087e337539b369b19570a88c493725020266387d22dac0  height=752167  log2_work=93.707707  date=2022-09-01T15:00:06Z
2022-09-05T00:28:56Z ERROR: ConnectTip: ConnectBlock 000000000000000000079ba062298aa7cef1888f870c23e303a361c19b097ff8 failed, bad-txns-inputs-missingorspent, CheckTxInputs: inputs missing/spent
2022-09-05T00:28:56Z InvalidChainFound: invalid block=000000000000000000079ba062298aa7cef1888f870c23e303a361c19b097ff8  height=752168  log2_work=93.707719  date=2022-09-01T15:13:42Z
2022-09-05T00:28:56Z InvalidChainFound:  current best=000000000000000000087e337539b369b19570a88c493725020266387d22dac0  height=752167  log2_work=93.707707  date=2022-09-01T15:00:06Z
```

#if HAVE_SYSTEM
    argsman.AddArg("-alertnotify=<cmd>", "Execute command when an alert is raised (%s in cmd is replaced by message)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
#endif
    argsman.AddArg("-assumevalid=<hex>", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), signetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-blocksdir=<dir>", "Specify directory to hold blocks subdirectory for *.dat files (default: <datadir>)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
#if HAVE_SYSTEM
    argsman.AddArg("-blocknotify=<cmd>", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
#endif
    argsman.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutsetinfo RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
    argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-minimumchainwork=<hex>", strprintf("Minimum work assumed to exist on a valid chain in hex (default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex(), signetChainParams->GetConsensus().nMinimumChainWork.GetHex()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
    argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)",
        -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
    argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex. "
            "Warning: Reverting this setting requires re-downloading the entire blockchain. "
            "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES  1024  1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);