Performing any long running task or CPU intensive work on transport_worker threads is problematic. This is typically very bad for cluster stability.

This issue has been filed to examine the usage of transport_worked threads by ESQL, with the intention of "offloading" as much as possible, including deserialising large(ish) amounts of data.
x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
final long maxOutputSeqNo;
        final Set<Integer> receivedSeqNos = ConcurrentCollections.newConcurrentSet();
        SeqNoCollector(long maxOutputSeqNo) {
            this.maxOutputSeqNo = maxOutputSeqNo;
        }
        @Override
        public String describe() {
            return "SeqNoCollector(maxOutputSeqNo=" + maxOutputSeqNo + ")";
        }
        @Override
        public SinkOperator get(DriverContext driverContext) {
            return new SinkOperator() {
                private boolean finished = false;
                @Override
                public boolean needsInput() {
                    return isFinished() == false;
                }
                @Override
                public void addInput(Page page) {
                    assertFalse("already finished", finished);
                    IntBlock block = page.getBlock(0);
                    for (int i = 0; i < block.getPositionCount(); i++) {
                        int v = block.getInt(i);
                        if (v < maxOutputSeqNo) {
                            assertTrue(receivedSeqNos.add(v));
                            
                            if (receivedSeqNos.size() >= maxOutputSeqNo && randomBoolean()) {
                                finished = true;
                            }