Fix toy_text rendering on headless machines
diff --git a/gym/envs/toy_text/cliffwalking.py b/gym/envs/toy_text/cliffwalking.py
index 2219f51ffbf..95224f62c78 100644
--- a/gym/envs/toy_text/cliffwalking.py
+++ b/gym/envs/toy_text/cliffwalking.py
@@ -183,9 +183,10 @@ def _render_gui(self, mode):
 
         if self.window_surface is None:
             pygame.init()
-            pygame.display.init()
-            pygame.display.set_caption("CliffWalking")
+
             if mode == "human":
+                pygame.display.init()
+                pygame.display.set_caption("CliffWalking")
                 self.window_surface = pygame.display.set_mode(self.window_size)
             else:  # rgb_array
                 self.window_surface = pygame.Surface(self.window_size)
diff --git a/gym/envs/toy_text/frozen_lake.py b/gym/envs/toy_text/frozen_lake.py
index 9ceede61c60..1781db6c122 100644
--- a/gym/envs/toy_text/frozen_lake.py
+++ b/gym/envs/toy_text/frozen_lake.py
@@ -294,9 +294,10 @@ def _render_gui(self, mode):
 
         if self.window_surface is None:
             pygame.init()
-            pygame.display.init()
-            pygame.display.set_caption("Frozen Lake")
+
             if mode == "human":
+                pygame.display.init()
+                pygame.display.set_caption("Frozen Lake")
                 self.window_surface = pygame.display.set_mode(self.window_size)
             elif mode in {"rgb_array", "single_rgb_array"}:
                 self.window_surface = pygame.Surface(self.window_size)

# Description

Fixes #3032

The pygame code did stuff with the display even in `rgb_array` mode, which makes it impossible to run it on a headless machine. The relevant code has been moved into the `if` condition.

WIP because I need to check if it actually works on colab and on a head...ful? computer, and probably address potential similar problems in other envs.