diff --git a/binaries/data/mods/public/gui/session/minimap_panel.xml b/binaries/data/mods/public/gui/session/minimap_panel.xml
index ae8d207941..6980215203 100644
--- a/binaries/data/mods/public/gui/session/minimap_panel.xml
+++ b/binaries/data/mods/public/gui/session/minimap_panel.xml
@@ -5,28 +5,27 @@
 	type="image"
 	sprite="mapPanel"
 >
-	<!-- Idle Worker Button -->
-	<object size="100%-80 100%-80 100%-5 100%-5">
-		<object type="button"
-			name="idleWorkerButton"
-			tooltip_style="sessionToolTip"
-			hotkey="selection.idleworker"
-		>
-			<action on="Press">if (g_HasIdleWorker) findIdleUnit(g_WorkerTypes);</action>
-			<action on="MouseEnter">if (g_HasIdleWorker) Engine.GetGUIObjectByName("idleOverlay").sprite = "stretched:session/minimap-idle-highlight.png";</action>
-			<action on="MouseLeave">if (g_HasIdleWorker) Engine.GetGUIObjectByName("idleOverlay").sprite = "stretched:session/minimap-idle.png";</action>
-			<action on="MouseLeftPress">if (g_HasIdleWorker) Engine.GetGUIObjectByName("idleOverlay").sprite = "stretched:session/minimap-idle.png";</action>
-			<action on="MouseLeftRelease">if (g_HasIdleWorker) Engine.GetGUIObjectByName("idleOverlay").sprite = "stretched:session/minimap-idle-highlight.png";</action>
-		</object>
-	</object>
 	<!-- Minimap -->
-	<object name="minimap"
-		type="minimap" z="20"
-		size="8 8 100%-8 100%-8"
-	>
+	<object size="8 8 100%-8 100%-8" type="minimap" z="20">
 		<action on="WorldClick">handleMinimapEvent(arguments[0]);</action>
 	</object>
-	<!-- Overlays -->
-	<object name="minimapOverlay" size="4 4 100%-4 100%-4" type="image" sprite="stretched:session/minimap_circle_modern.png" ghost="true"/>
-	<object name="idleOverlay" z="100" size="100%-125 100%-125 100%-5 100%-5" type="image" sprite="stretched:session/minimap-idle.png" ghost="true"/>
+	<object
+		type="image"
+		size="4 4 100%-4 100%-4"
+		sprite="stretched:session/minimap_circle_modern.png"
+		ghost="true"
+	/>
+
+	<!-- Idle Worker Button -->
+	<object
+		name="idleWorkerButton"
+		type="button"
+		size="100%-120 100%-120 100%-5 100%-5"
+		tooltip_style="sessionToolTip" hotkey="selection.idleworker"
+		sprite="stretched:session/minimap-idle.png"
+		sprite_over="stretched:session/minimap-idle-highlight.png"
+		sprite_disabled="stretched:session/minimap-idle-disabled.png"
+	>
+		<action on="Press">findIdleUnit(g_WorkerTypes);</action>
+	</object>
 </object>
diff --git a/binaries/data/mods/public/gui/session/session.js b/binaries/data/mods/public/gui/session/session.js
index 132b0a7062..5cb74db27e 100644
--- a/binaries/data/mods/public/gui/session/session.js
+++ b/binaries/data/mods/public/gui/session/session.js
@@ -183,10 +183,6 @@ var g_WorkerTypes = ["FemaleCitizen", "Trader", "FishingBoat", "CitizenSoldier"]
  * Unit classes to be checked for the military-only-selection modifier and for the idle-warrior-hotkey.
  */
 var g_MilitaryTypes = ["Melee", "Ranged"];
-/**
- * Cache the idle worker status.
- */
-var g_HasIdleWorker = false;
 
 function GetSimState()
 {
@@ -778,27 +774,15 @@ function changeGameSpeed(speed)
 		Engine.SetSimRate(speed);
 }
 
-function hasIdleWorker()
+function updateIdleWorkerButton()
 {
-	return Engine.GuiInterfaceCall("HasIdleUnits", {
+	Engine.GetGUIObjectByName("idleWorkerButton").enabled = Engine.GuiInterfaceCall("HasIdleUnits", {
 		"viewedPlayer": g_ViewedPlayer,
 		"idleClasses": g_WorkerTypes,
 		"excludeUnits": []
 	});
 }
 
-function updateIdleWorkerButton()
-{
-	g_HasIdleWorker = hasIdleWorker();
-
-	let idleWorkerButton = Engine.GetGUIObjectByName("idleOverlay");
-	let prefix = "stretched:session/";
-
-	if (!g_HasIdleWorker)
-		idleWorkerButton.sprite = prefix + "minimap-idle-disabled.png";
-	else if (idleWorkerButton.sprite != prefix + "minimap-idle-highlight.png")
-		idleWorkerButton.sprite = prefix + "minimap-idle.png";
-}
 function onSimulationUpdate()
 {
 	// Templates change depending on technologies and auras, so they have to be reloaded every turn.