diff --git a/websocket-sharp/Net/HttpListenerPrefixCollection.cs b/websocket-sharp/Net/HttpListenerPrefixCollection.cs
index 450c2986..6e2b2118 100644
--- a/websocket-sharp/Net/HttpListenerPrefixCollection.cs
+++ b/websocket-sharp/Net/HttpListenerPrefixCollection.cs
@@ -163,8 +163,11 @@ namespace WebSocketSharp.Net
     {
       _listener.CheckDisposed ();
       _prefixes.Clear ();
-      if (_listener.IsListening)
-        EndPointManager.RemoveListener (_listener);
+
+      if (!_listener.IsListening)
+        return;
+
+      EndPointManager.RemoveListener (_listener);
     }
 
     /// <summary>