diff --git a/websocket-sharp/Net/HttpListenerPrefixCollection.cs b/websocket-sharp/Net/HttpListenerPrefixCollection.cs
index 53532a1f..9d241633 100644
--- a/websocket-sharp/Net/HttpListenerPrefixCollection.cs
+++ b/websocket-sharp/Net/HttpListenerPrefixCollection.cs
@@ -166,12 +166,10 @@ namespace WebSocketSharp.Net
       if (_listener.IsDisposed)
         throw new ObjectDisposedException (_listener.GetType ().ToString ());
 
+      if (_listener.IsListening)
+        EndPointManager.RemoveListener (_listener);
+
       _prefixes.Clear ();
-
-      if (!_listener.IsListening)
-        return;
-
-      EndPointManager.RemoveListener (_listener);
     }
 
     /// <summary>