[Modify] Move it

websocket-sharp/Net/HttpListener.cs

@@ -813,8 +813,6 @@ namespace WebSocketSharp.Net
         if (!_listening)
           return;
 
-        EndPointManager.RemoveListener (this);
-
         cleanupContextQueue (true);
         cleanupContextRegistry ();
 
@@ -822,6 +820,8 @@ namespace WebSocketSharp.Net
         var ex = new HttpListenerException (995, msg);
         cleanupWaitQueue (ex);
 
+        EndPointManager.RemoveListener (this);
+
         _listening = false;
       }
     }