diff --git a/websocket-sharp/Server/WebSocketServer.cs b/websocket-sharp/Server/WebSocketServer.cs
index b4595a35..4b33e064 100644
--- a/websocket-sharp/Server/WebSocketServer.cs
+++ b/websocket-sharp/Server/WebSocketServer.cs
@@ -838,6 +838,18 @@ namespace WebSocketSharp.Server
 
           break;
         }
+        catch (InvalidOperationException ex) {
+          if (_state == ServerState.ShuttingDown) {
+            _log.Info ("The underlying listener is stopped.");
+
+            return;
+          }
+
+          _log.Fatal (ex.Message);
+          _log.Debug (ex.ToString ());
+
+          break;
+        }
         catch (Exception ex) {
           _log.Fatal (ex.Message);
           _log.Debug (ex.ToString ());