diff --git a/websocket-sharp/Net/HttpListener.cs b/websocket-sharp/Net/HttpListener.cs
index ea9cbee5..f7b3256c 100644
--- a/websocket-sharp/Net/HttpListener.cs
+++ b/websocket-sharp/Net/HttpListener.cs
@@ -669,18 +669,7 @@ namespace WebSocketSharp.Net
       if (!ares.IsCompleted)
         ares.AsyncWaitHandle.WaitOne ();
 
-      lock (_waitQueueSync) {
-        var i = _waitQueue.IndexOf (ares);
-        if (i >= 0)
-          _waitQueue.RemoveAt (i);
-      }
-
-      var ctx = ares.GetContext ();
-      var schm = SelectAuthenticationScheme (ctx);
-      if (schm != AuthenticationSchemes.Anonymous)
-        ctx.SetUser (schm, Realm, UserCredentialsFinder);
-
-      return ctx; // This will throw on error.
+      return ares.GetContext (); // This will throw on error.
     }
 
     /// <summary>
diff --git a/websocket-sharp/Net/ListenerAsyncResult.cs b/websocket-sharp/Net/ListenerAsyncResult.cs
index e4a9fe1a..ac53fdf3 100644
--- a/websocket-sharp/Net/ListenerAsyncResult.cs
+++ b/websocket-sharp/Net/ListenerAsyncResult.cs
@@ -107,6 +107,57 @@ namespace WebSocketSharp.Net
 
     #region Private Methods
 
+    private static bool authenticate (HttpListenerContext context)
+    {
+      var listener = context.Listener;
+      var schm = listener.SelectAuthenticationScheme (context);
+      if (schm == AuthenticationSchemes.Anonymous)
+        return true;
+
+      if (schm == AuthenticationSchemes.None) {
+        context.Response.Close (HttpStatusCode.Forbidden);
+        return false;
+      }
+
+      var req = context.Request;
+      var authRes = req.Headers["Authorization"];
+      if (schm == AuthenticationSchemes.Basic) {
+        if (authRes == null || !authRes.StartsWith ("basic", StringComparison.OrdinalIgnoreCase)) {
+          context.Response.CloseWithAuthChallenge (
+            AuthenticationChallenge.CreateBasicChallenge (listener.Realm).ToBasicString ());
+
+          return false;
+        }
+      }
+      else if (schm == AuthenticationSchemes.Digest) {
+        if (authRes == null || !authRes.StartsWith ("digest", StringComparison.OrdinalIgnoreCase)) {
+          context.Response.CloseWithAuthChallenge (
+            AuthenticationChallenge.CreateDigestChallenge (listener.Realm).ToDigestString ());
+
+          return false;
+        }
+      }
+      else {
+        context.Response.Close (HttpStatusCode.Forbidden);
+        return false;
+      }
+
+      var realm = listener.Realm;
+      context.SetUser (schm, realm, listener.UserCredentialsFinder);
+      if (req.IsAuthenticated)
+        return true;
+
+      if (schm == AuthenticationSchemes.Basic)
+        context.Response.CloseWithAuthChallenge (
+          AuthenticationChallenge.CreateBasicChallenge (realm).ToBasicString ());
+
+      if (schm == AuthenticationSchemes.Digest)
+        context.Response.CloseWithAuthChallenge (
+          AuthenticationChallenge.CreateDigestChallenge (realm).ToDigestString ());
+
+      return false;
+    }
+
     private static void complete (ListenerAsyncResult asyncResult)
     {
       asyncResult._completed = true;
@@ -149,31 +200,8 @@ namespace WebSocketSharp.Net
 
     internal void Complete (HttpListenerContext context, bool syncCompleted)
     {
-      var listener = context.Listener;
-      var schm = listener.SelectAuthenticationScheme (context);
-      if (schm == AuthenticationSchemes.None) {
-        context.Response.Close (HttpStatusCode.Forbidden);
-        listener.BeginGetContext (this);
-
-        return;
-      }
-
-      var res = context.Request.Headers["Authorization"];
-      if (schm == AuthenticationSchemes.Basic &&
-          (res == null || !res.StartsWith ("basic", StringComparison.OrdinalIgnoreCase))) {
-        context.Response.CloseWithAuthChallenge (
-          AuthenticationChallenge.CreateBasicChallenge (listener.Realm).ToBasicString ());
-
-        listener.BeginGetContext (this);
-        return;
-      }
-
-      if (schm == AuthenticationSchemes.Digest &&
-          (res == null || !res.StartsWith ("digest", StringComparison.OrdinalIgnoreCase))) {
-        context.Response.CloseWithAuthChallenge (
-          AuthenticationChallenge.CreateDigestChallenge (listener.Realm).ToDigestString ());
-
-        listener.BeginGetContext (this);
+      if (!authenticate (context)) {
+        context.Listener.BeginGetContext (this);
         return;
       }
 
diff --git a/websocket-sharp/Server/HttpServer.cs b/websocket-sharp/Server/HttpServer.cs
index 09e15e2c..d651c00e 100644
--- a/websocket-sharp/Server/HttpServer.cs
+++ b/websocket-sharp/Server/HttpServer.cs
@@ -486,23 +486,6 @@ namespace WebSocketSharp.Server
       _state = ServerState.Stop;
     }
 
-    private bool authenticateRequest (AuthenticationSchemes scheme, HttpListenerContext context)
-    {
-      if (context.Request.IsAuthenticated)
-        return true;
-
-      if (scheme == AuthenticationSchemes.Basic)
-        context.Response.CloseWithAuthChallenge (
-          AuthenticationChallenge.CreateBasicChallenge (_listener.Realm).ToBasicString ());
-      else if (scheme == AuthenticationSchemes.Digest)
-        context.Response.CloseWithAuthChallenge (
-          AuthenticationChallenge.CreateDigestChallenge (_listener.Realm).ToDigestString ());
-      else
-        context.Response.Close (HttpStatusCode.Forbidden);
-
-      return false;
-    }
-
     private string checkIfCertificateExists ()
     {
       if (!_secure)
@@ -570,11 +553,6 @@ namespace WebSocketSharp.Server
           ThreadPool.QueueUserWorkItem (
             state => {
               try {
-                var schm = _listener.SelectAuthenticationScheme (ctx);
-                if (schm != AuthenticationSchemes.Anonymous &&
-                    !authenticateRequest (schm, ctx))
-                  return;
-
                 if (ctx.Request.IsUpgradeTo ("websocket")) {
                   processWebSocketRequest (ctx.AcceptWebSocket (null, _logger));
                   return;