Limit unnecessary concurrent CDN token requests. Combine CDN token requests for steamcontent.com. Added a diagnostic timeout for lost CDN token requests.

DepotDownloader/CDNClientPool.cs

@@ -20,8 +20,6 @@ namespace DepotDownloader
 
         public CDNClient CDNClient { get; }
 
-        private readonly ConcurrentDictionary<Tuple<uint, string>, string> depotCdnKeys;
-
         private readonly ConcurrentBag<CDNClient.Server> activeConnectionPool;
         private readonly BlockingCollection<CDNClient.Server> availableServerEndpoints;
 
@@ -34,7 +32,6 @@ namespace DepotDownloader
         {
             this.steamSession = steamSession;
             CDNClient = new CDNClient(steamSession.steamClient);
-            depotCdnKeys = new ConcurrentDictionary<Tuple<uint, string>, string>();
 
             activeConnectionPool = new ConcurrentBag<CDNClient.Server>();
             availableServerEndpoints = new BlockingCollection<CDNClient.Server>();
@@ -125,15 +122,17 @@ namespace DepotDownloader
             }
         }
 
-        private string AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server)
+        private async Task<string> AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server)
         {
-            steamSession.RequestCDNAuthToken(appId, depotId, server.Host);
+            var host = steamSession.ResolveCDNTopLevelHost(server.Host);
+            var cdnKey = $"{depotId:D}:{host}";
 
-            var cdnKey = $"{depotId:D}:{steamSession.ResolveCDNTopLevelHost(server.Host)}";
+            steamSession.RequestCDNAuthToken(appId, depotId, host, cdnKey);
 
-            if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallback))
+            if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallbackPromise))
             {
-                return authTokenCallback.Token;
+                var result = await authTokenCallbackPromise.Task;
+                return result.Token;
             }
             else
             {
@@ -151,7 +150,7 @@ namespace DepotDownloader
             return availableServerEndpoints.Take(token);
         }
 
-        public Tuple<CDNClient.Server, string> GetConnectionForDepot(uint appId, uint depotId, CancellationToken token)
+        public async Task<Tuple<CDNClient.Server, string>> GetConnectionForDepot(uint appId, uint depotId, CancellationToken token)
         {
             // Take a free connection from the connection pool
             // If there were no free connections, create a new one from the server list
@@ -160,14 +159,8 @@ namespace DepotDownloader
                 server = BuildConnection(token);
             }
 
-            var topLevelHost = steamSession.ResolveCDNTopLevelHost(server.Host);
-            var depotKey = Tuple.Create(depotId, topLevelHost);
-
             // If we don't have a CDN token yet for this server and depot, fetch one now
-            if (!depotCdnKeys.TryGetValue(depotKey, out var cdnToken))
-            {
-                depotCdnKeys[depotKey] = cdnToken = AuthenticateConnection(appId, depotId, server);
-            }
+            var cdnToken = await AuthenticateConnection(appId, depotId, server);
 
             return Tuple.Create(server, cdnToken);
         }

DepotDownloader/ContentDownloader.cs

@@ -665,7 +665,7 @@ namespace DepotDownloader
                             Tuple<CDNClient.Server, string> connection = null;
                             try
                             {
-                                connection = cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None );
+                                connection = await cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None );
 
                                 depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.id, depot.manifestId,
                                     connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait(false);
@@ -890,7 +890,7 @@ namespace DepotDownloader
                                     Tuple<CDNClient.Server, string> connection;
                                     try
                                     {
-                                        connection = cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token );
+                                        connection = await cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token );
                                     }
                                     catch ( OperationCanceledException )
                                     {

DepotDownloader/Steam3Session.cs

@@ -7,6 +7,7 @@ using System.Collections.ObjectModel;
 using System.IO;
 using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace DepotDownloader
 {
@@ -33,7 +34,7 @@ namespace DepotDownloader
         public Dictionary<uint, byte[]> AppTickets { get; private set; }
         public Dictionary<uint, ulong> AppTokens { get; private set; }
         public Dictionary<uint, byte[]> DepotKeys { get; private set; }
-        public ConcurrentDictionary<string, SteamApps.CDNAuthTokenCallback> CDNAuthTokens { get; private set; }
+        public ConcurrentDictionary<string, TaskCompletionSource<SteamApps.CDNAuthTokenCallback>> CDNAuthTokens { get; private set; }
         public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> AppInfo { get; private set; }
         public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> PackageInfo { get; private set; }
         public Dictionary<string, byte[]> AppBetaPasswords { get; private set; }
@@ -82,7 +83,7 @@ namespace DepotDownloader
             this.AppTickets = new Dictionary<uint, byte[]>();
             this.AppTokens = new Dictionary<uint, ulong>();
             this.DepotKeys = new Dictionary<uint, byte[]>();
-            this.CDNAuthTokens = new ConcurrentDictionary<string, SteamApps.CDNAuthTokenCallback>();
+            this.CDNAuthTokens = new ConcurrentDictionary<string, TaskCompletionSource<SteamApps.CDNAuthTokenCallback>>();
             this.AppInfo = new Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo>();
             this.PackageInfo = new Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo>();
             this.AppBetaPasswords = new Dictionary<string, byte[]>();
@@ -328,19 +329,24 @@ namespace DepotDownloader
             {
                 return "steampipe.steamcontent.com";
             }
+            else if (host.EndsWith(".steamcontent.com"))
+            {
+                return "steamcontent.com";
+            }
 
             return host;
         }
 
-        public void RequestCDNAuthToken( uint appid, uint depotid, string host )
+        public void RequestCDNAuthToken( uint appid, uint depotid, string host, string cdnKey )
         {
-            host = ResolveCDNTopLevelHost( host );
-            var cdnKey = string.Format( "{0:D}:{1}", depotid, host );
-
             if ( CDNAuthTokens.ContainsKey( cdnKey ) || bAborted )
                 return;
 
+            if ( !CDNAuthTokens.TryAdd( cdnKey, new TaskCompletionSource<SteamApps.CDNAuthTokenCallback>() ) )
+                return;
+
             bool completed = false;
+            var timeoutDate = DateTime.Now.AddSeconds( 10 );
             Action<SteamApps.CDNAuthTokenCallback> cbMethod = ( cdnAuth ) =>
             {
                 completed = true;
@@ -352,13 +358,13 @@ namespace DepotDownloader
                     return;
                 }
 
-                CDNAuthTokens.TryAdd( cdnKey, cdnAuth );
+                CDNAuthTokens[cdnKey].TrySetResult( cdnAuth );
             };
 
             WaitUntilCallback( () =>
             {
                 callbacks.Subscribe( steamApps.GetCDNAuthToken( appid, depotid, host ), cbMethod );
-            }, () => { return completed; } );
+            }, () => { return completed || DateTime.Now >= timeoutDate; } );
         }
 
         public void CheckAppBetaPassword( uint appid, string password )