Merge pull request #45 from SteamRE/ugc-cmd-cleanup

Add new -ugc option, removing -force-depot. (#41)

DepotDownloader/ContentDownloader.cs

@@ -16,6 +16,7 @@ namespace DepotDownloader
         public const uint INVALID_APP_ID = uint.MaxValue;
         public const uint INVALID_DEPOT_ID = uint.MaxValue;
         public const ulong INVALID_MANIFEST_ID = ulong.MaxValue;
+        public const string DEFAULT_BRANCH = "Public";
 
         public static DownloadConfig Config = new DownloadConfig();
 
@@ -125,10 +126,10 @@ namespace DepotDownloader
                 SteamApps.PICSProductInfoCallback.PICSProductInfo package;
                 if ( steam3.PackageInfo.TryGetValue( license, out package ) && package != null )
                 {
-                    if ( package.KeyValues[ "appids" ].Children.Any( child => child.AsInteger() == depotId ) )
+                    if ( package.KeyValues[ "appids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) )
                         return true;
 
-                    if ( package.KeyValues[ "depotids" ].Children.Any( child => child.AsInteger() == depotId ) )
+                    if ( package.KeyValues[ "depotids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) )
                         return true;
                 }
             }
@@ -197,9 +198,6 @@ namespace DepotDownloader
 
         static ulong GetSteam3DepotManifest( uint depotId, uint appId, string branch )
         {
-            if ( Config.ManifestId != INVALID_MANIFEST_ID )
-                return Config.ManifestId;
-
             KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots );
             KeyValue depotChild = depots[ depotId.ToString() ];
 
@@ -211,7 +209,7 @@ namespace DepotDownloader
             // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680.
             if ( depotChild[ "manifests" ] == KeyValue.Invalid && depotChild[ "depotfromapp" ] != KeyValue.Invalid )
             {
-                uint otherAppId = ( uint )depotChild[ "depotfromapp" ].AsInteger();
+                uint otherAppId = depotChild["depotfromapp"].AsUnsignedInteger();
                 if ( otherAppId == appId )
                 {
                     // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop.
@@ -376,7 +374,13 @@ namespace DepotDownloader
             steam3.Disconnect();
         }
 
-        public static async Task DownloadAppAsync( uint appId, uint depotId, string branch, string os = null, bool forceDepot = false )
+        public static async Task DownloadPubfileAsync( ulong publishedFileId )
+        {
+            var details = steam3.GetPubfileDetails(publishedFileId);
+            await DownloadAppAsync( details.consumer_appid, details.consumer_appid, details.hcontent_file, DEFAULT_BRANCH, null, true );
+        }
+
+        public static async Task DownloadAppAsync( uint appId, uint depotId, ulong manifestId, string branch, string os, bool isUgc )
         {
             if ( steam3 != null )
                 steam3.RequestAppInfo( appId );
@@ -395,18 +399,21 @@ namespace DepotDownloader
                 }
             }
 
-            Console.WriteLine( "Using app branch: '{0}'.", branch );
-
             var depotIDs = new List<uint>();
             KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots );
 
-
-            if ( forceDepot )
+            if ( isUgc )
             {
+                var workshopDepot = depots["workshopdepot"].AsUnsignedInteger();
+                if (workshopDepot != 0)
+                    depotId = workshopDepot;
+
                 depotIDs.Add( depotId );
             }
             else
             {
+                Console.WriteLine( "Using app branch: '{0}'.", branch );
+
                 if ( depots != null )
                 {
                     foreach ( var depotSection in depots.Children )
@@ -456,7 +463,7 @@ namespace DepotDownloader
 
             foreach ( var depot in depotIDs )
             {
-                var info = GetDepotInfo( depot, appId, branch );
+                var info = GetDepotInfo( depot, appId, manifestId, branch );
                 if ( info != null )
                 {
                     infos.Add( info );
@@ -473,7 +480,7 @@ namespace DepotDownloader
             }
         }
 
-        static DepotDownloadInfo GetDepotInfo( uint depotId, uint appId, string branch )
+        static DepotDownloadInfo GetDepotInfo( uint depotId, uint appId, ulong manifestId, string branch )
         {
             if ( steam3 != null && appId != INVALID_APP_ID )
                 steam3.RequestAppInfo( ( uint )appId );
@@ -490,18 +497,21 @@ namespace DepotDownloader
             // Skip requesting an app ticket
             steam3.AppTickets[ depotId ] = null;
 
-            ulong manifestID = GetSteam3DepotManifest( depotId, appId, branch );
-            if ( manifestID == INVALID_MANIFEST_ID && branch != "public" )
+            if (manifestId == INVALID_MANIFEST_ID)
             {
-                Console.WriteLine( "Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch );
-                branch = "public";
-                manifestID = GetSteam3DepotManifest( depotId, appId, branch );
-            }
+                manifestId = GetSteam3DepotManifest(depotId, appId, branch);
+                if (manifestId == INVALID_MANIFEST_ID && branch != "public")
+                {
+                    Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch);
+                    branch = "public";
+                    manifestId = GetSteam3DepotManifest(depotId, appId, branch);
+                }
 
-            if ( manifestID == INVALID_MANIFEST_ID )
-            {
-                Console.WriteLine( "Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName );
-                return null;
+                if (manifestId == INVALID_MANIFEST_ID)
+                {
+                    Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName);
+                    return null;
+                }
             }
 
             uint uVersion = GetSteam3AppBuildNumber( appId, branch );
@@ -522,7 +532,7 @@ namespace DepotDownloader
 
             byte[] depotKey = steam3.DepotKeys[ depotId ];
 
-            var info = new DepotDownloadInfo( depotId, manifestID, installDir, contentName );
+            var info = new DepotDownloadInfo( depotId, manifestId, installDir, contentName );
             info.depotKey = depotKey;
             return info;
         }
@@ -914,7 +924,7 @@ namespace DepotDownloader
                 ConfigStore.TheConfig.LastManifests[ depot.id ] = depot.manifestId;
                 ConfigStore.Save();
 
-                Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed );
+                Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed );
             }
 
             Console.WriteLine( "Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count );

DepotDownloader/DepotDownloader.csproj

@@ -6,6 +6,6 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="protobuf-net" Version="2.1.0" />
-    <PackageReference Include="SteamKit2" Version="2.0.0" />
+    <PackageReference Include="SteamKit2" Version="2.1.0" />
   </ItemGroup>
 </Project>

DepotDownloader/DownloadConfig.cs

@@ -18,8 +18,6 @@ namespace DepotDownloader
 
         public string BetaPassword { get; set; }
 
-        public ulong ManifestId { get; set; }
-
         public bool VerifyAll { get; set; }
 
         public int MaxServers { get; set; }

DepotDownloader/Program.cs

@@ -25,24 +25,13 @@ namespace DepotDownloader
 
             ConfigStore.LoadFromFile( Path.Combine( Directory.GetCurrentDirectory(), "DepotDownloader.config" ) );
 
-            bool bDumpManifest = HasParameter( args, "-manifest-only" );
-            uint appId = GetParameter<uint>( args, "-app", ContentDownloader.INVALID_APP_ID );
-            uint depotId = GetParameter<uint>( args, "-depot", ContentDownloader.INVALID_DEPOT_ID );
-            ContentDownloader.Config.ManifestId = GetParameter<ulong>( args, "-manifest", ContentDownloader.INVALID_MANIFEST_ID );
+            #region Common Options
 
-            if ( appId == ContentDownloader.INVALID_APP_ID )
-            {
-                Console.WriteLine( "Error: -app not specified!" );
-                return;
-            }
+            string username = GetParameter<string>( args, "-username" ) ?? GetParameter<string>( args, "-user" );
+            string password = GetParameter<string>( args, "-password" ) ?? GetParameter<string>( args, "-pass" );
+            ContentDownloader.Config.RememberPassword = HasParameter( args, "-remember-password" );
 
-            if ( depotId == ContentDownloader.INVALID_DEPOT_ID && ContentDownloader.Config.ManifestId != ContentDownloader.INVALID_MANIFEST_ID )
-            {
-                Console.WriteLine( "Error: -manifest requires -depot to be specified" );
-                return;
-            }
-
-            ContentDownloader.Config.DownloadManifestOnly = bDumpManifest;
+            ContentDownloader.Config.DownloadManifestOnly = HasParameter( args, "-manifest-only" );
 
             int cellId = GetParameter<int>( args, "-cellid", -1 );
             if ( cellId == -1 )
@@ -51,7 +40,6 @@ namespace DepotDownloader
             }
 
             ContentDownloader.Config.CellID = cellId;
-            ContentDownloader.Config.BetaPassword = GetParameter<string>( args, "-betapassword" );
 
             string fileList = GetParameter<string>( args, "-filelist" );
             string[] files = null;
@@ -60,7 +48,7 @@ namespace DepotDownloader
             {
                 try
                 {
-                    string fileListData = File.ReadAllText( fileList );
+                    string fileListData = File.ReadAllText(fileList);
                     files = fileListData.Split( new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries );
 
                     ContentDownloader.Config.UsingFileList = true;
@@ -83,37 +71,94 @@ namespace DepotDownloader
 
                     Console.WriteLine( "Using filelist: '{0}'.", fileList );
                 }
-                catch ( Exception ex )
+                catch (Exception ex)
                 {
                     Console.WriteLine( "Warning: Unable to load filelist: {0}", ex.ToString() );
                 }
             }
 
-            string username = GetParameter<string>( args, "-username" ) ?? GetParameter<string>( args, "-user" );
-            string password = GetParameter<string>( args, "-password" ) ?? GetParameter<string>( args, "-pass" );
-            ContentDownloader.Config.RememberPassword = HasParameter( args, "-remember-password" );
             ContentDownloader.Config.InstallDirectory = GetParameter<string>( args, "-dir" );
-            ContentDownloader.Config.DownloadAllPlatforms = HasParameter( args, "-all-platforms" );
+
             ContentDownloader.Config.VerifyAll = HasParameter( args, "-verify-all" ) || HasParameter( args, "-verify_all" ) || HasParameter( args, "-validate" );
             ContentDownloader.Config.MaxServers = GetParameter<int>( args, "-max-servers", 20 );
             ContentDownloader.Config.MaxDownloads = GetParameter<int>( args, "-max-downloads", 4 );
-            string branch = GetParameter<string>( args, "-branch" ) ?? GetParameter<string>( args, "-beta" ) ?? "Public";
-            bool forceDepot = HasParameter( args, "-force-depot" );
-            string os = GetParameter<string>( args, "-os", null );
-
-            if ( ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty( os ) )
-            {
-                Console.WriteLine( "Error: Cannot specify -os when -all-platforms is specified." );
-                return;
-            }
-
             ContentDownloader.Config.MaxServers = Math.Max( ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads );
 
+            #endregion
+
+            ulong pubFile = GetParameter<ulong>( args, "-pubfile", ContentDownloader.INVALID_MANIFEST_ID );
+            if ( pubFile != ContentDownloader.INVALID_MANIFEST_ID )
+            {
+                #region Pubfile Downloading
+
+                if ( InitializeSteam( username, password ) )
+                {
+                    await ContentDownloader.DownloadPubfileAsync( pubFile ).ConfigureAwait( false );
+                    ContentDownloader.ShutdownSteam3();
+                }
+
+                #endregion
+            }
+            else
+            {
+                #region App downloading
+
+                string branch = GetParameter<string>( args, "-branch" ) ?? GetParameter<string>( args, "-beta" ) ?? ContentDownloader.DEFAULT_BRANCH;
+                ContentDownloader.Config.BetaPassword = GetParameter<string>( args, "-betapassword" );
+
+                ContentDownloader.Config.DownloadAllPlatforms = HasParameter( args, "-all-platforms" );
+                string os = GetParameter<string>( args, "-os", null );
+
+                if ( ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty( os ) )
+                {
+                    Console.WriteLine("Error: Cannot specify -os when -all-platforms is specified.");
+                    return;
+                }
+
+                uint appId = GetParameter<uint>( args, "-app", ContentDownloader.INVALID_APP_ID );
+                if ( appId == ContentDownloader.INVALID_APP_ID )
+                {
+                    Console.WriteLine( "Error: -app not specified!" );
+                    return;
+                }
+
+                uint depotId;
+                bool isUGC = false;
+
+                ulong manifestId = GetParameter<ulong>( args, "-ugc", ContentDownloader.INVALID_MANIFEST_ID );
+                if ( manifestId != ContentDownloader.INVALID_MANIFEST_ID )
+                {
+                    depotId = appId;
+                    isUGC = true;
+                }
+                else
+                {
+                    depotId = GetParameter<uint>( args, "-depot", ContentDownloader.INVALID_DEPOT_ID );
+                    manifestId = GetParameter<ulong>( args, "-manifest", ContentDownloader.INVALID_MANIFEST_ID );
+                    if ( depotId == ContentDownloader.INVALID_DEPOT_ID && manifestId != ContentDownloader.INVALID_MANIFEST_ID )
+                    {
+                        Console.WriteLine( "Error: -manifest requires -depot to be specified" );
+                        return;
+                    }
+                }
+
+                if ( InitializeSteam( username, password ) )
+                {
+                    await ContentDownloader.DownloadAppAsync( appId, depotId, manifestId, branch, os, isUGC ).ConfigureAwait( false );
+                    ContentDownloader.ShutdownSteam3();
+                }
+
+                #endregion
+            }
+        }
+
+        static bool InitializeSteam( string username, string password )
+        {
             if ( username != null && password == null && ( !ContentDownloader.Config.RememberPassword || !ConfigStore.TheConfig.LoginKeys.ContainsKey( username ) ) )
             {
                 do
                 {
-                    Console.Write("Enter account password for \"{0}\": ", username);
+                    Console.Write( "Enter account password for \"{0}\": ", username );
                     password = Util.ReadPassword();
                     Console.WriteLine();
                 } while ( String.Empty == password );
@@ -126,11 +171,7 @@ namespace DepotDownloader
             // capture the supplied password in case we need to re-use it after checking the login key
             ContentDownloader.Config.SuppliedPassword = password;
 
-            if ( ContentDownloader.InitializeSteam3( username, password ) )
-            {
-                await ContentDownloader.DownloadAppAsync( appId, depotId, branch, os, forceDepot ).ConfigureAwait( false );
-                ContentDownloader.ShutdownSteam3();
-            }
+            return ContentDownloader.InitializeSteam3( username, password );
         }
 
         static int IndexOfParam( string[] args, string param )
@@ -167,28 +208,38 @@ namespace DepotDownloader
 
         static void PrintUsage()
         {
-            Console.WriteLine( "\nUsage: depotdownloader <parameters> [optional parameters]\n" );
-
+            Console.WriteLine();
+            Console.WriteLine( "Usage - downloading one or all depots for an app:" );
+            Console.WriteLine( "\tdepotdownloader -app <id> [-depot <id> [-manifest <id>] | [-ugc <id>]]" );
+            Console.WriteLine( "\t\t[-username <username> [-password <password>]] [other options]" );
+            Console.WriteLine();
+            Console.WriteLine( "Usage - downloading a Workshop item published via SteamUGC" );
+            Console.WriteLine( "\tdepotdownloader -pubfile <id> [-username <username> [-password <password>]]" );
+            Console.WriteLine();
             Console.WriteLine( "Parameters:" );
             Console.WriteLine( "\t-app <#>\t\t\t\t- the AppID to download." );
-            Console.WriteLine();
-
-            Console.WriteLine( "Optional Parameters:" );
-            Console.WriteLine( "\t-depot <#>\t\t\t- the DepotID to download." );
-            Console.WriteLine( "\t-cellid <#>\t\t\t- the overridden CellID of the content server to download from." );
-            Console.WriteLine( "\t-username <user>\t\t\t- the username of the account to login to for restricted content." );
-            Console.WriteLine( "\t-password <pass>\t\t\t- the password of the account to login to for restricted content." );
-            Console.WriteLine( "\t-remember-password\t\t\t- if set, remember the password for subsequent logins of this user." );
-            Console.WriteLine( "\t-dir <installdir>\t\t\t- the directory in which to place downloaded files." );
-            Console.WriteLine( "\t-os <os>\t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)" );
-            Console.WriteLine( "\t-filelist <filename.txt>\t\t- a list of files to download (from the manifest). Can optionally use regex to download only certain files." );
-            Console.WriteLine( "\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used." );
-            Console.WriteLine( "\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded." );
-            Console.WriteLine( "\t-beta <branchname>\t\t\t\t- download from specified branch if available (default: Public)." );
-            Console.WriteLine( "\t-betapassword <pass>\t\t\t- branch password if applicable." );
+            Console.WriteLine( "\t-depot <#>\t\t\t\t- the DepotID to download." );
             Console.WriteLine( "\t-manifest <id>\t\t\t- manifest id of content to download (requires -depot, default: current for branch)." );
-            Console.WriteLine( "\t-max-servers <#>\t\t\t- maximum number of content servers to use. (default: 8)." );
-            Console.WriteLine( "\t-max-downloads <#>\t\t\t- maximum number of chunks to download concurrently. (default: 4)." );
+            Console.WriteLine( "\t-ugc <#>\t\t\t\t- the UGC ID to download." );
+            Console.WriteLine( "\t-beta <branchname>\t\t\t- download from specified branch if available (default: Public)." );
+            Console.WriteLine( "\t-betapassword <pass>\t\t- branch password if applicable." );
+            Console.WriteLine( "\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used." );
+            Console.WriteLine( "\t-os <os>\t\t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)" );
+            Console.WriteLine();
+            Console.WriteLine( "\t-pubfile <#>\t\t\t- the PublishedFileId to download. (Will automatically resolve to UGC id)" );
+            Console.WriteLine();
+            Console.WriteLine( "\t-username <user>\t\t- the username of the account to login to for restricted content.");
+            Console.WriteLine( "\t-password <pass>\t\t- the password of the account to login to for restricted content." );
+            Console.WriteLine( "\t-remember-password\t\t- if set, remember the password for subsequent logins of this user." );
+            Console.WriteLine();
+            Console.WriteLine( "\t-dir <installdir>\t\t- the directory in which to place downloaded files." );
+            Console.WriteLine( "\t-filelist <file.txt>\t- a list of files to download (from the manifest). Can optionally use regex to download only certain files." );
+            Console.WriteLine( "\t-validate\t\t\t\t- Include checksum verification of files already downloaded" );
+            Console.WriteLine();
+            Console.WriteLine( "\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded." );
+            Console.WriteLine( "\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from." );
+            Console.WriteLine( "\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 8)." );
+            Console.WriteLine( "\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 4)." );
         }
     }
 }

DepotDownloader/Steam3Session.cs

@@ -1,4 +1,5 @@
 using SteamKit2;
+using SteamKit2.Unified.Internal;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
@@ -40,6 +41,7 @@ namespace DepotDownloader
         public SteamClient steamClient;
         public SteamUser steamUser;
         SteamApps steamApps;
+        SteamUnifiedMessages.UnifiedService<IPublishedFile> steamPublishedFile;
 
         CallbackManager callbacks;
 
@@ -85,6 +87,8 @@ namespace DepotDownloader
 
             this.steamUser = this.steamClient.GetHandler<SteamUser>();
             this.steamApps = this.steamClient.GetHandler<SteamApps>();
+            var steamUnifiedMessages = this.steamClient.GetHandler<SteamUnifiedMessages>();
+            this.steamPublishedFile = steamUnifiedMessages.CreateService<IPublishedFile>();
 
             this.callbacks = new CallbackManager( this.steamClient );
 
@@ -374,6 +378,36 @@ namespace DepotDownloader
             }, () => { return completed; } );
         }
 
+        public PublishedFileDetails GetPubfileDetails( PublishedFileID pubFile )
+        {
+            var pubFileRequest = new CPublishedFile_GetDetails_Request();
+            pubFileRequest.publishedfileids.Add( pubFile );
+
+            bool completed = false;
+            PublishedFileDetails details = null;
+
+            Action<SteamUnifiedMessages.ServiceMethodResponse> cbMethod = callback =>
+            {
+                completed = true;
+                if ( callback.Result == EResult.OK )
+                {
+                    var response = callback.GetDeserializedResponse<CPublishedFile_GetDetails_Response>();
+                    details = response.publishedfiledetails[0];
+                }
+                else
+                {
+                    throw new Exception( $"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC id for pubfile {pubFile}.");
+                }
+            };
+
+            WaitUntilCallback(() =>
+            {
+                callbacks.Subscribe( steamPublishedFile.SendMessage( api => api.GetDetails( pubFileRequest ) ), cbMethod );
+            }, () => { return completed; });
+
+            return details;
+        }
+
         void Connect()
         {
             bAborted = false;