Use Steam manifest format

DepotDownloader/ContentDownloader.cs

@@ -598,10 +598,10 @@ namespace DepotDownloader
             return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey);
         }
 
-        private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
+        private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
         {
-            public ProtoManifest.ChunkData OldChunk { get; } = oldChunk;
-            public ProtoManifest.ChunkData NewChunk { get; } = newChunk;
+            public DepotManifest.ChunkData OldChunk { get; } = oldChunk;
+            public DepotManifest.ChunkData NewChunk { get; } = newChunk;
         }
 
         private class DepotFilesData
@@ -609,9 +609,9 @@ namespace DepotDownloader
             public DepotDownloadInfo depotDownloadInfo;
             public DepotDownloadCounter depotCounter;
             public string stagingDir;
-            public ProtoManifest manifest;
-            public ProtoManifest previousManifest;
-            public List<ProtoManifest.FileData> filteredFiles;
+            public DepotManifest manifest;
+            public DepotManifest previousManifest;
+            public List<DepotManifest.FileData> filteredFiles;
             public HashSet<string> allFileNames;
         }
 
@@ -694,8 +694,8 @@ namespace DepotDownloader
 
             Console.WriteLine("Processing depot {0}", depot.DepotId);
 
-            ProtoManifest oldProtoManifest = null;
-            ProtoManifest newProtoManifest = null;
+            DepotManifest oldManifest = null;
+            DepotManifest newManifest = null;
             var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);
 
             var lastManifestId = INVALID_MANIFEST_ID;
@@ -707,64 +707,21 @@ namespace DepotDownloader
 
             if (lastManifestId != INVALID_MANIFEST_ID)
             {
-                var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId));
-
-                if (File.Exists(oldManifestFileName))
-                {
-                    byte[] expectedChecksum;
-
-                    try
-                    {
-                        expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
-                    }
-                    catch (IOException)
-                    {
-                        expectedChecksum = null;
-                    }
-
-                    oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum);
-
-                    if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
-                    {
-                        // We only have to show this warning if the old manifest ID was different
-                        if (lastManifestId != depot.ManifestId)
-                            Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
-                        oldProtoManifest = null;
-                    }
-                }
+                // We only have to show this warning if the old manifest ID was different
+                var badHashWarning = (lastManifestId != depot.ManifestId);
+                oldManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, lastManifestId, badHashWarning);
             }
 
-            if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
+            if (lastManifestId == depot.ManifestId && oldManifest != null)
             {
-                newProtoManifest = oldProtoManifest;
+                newManifest = oldManifest;
                 Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
             }
             else
             {
-                var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId));
-                if (newManifestFileName != null)
-                {
-                    byte[] expectedChecksum;
+                newManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, depot.ManifestId, true);
 
-                    try
-                    {
-                        expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
-                    }
-                    catch (IOException)
-                    {
-                        expectedChecksum = null;
-                    }
-
-                    newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum);
-
-                    if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
-                    {
-                        Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
-                        newProtoManifest = null;
-                    }
-                }
-
-                if (newProtoManifest != null)
+                if (newManifest != null)
                 {
                     Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
                 }
@@ -772,7 +729,6 @@ namespace DepotDownloader
                 {
                     Console.Write("Downloading depot manifest... ");
 
-                    DepotManifest depotManifest = null;
                     ulong manifestRequestCode = 0;
                     var manifestRequestCodeExpiration = DateTime.MinValue;
 
@@ -820,7 +776,7 @@ namespace DepotDownloader
                                 depot.ManifestId,
                                 connection,
                                 cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
-                            depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(
+                            newManifest = await cdnPool.CDNClient.DownloadManifestAsync(
                                 depot.DepotId,
                                 depot.ManifestId,
                                 manifestRequestCode,
@@ -872,9 +828,9 @@ namespace DepotDownloader
                             cdnPool.ReturnBrokenConnection(connection);
                             Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
                         }
-                    } while (depotManifest == null);
+                    } while (newManifest == null);
 
-                    if (depotManifest == null)
+                    if (newManifest == null)
                     {
                         Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId);
                         cts.Cancel();
@@ -883,28 +839,22 @@ namespace DepotDownloader
                     // Throw the cancellation exception if requested so that this task is marked failed
                     cts.Token.ThrowIfCancellationRequested();
 
-
-                    newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
-                    newProtoManifest.SaveToFile(newManifestFileName, out var checksum);
-                    File.WriteAllBytes(newManifestFileName + ".sha", checksum);
-
+                    Util.SaveManifestToFile(configDir, newManifest);
                     Console.WriteLine(" Done!");
                 }
             }
 
-            newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
-
-            Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
+            Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime);
 
             if (Config.DownloadManifestOnly)
             {
-                DumpManifestToTextFile(depot, newProtoManifest);
+                DumpManifestToTextFile(depot, newManifest);
                 return null;
             }
 
             var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);
 
-            var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
+            var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
             var allFileNames = new HashSet<string>(filesAfterExclusions.Count);
 
             // Pre-process
@@ -936,8 +886,8 @@ namespace DepotDownloader
                 depotDownloadInfo = depot,
                 depotCounter = depotCounter,
                 stagingDir = stagingDir,
-                manifest = newProtoManifest,
-                previousManifest = oldProtoManifest,
+                manifest = newManifest,
+                previousManifest = oldManifest,
                 filteredFiles = filesAfterExclusions,
                 allFileNames = allFileNames
             };
@@ -952,7 +902,7 @@ namespace DepotDownloader
             Console.WriteLine("Downloading depot {0}", depot.DepotId);
 
             var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
-            var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
+            var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>();
 
             await Util.InvokeAsync(
                 files.Select(file => new Func<Task>(async () =>
@@ -1006,8 +956,8 @@ namespace DepotDownloader
             CancellationTokenSource cts,
             GlobalDownloadCounter downloadCounter,
             DepotFilesData depotFilesData,
-            ProtoManifest.FileData file,
-            ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
+            DepotManifest.FileData file,
+            ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue)
         {
             cts.Token.ThrowIfCancellationRequested();
 
@@ -1015,7 +965,7 @@ namespace DepotDownloader
             var stagingDir = depotFilesData.stagingDir;
             var depotDownloadCounter = depotFilesData.depotCounter;
             var oldProtoManifest = depotFilesData.previousManifest;
-            ProtoManifest.FileData oldManifestFile = null;
+            DepotManifest.FileData oldManifestFile = null;
             if (oldProtoManifest != null)
             {
                 oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
@@ -1030,7 +980,7 @@ namespace DepotDownloader
                 File.Delete(fileStagingPath);
             }
 
-            List<ProtoManifest.ChunkData> neededChunks;
+            List<DepotManifest.ChunkData> neededChunks;
             var fi = new FileInfo(fileFinalPath);
             var fileDidExist = fi.Exists;
             if (!fileDidExist)
@@ -1048,7 +998,7 @@ namespace DepotDownloader
                     throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
                 }
 
-                neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
+                neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
             }
             else
             {
@@ -1092,7 +1042,7 @@ namespace DepotDownloader
                                 fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
 
                                 var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength);
-                                if (!adler.SequenceEqual(match.OldChunk.Checksum))
+                                if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum)))
                                 {
                                     neededChunks.Add(match.NewChunk);
                                 }
@@ -1211,9 +1161,9 @@ namespace DepotDownloader
             CancellationTokenSource cts,
             GlobalDownloadCounter downloadCounter,
             DepotFilesData depotFilesData,
-            ProtoManifest.FileData file,
+            DepotManifest.FileData file,
             FileStreamData fileStreamData,
-            ProtoManifest.ChunkData chunk)
+            DepotManifest.ChunkData chunk)
         {
             cts.Token.ThrowIfCancellationRequested();
 
@@ -1222,17 +1172,8 @@ namespace DepotDownloader
 
             var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant();
 
-            var data = new DepotManifest.ChunkData
-            {
-                ChunkID = chunk.ChunkID,
-                Checksum = BitConverter.ToUInt32(chunk.Checksum),
-                Offset = chunk.Offset,
-                CompressedLength = chunk.CompressedLength,
-                UncompressedLength = chunk.UncompressedLength
-            };
-
             var written = 0;
-            var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)data.UncompressedLength);
+            var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)chunk.UncompressedLength);
 
             try
             {
@@ -1256,7 +1197,7 @@ namespace DepotDownloader
                         DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
                         written = await cdnPool.CDNClient.DownloadDepotChunkAsync(
                             depot.DepotId,
-                            data,
+                            chunk,
                             connection,
                             chunkBuffer,
                             depot.DepotKey,
@@ -1325,7 +1266,7 @@ namespace DepotDownloader
                         fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
                     }
 
-                    fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin);
+                    fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin);
                     await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token);
                 }
                 finally
@@ -1369,44 +1310,55 @@ namespace DepotDownloader
             }
         }
 
-        static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest)
+        class ChunkIdComparer : IEqualityComparer<byte[]>
+        {
+            public bool Equals(byte[] x, byte[] y)
+            {
+                if (ReferenceEquals(x, y)) return true;
+                if (x == null || y == null) return false;
+                return x.SequenceEqual(y);
+            }
+
+            public int GetHashCode(byte[] obj)
+            {
+                ArgumentNullException.ThrowIfNull(obj);
+
+                // ChunkID is SHA-1, so we can just use the first 4 bytes
+                return BitConverter.ToInt32(obj, 0);
+            }
+        }
+
+        static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest)
         {
             var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt");
             using var sw = new StreamWriter(txtManifest);
 
-            sw.WriteLine($"Content Manifest for Depot {depot.DepotId}");
+            sw.WriteLine($"Content Manifest for Depot {depot.DepotId} ");
             sw.WriteLine();
-            sw.WriteLine($"Manifest ID / date     : {depot.ManifestId} / {manifest.CreationTime}");
+            sw.WriteLine($"Manifest ID / date     : {depot.ManifestId} / {manifest.CreationTime} ");
 
-            int numFiles = 0, numChunks = 0;
-            ulong uncompressedSize = 0, compressedSize = 0;
+            var uniqueChunks = new HashSet<byte[]>(new ChunkIdComparer());
 
             foreach (var file in manifest.Files)
             {
-                if (file.Flags.HasFlag(EDepotFileFlag.Directory))
-                    continue;
-
-                numFiles++;
-                numChunks += file.Chunks.Count;
-
                 foreach (var chunk in file.Chunks)
                 {
-                    uncompressedSize += chunk.UncompressedLength;
-                    compressedSize += chunk.CompressedLength;
+                    uniqueChunks.Add(chunk.ChunkID);
                 }
             }
 
-            sw.WriteLine($"Total number of files  : {numFiles}");
-            sw.WriteLine($"Total number of chunks : {numChunks}");
-            sw.WriteLine($"Total bytes on disk    : {uncompressedSize}");
-            sw.WriteLine($"Total bytes compressed : {compressedSize}");
+            sw.WriteLine($"Total number of files  : {manifest.Files.Count} ");
+            sw.WriteLine($"Total number of chunks : {uniqueChunks.Count} ");
+            sw.WriteLine($"Total bytes on disk    : {manifest.TotalUncompressedSize} ");
+            sw.WriteLine($"Total bytes compressed : {manifest.TotalCompressedSize} ");
+            sw.WriteLine();
             sw.WriteLine();
             sw.WriteLine("          Size Chunks File SHA                                 Flags Name");
 
             foreach (var file in manifest.Files)
             {
-                var sha1Hash = Convert.ToHexString(file.FileHash);
-                sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}");
+                var sha1Hash = Convert.ToHexString(file.FileHash).ToLower();
+                sw.WriteLine($"{file.TotalSize,14:d} {file.Chunks.Count,6:d} {sha1Hash} {(int)file.Flags,5:x} {file.FileName}");
             }
         }
     }

DepotDownloader/ProtoManifest.cs

@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
 using System.Security.Cryptography;
+using System.Text;
 using ProtoBuf;
 using SteamKit2;
 
@@ -157,5 +158,39 @@ namespace DepotDownloader
             using var ds = new DeflateStream(fs, CompressionMode.Compress);
             ms.CopyTo(ds);
         }
+
+        public DepotManifest ConvertToSteamManifest(uint depotId)
+        {
+            ulong uncompressedSize = 0, compressedSize = 0;
+            var newManifest = new DepotManifest();
+            newManifest.Files = new List<DepotManifest.FileData>(Files.Count);
+
+            foreach (var file in Files)
+            {
+                var fileNameHash = SHA1.HashData(Encoding.UTF8.GetBytes(file.FileName.Replace('/', '\\').ToLowerInvariant()));
+                var newFile = new DepotManifest.FileData(file.FileName, fileNameHash, file.Flags, file.TotalSize, file.FileHash, null, false, file.Chunks.Count);
+
+                foreach (var chunk in file.Chunks)
+                {
+                    var newChunk = new DepotManifest.ChunkData(chunk.ChunkID, BitConverter.ToUInt32(chunk.Checksum, 0), chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength);
+                    newFile.Chunks.Add(newChunk);
+
+                    uncompressedSize += chunk.UncompressedLength;
+                    compressedSize += chunk.CompressedLength;
+                }
+
+                newManifest.Files.Add(newFile);
+            }
+
+            newManifest.FilenamesEncrypted = false;
+            newManifest.DepotID = depotId;
+            newManifest.ManifestGID = ID;
+            newManifest.CreationTime = CreationTime;
+            newManifest.TotalUncompressedSize = uncompressedSize;
+            newManifest.TotalCompressedSize = compressedSize;
+            newManifest.EncryptedCRC = 0;
+
+            return newManifest;
+        }
     }
 }

DepotDownloader/Util.cs

@@ -9,6 +9,7 @@ using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
+using SteamKit2;
 
 namespace DepotDownloader
 {
@@ -78,16 +79,16 @@ namespace DepotDownloader
         }
 
         // Validate a file against Steam3 Chunk data
-        public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata)
+        public static List<DepotManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata)
         {
-            var neededChunks = new List<ProtoManifest.ChunkData>();
+            var neededChunks = new List<DepotManifest.ChunkData>();
 
             foreach (var data in chunkdata)
             {
                 fs.Seek((long)data.Offset, SeekOrigin.Begin);
 
                 var adler = AdlerHash(fs, (int)data.UncompressedLength);
-                if (!adler.SequenceEqual(data.Checksum))
+                if (!adler.SequenceEqual(BitConverter.GetBytes(data.Checksum)))
                 {
                     neededChunks.Add(data);
                 }
@@ -110,6 +111,100 @@ namespace DepotDownloader
             return BitConverter.GetBytes(a | (b << 16));
         }
 
+        public static byte[] FileSHAHash(string filename)
+        {
+            using (var fs = File.Open(filename, FileMode.Open))
+            using (var sha = SHA1.Create())
+            {
+                var output = sha.ComputeHash(fs);
+
+                return output;
+            }
+        }
+
+        public static DepotManifest LoadManifestFromFile(string directory, uint depotId, ulong manifestId, bool badHashWarning)
+        {
+            // Try loading Steam format manifest first.
+            var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", depotId, manifestId));
+
+            if (File.Exists(filename))
+            {
+                byte[] expectedChecksum;
+
+                try
+                {
+                    expectedChecksum = File.ReadAllBytes(filename + ".sha");
+                }
+                catch (IOException)
+                {
+                    expectedChecksum = null;
+                }
+
+                var currentChecksum = FileSHAHash(filename);
+
+                if (expectedChecksum != null && expectedChecksum.SequenceEqual(currentChecksum))
+                {
+                    return DepotManifest.LoadFromFile(filename);
+                }
+                else if (badHashWarning)
+                {
+                    Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId);
+                }
+            }
+
+            // Try converting legacy manifest format.
+            filename = Path.Combine(directory, string.Format("{0}_{1}.bin", depotId, manifestId));
+
+            if (File.Exists(filename))
+            {
+                byte[] expectedChecksum;
+
+                try
+                {
+                    expectedChecksum = File.ReadAllBytes(filename + ".sha");
+                }
+                catch (IOException)
+                {
+                    expectedChecksum = null;
+                }
+
+                byte[] currentChecksum;
+                var oldManifest = ProtoManifest.LoadFromFile(filename, out currentChecksum);
+
+                if (oldManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
+                {
+                    oldManifest = null;
+
+                    if (badHashWarning)
+                    {
+                        Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId);
+                    }
+                }
+
+                if (oldManifest != null)
+                {
+                    return oldManifest.ConvertToSteamManifest(depotId);
+                }
+            }
+
+            return null;
+        }
+
+        public static bool SaveManifestToFile(string directory, DepotManifest manifest)
+        {
+            try
+            {
+                var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", manifest.DepotID, manifest.ManifestGID));
+                manifest.SaveToFile(filename);
+                File.WriteAllBytes(filename + ".sha", FileSHAHash(filename));
+                return true; // If serialization completes without throwing an exception, return true
+            }
+            catch (Exception)
+            {
+                return false; // Return false if an error occurs
+            }
+        }
+
         public static byte[] DecodeHexString(string hex)
         {
             if (hex == null)