Nb1X/DepotDownloader
1
0 Fork 0
mirror of https://github.com/SteamRE/DepotDownloader.git synced 2026-10-20 01:46:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #474 from SteamRE/cleanup

Browse Source 
Cleanup
This commit is contained in:
Pavel Djundik authored and GitHub committed 2024-02-09 10:07:29 +02:00
2 parents 93318f2b11 0f00ea592e
commit 9d2446404a
8 files changed
+203 -291
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

DepotDownloader/AccountSettingsStore.cs
+8 -12
View File
@@ -29,8 +29,8 @@ namespace DepotDownloader
AccountSettingsStore()
{
ContentServerPenalty = new ConcurrentDictionary<string, int>();
LoginTokens = new Dictionary<string, string>();
GuardData = new Dictionary<string, string>();
LoginTokens = [];
GuardData = [];
}
static bool Loaded
@@ -50,11 +50,9 @@ namespace DepotDownloader
{
try
{
using (var fs = IsolatedStorage.OpenFile(filename, FileMode.Open, FileAccess.Read))
using (var ds = new DeflateStream(fs, CompressionMode.Decompress))
{
Instance = Serializer.Deserialize<AccountSettingsStore>(ds);
}
using var fs = IsolatedStorage.OpenFile(filename, FileMode.Open, FileAccess.Read);
using var ds = new DeflateStream(fs, CompressionMode.Decompress);
Instance = Serializer.Deserialize<AccountSettingsStore>(ds);
}
catch (IOException ex)
{
@@ -77,11 +75,9 @@ namespace DepotDownloader
try
{
using (var fs = IsolatedStorage.OpenFile(Instance.FileName, FileMode.Create, FileAccess.Write))
using (var ds = new DeflateStream(fs, CompressionMode.Compress))
{
Serializer.Serialize(ds, Instance);
}
using var fs = IsolatedStorage.OpenFile(Instance.FileName, FileMode.Create, FileAccess.Write);
using var ds = new DeflateStream(fs, CompressionMode.Compress);
Serializer.Serialize(ds, Instance);
}
catch (IOException ex)
{
DepotDownloader/CDNClientPool.cs
+4 -10
View File
@@ -20,12 +20,12 @@ namespace DepotDownloader
public Client CDNClient { get; }
public Server ProxyServer { get; private set; }
private readonly ConcurrentStack<Server> activeConnectionPool;
private readonly BlockingCollection<Server> availableServerEndpoints;
private readonly ConcurrentStack<Server> activeConnectionPool = [];
private readonly BlockingCollection<Server> availableServerEndpoints = [];
private readonly AutoResetEvent populatePoolEvent;
private readonly AutoResetEvent populatePoolEvent = new(true);
private readonly Task monitorTask;
private readonly CancellationTokenSource shutdownToken;
private readonly CancellationTokenSource shutdownToken = new();
public CancellationTokenSource ExhaustedToken { get; set; }
public CDNClientPool(Steam3Session steamSession, uint appId)
@@ -34,12 +34,6 @@ namespace DepotDownloader
this.appId = appId;
CDNClient = new Client(steamSession.steamClient);
activeConnectionPool = new ConcurrentStack<Server>();
availableServerEndpoints = new BlockingCollection<Server>();
populatePoolEvent = new AutoResetEvent(true);
shutdownToken = new CancellationTokenSource();
monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap();
}
DepotDownloader/ContentDownloader.cs
+136 -179
View File
@@ -11,9 +11,8 @@ using SteamKit2.CDN;
namespace DepotDownloader
{
class ContentDownloaderException : Exception
class ContentDownloaderException(string value) : Exception(value)
{
public ContentDownloaderException(String value) : base(value) { }
}
static class ContentDownloader
@@ -23,38 +22,25 @@ namespace DepotDownloader
public const ulong INVALID_MANIFEST_ID = ulong.MaxValue;
public const string DEFAULT_BRANCH = "public";
public static DownloadConfig Config = new DownloadConfig();
public static DownloadConfig Config = new();
private static Steam3Session steam3;
private static Steam3Session.Credentials steam3Credentials;
private static CDNClientPool cdnPool;
private const string DEFAULT_DOWNLOAD_DIR = "depots";
private const string CONFIG_DIR = ".DepotDownloader";
private static readonly string STAGING_DIR = Path.Combine(CONFIG_DIR, "staging");
private sealed class DepotDownloadInfo
private sealed class DepotDownloadInfo(
uint depotid, uint appId, ulong manifestId, string branch,
string installDir, byte[] depotKey)
{
public uint id { get; private set; }
public uint appId { get; private set; }
public ulong manifestId { get; private set; }
public string branch { get; private set; }
public string installDir { get; private set; }
public byte[] depotKey { get; private set; }
public DepotDownloadInfo(
uint depotid, uint appId, ulong manifestId, string branch,
string installDir, byte[] depotKey)
{
this.id = depotid;
this.appId = appId;
this.manifestId = manifestId;
this.branch = branch;
this.installDir = installDir;
this.depotKey = depotKey;
}
public uint DepotId { get; } = depotid;
public uint AppId { get; } = appId;
public ulong ManifestId { get; } = manifestId;
public string Branch { get; } = branch;
public string InstallDir { get; } = installDir;
public byte[] DepotKey { get; } = depotKey;
}
static bool CreateDirectories(uint depotId, uint depotVersion, out string installDir)
@@ -135,8 +121,7 @@ namespace DepotDownloader
foreach (var license in licenseQuery)
{
SteamApps.PICSProductInfoCallback.PICSProductInfo package;
if (steam3.PackageInfo.TryGetValue(license, out package) && package != null)
if (steam3.PackageInfo.TryGetValue(license, out var package) && package != null)
{
if (package.KeyValues["appids"].Children.Any(child => child.AsUnsignedInteger() == depotId))
return true;
@@ -156,33 +141,20 @@ namespace DepotDownloader
return null;
}
SteamApps.PICSProductInfoCallback.PICSProductInfo app;
if (!steam3.AppInfo.TryGetValue(appId, out app) || app == null)
if (!steam3.AppInfo.TryGetValue(appId, out var app) || app == null)
{
return null;
}
var appinfo = app.KeyValues;
string section_key;
switch (section)
var section_key = section switch
{
case EAppInfoSection.Common:
section_key = "common";
break;
case EAppInfoSection.Extended:
section_key = "extended";
break;
case EAppInfoSection.Config:
section_key = "config";
break;
case EAppInfoSection.Depots:
section_key = "depots";
break;
default:
throw new NotImplementedException();
}
EAppInfoSection.Common => "common",
EAppInfoSection.Extended => "extended",
EAppInfoSection.Config => "config",
EAppInfoSection.Depots => "depots",
_ => throw new NotImplementedException(),
};
var section_kv = appinfo.Children.Where(c => c.Name == section_key).FirstOrDefault();
return section_kv;
}
@@ -262,7 +234,7 @@ namespace DepotDownloader
// Submit the password to Steam now to get encryption keys
steam3.CheckAppBetaPassword(appId, Config.BetaPassword);
if (!steam3.AppBetaPasswords.ContainsKey(branch))
if (!steam3.AppBetaPasswords.TryGetValue(branch, out var appBetaPassword))
{
Console.WriteLine("Password was invalid for branch {0}", branch);
return INVALID_MANIFEST_ID;
@@ -272,7 +244,7 @@ namespace DepotDownloader
byte[] manifest_bytes;
try
{
manifest_bytes = CryptoHelper.SymmetricDecryptECB(input, steam3.AppBetaPasswords[branch]);
manifest_bytes = CryptoHelper.SymmetricDecryptECB(input, appBetaPassword);
}
catch (Exception e)
{
@@ -293,14 +265,14 @@ namespace DepotDownloader
if (node.Value == null)
return INVALID_MANIFEST_ID;
return UInt64.Parse(node.Value);
return ulong.Parse(node.Value);
}
static string GetAppName(uint appId)
{
var info = GetSteam3AppSection(appId, EAppInfoSection.Common);
if (info == null)
return String.Empty;
return string.Empty;
return info["name"].AsString();
}
@@ -325,9 +297,7 @@ namespace DepotDownloader
}
);
steam3Credentials = steam3.WaitForCredentials();
if (!steam3Credentials.IsValid)
if (!steam3.WaitForCredentials())
{
Console.WriteLine("Unable to get steam3 credentials.");
return false;
@@ -393,8 +363,7 @@ namespace DepotDownloader
private static async Task DownloadWebFile(uint appId, string fileName, string url)
{
string installDir;
if (!CreateDirectories(appId, 0, out installDir))
if (!CreateDirectories(appId, 0, out var installDir))
{
Console.WriteLine("Error: Unable to create install directories!");
return;
@@ -437,8 +406,7 @@ namespace DepotDownloader
Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR));
DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config"));
if (steam3 != null)
steam3.RequestAppInfo(appId);
steam3?.RequestAppInfo(appId);
if (!AccountHasAccess(appId))
{
@@ -452,13 +420,13 @@ namespace DepotDownloader
else
{
var contentName = GetAppName(appId);
throw new ContentDownloaderException(String.Format("App {0} ({1}) is not available from this account.", appId, contentName));
throw new ContentDownloaderException(string.Format("App {0} ({1}) is not available from this account.", appId, contentName));
}
}
var hasSpecificDepots = depotManifestIds.Count > 0;
var depotIdsFound = new List<uint>();
var depotIdsExpected = depotManifestIds.Select(x => x.Item1).ToList();
var depotIdsExpected = depotManifestIds.Select(x => x.depotId).ToList();
var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots);
if (isUgc)
@@ -537,21 +505,21 @@ namespace DepotDownloader
if (depotManifestIds.Count == 0 && !hasSpecificDepots)
{
throw new ContentDownloaderException(String.Format("Couldn't find any depots to download for app {0}", appId));
throw new ContentDownloaderException(string.Format("Couldn't find any depots to download for app {0}", appId));
}
if (depotIdsFound.Count < depotIdsExpected.Count)
{
var remainingDepotIds = depotIdsExpected.Except(depotIdsFound);
throw new ContentDownloaderException(String.Format("Depot {0} not listed for app {1}", string.Join(", ", remainingDepotIds), appId));
throw new ContentDownloaderException(string.Format("Depot {0} not listed for app {1}", string.Join(", ", remainingDepotIds), appId));
}
}
var infos = new List<DepotDownloadInfo>();
foreach (var depotManifest in depotManifestIds)
foreach (var (depotId, manifestId) in depotManifestIds)
{
var info = GetDepotInfo(depotManifest.Item1, appId, depotManifest.Item2, branch);
var info = GetDepotInfo(depotId, appId, manifestId, branch);
if (info != null)
{
infos.Add(info);
@@ -560,7 +528,7 @@ namespace DepotDownloader
try
{
await DownloadSteam3Async(appId, infos).ConfigureAwait(false);
await DownloadSteam3Async(infos).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -599,7 +567,7 @@ namespace DepotDownloader
}
steam3.RequestDepotKey(depotId, appId);
if (!steam3.DepotKeys.ContainsKey(depotId))
if (!steam3.DepotKeys.TryGetValue(depotId, out var depotKey))
{
Console.WriteLine("No valid depot key for {0}, unable to download.", depotId);
return null;
@@ -607,28 +575,19 @@ namespace DepotDownloader
var uVersion = GetSteam3AppBuildNumber(appId, branch);
string installDir;
if (!CreateDirectories(depotId, uVersion, out installDir))
if (!CreateDirectories(depotId, uVersion, out var installDir))
{
Console.WriteLine("Error: Unable to create install directories!");
return null;
}
var depotKey = steam3.DepotKeys[depotId];
return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey);
}
private class ChunkMatch
private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
{
public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
{
OldChunk = oldChunk;
NewChunk = newChunk;
}
public ProtoManifest.ChunkData OldChunk { get; private set; }
public ProtoManifest.ChunkData NewChunk { get; private set; }
public ProtoManifest.ChunkData OldChunk { get; } = oldChunk;
public ProtoManifest.ChunkData NewChunk { get; } = newChunk;
}
private class DepotFilesData
@@ -663,19 +622,19 @@ namespace DepotDownloader
public ulong DepotBytesUncompressed;
}
private static async Task DownloadSteam3Async(uint appId, List<DepotDownloadInfo> depots)
private static async Task DownloadSteam3Async(List<DepotDownloadInfo> depots)
{
var cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
var downloadCounter = new GlobalDownloadCounter();
var depotsToDownload = new List<DepotFilesData>(depots.Count);
var allFileNamesAllDepots = new HashSet<String>();
var allFileNamesAllDepots = new HashSet<string>();
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
foreach (var depot in depots)
{
var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot);
var depotFileData = await ProcessDepotManifestAndFiles(cts, depot);
if (depotFileData != null)
{
@@ -690,7 +649,7 @@ namespace DepotDownloader
// This is in last-depot-wins order, from Steam or the list of depots supplied by the user
if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0)
{
var claimedFileNames = new HashSet<String>();
var claimedFileNames = new HashSet<string>();
for (var i = depotsToDownload.Count - 1; i >= 0; i--)
{
@@ -703,38 +662,37 @@ namespace DepotDownloader
foreach (var depotFileData in depotsToDownload)
{
await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots);
await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count);
}
private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts,
uint appId, DepotDownloadInfo depot)
private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot)
{
var depotCounter = new DepotDownloadCounter();
Console.WriteLine("Processing depot {0}", depot.id);
Console.WriteLine("Processing depot {0}", depot.DepotId);
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
var configDir = Path.Combine(depot.installDir, CONFIG_DIR);
var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);
var lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.DepotId, out lastManifestId);
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId));
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId));
if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum, currentChecksum;
byte[] expectedChecksum;
try
{
@@ -745,29 +703,29 @@ namespace DepotDownloader
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum);
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.manifestId)
if (lastManifestId != depot.ManifestId)
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId));
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum, currentChecksum;
byte[] expectedChecksum;
try
{
@@ -778,18 +736,18 @@ namespace DepotDownloader
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum);
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}
if (newProtoManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
@@ -816,39 +774,39 @@ namespace DepotDownloader
if (manifestRequestCode == 0 || now >= manifestRequestCodeExpiration)
{
manifestRequestCode = await steam3.GetDepotManifestRequestCodeAsync(
depot.id,
depot.appId,
depot.manifestId,
depot.branch);
depot.DepotId,
depot.AppId,
depot.ManifestId,
depot.Branch);
// This code will hopefully be valid for one period following the issuing period
manifestRequestCodeExpiration = now.Add(TimeSpan.FromMinutes(5));
// If we could not get the manifest code, this is a fatal error
if (manifestRequestCode == 0)
{
Console.WriteLine("No manifest request code was returned for {0} {1}", depot.id, depot.manifestId);
Console.WriteLine("No manifest request code was returned for {0} {1}", depot.DepotId, depot.ManifestId);
cts.Cancel();
}
}
DebugLog.WriteLine("ContentDownloader",
"Downloading manifest {0} from {1} with {2}",
depot.manifestId,
depot.ManifestId,
connection,
cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(
depot.id,
depot.manifestId,
depot.DepotId,
depot.ManifestId,
manifestRequestCode,
connection,
depot.depotKey,
depot.DepotKey,
cdnPool.ProxyServer).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading depot manifest {0} {1}. Retrying.", depot.id, depot.manifestId);
Console.WriteLine("Connection timeout downloading depot manifest {0} {1}. Retrying.", depot.DepotId, depot.ManifestId);
}
catch (SteamKitWebRequestException e)
{
@@ -856,17 +814,17 @@ namespace DepotDownloader
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.DepotId, depot.ManifestId);
break;
}
if (e.StatusCode == HttpStatusCode.NotFound)
{
Console.WriteLine("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
Console.WriteLine("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.DepotId, depot.ManifestId);
break;
}
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.StatusCode);
}
catch (OperationCanceledException)
{
@@ -875,23 +833,22 @@ namespace DepotDownloader
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
}
} while (depotManifest == null);
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out var checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
Console.WriteLine(" Done!");
@@ -900,7 +857,7 @@ namespace DepotDownloader
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime);
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
if (Config.DownloadManifestOnly)
{
@@ -908,7 +865,7 @@ namespace DepotDownloader
return null;
}
var stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet<string>(filesAfterExclusions.Count);
@@ -918,7 +875,7 @@ namespace DepotDownloader
{
allFileNames.Add(file.FileName);
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
@@ -948,13 +905,13 @@ namespace DepotDownloader
};
}
private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet<String> allFileNamesAllDepots)
private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet<string> allFileNamesAllDepots)
{
var depot = depotFilesData.depotDownloadInfo;
var depotCounter = depotFilesData.depotCounter;
Console.WriteLine("Downloading depot {0}", depot.id);
Console.WriteLine("Downloading depot {0}", depot.DepotId);
var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
@@ -967,7 +924,7 @@ namespace DepotDownloader
await Util.InvokeAsync(
networkChunkQueue.Select(q => new Func<Task>(async () =>
await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData,
await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, downloadCounter, depotFilesData,
q.fileData, q.fileStreamData, q.chunk)))),
maxDegreeOfParallelism: Config.MaxDownloads
);
@@ -991,7 +948,7 @@ namespace DepotDownloader
foreach (var existingFileName in previousFilteredFiles)
{
var fileFinalPath = Path.Combine(depot.installDir, existingFileName);
var fileFinalPath = Path.Combine(depot.InstallDir, existingFileName);
if (!File.Exists(fileFinalPath))
continue;
@@ -1001,10 +958,10 @@ namespace DepotDownloader
}
}
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed);
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed);
}
private static void DownloadSteam3AsyncDepotFile(
@@ -1025,7 +982,7 @@ namespace DepotDownloader
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
@@ -1049,7 +1006,7 @@ namespace DepotDownloader
}
catch (IOException ex)
{
throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
}
neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
@@ -1059,7 +1016,7 @@ namespace DepotDownloader
// open existing
if (oldManifestFile != null)
{
neededChunks = new List<ProtoManifest.ChunkData>();
neededChunks = [];
var hashMatches = oldManifestFile.FileHash.SequenceEqual(file.FileHash);
if (Config.VerifyAll || !hashMatches)
@@ -1123,7 +1080,7 @@ namespace DepotDownloader
}
catch (IOException ex)
{
throw new ContentDownloaderException(String.Format("Failed to resize file to expected size {0}: {1}", fileFinalPath, ex.Message));
throw new ContentDownloaderException(string.Format("Failed to resize file to expected size {0}: {1}", fileFinalPath, ex.Message));
}
foreach (var match in copyChunks)
@@ -1155,12 +1112,12 @@ namespace DepotDownloader
}
catch (IOException ex)
{
throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
}
}
Console.WriteLine("Validating {0}", fileFinalPath);
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
neededChunks = Util.ValidateSteam3FileChecksums(fs, [.. file.Chunks.OrderBy(x => x.Offset)]);
}
if (neededChunks.Count == 0)
@@ -1205,7 +1162,7 @@ namespace DepotDownloader
}
private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts, uint appId,
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
@@ -1219,12 +1176,14 @@ namespace DepotDownloader
var chunkID = Util.EncodeHexString(chunk.ChunkID);
var data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
var data = new DepotManifest.ChunkData
{
ChunkID = chunk.ChunkID,
Checksum = chunk.Checksum,
Offset = chunk.Offset,
CompressedLength = chunk.CompressedLength,
UncompressedLength = chunk.UncompressedLength
};
DepotChunk chunkData = null;
@@ -1240,10 +1199,10 @@ namespace DepotDownloader
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(
depot.id,
depot.DepotId,
data,
connection,
depot.depotKey,
depot.DepotKey,
cdnPool.ProxyServer).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
@@ -1277,7 +1236,7 @@ namespace DepotDownloader
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.DepotId);
cts.Cancel();
}
@@ -1290,12 +1249,12 @@ namespace DepotDownloader
if (fileStreamData.fileStream == null)
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
}
fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
await fileStreamData.fileStream.WriteAsync(chunkData.Data.AsMemory(0, chunkData.Data.Length), cts.Token);
}
finally
{
@@ -1326,51 +1285,49 @@ namespace DepotDownloader
if (remainingChunks == 0)
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
Console.WriteLine("{0,6:#00.00}% {1}", (sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
}
}
static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest)
{
var txtManifest = Path.Combine(depot.installDir, $"manifest_{depot.id}_{depot.manifestId}.txt");
var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt");
using var sw = new StreamWriter(txtManifest);
using (var sw = new StreamWriter(txtManifest))
sw.WriteLine($"Content Manifest for Depot {depot.DepotId}");
sw.WriteLine();
sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime}");
int numFiles = 0, numChunks = 0;
ulong uncompressedSize = 0, compressedSize = 0;
foreach (var file in manifest.Files)
{
sw.WriteLine($"Content Manifest for Depot {depot.id}");
sw.WriteLine();
sw.WriteLine($"Manifest ID / date : {depot.manifestId} / {manifest.CreationTime}");
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
continue;
int numFiles = 0, numChunks = 0;
ulong uncompressedSize = 0, compressedSize = 0;
numFiles++;
numChunks += file.Chunks.Count;
foreach (var file in manifest.Files)
foreach (var chunk in file.Chunks)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
continue;
numFiles++;
numChunks += file.Chunks.Count;
foreach (var chunk in file.Chunks)
{
uncompressedSize += chunk.UncompressedLength;
compressedSize += chunk.CompressedLength;
}
uncompressedSize += chunk.UncompressedLength;
compressedSize += chunk.CompressedLength;
}
}
sw.WriteLine($"Total number of files : {numFiles}");
sw.WriteLine($"Total number of chunks : {numChunks}");
sw.WriteLine($"Total bytes on disk : {uncompressedSize}");
sw.WriteLine($"Total bytes compressed : {compressedSize}");
sw.WriteLine();
sw.WriteLine(" Size Chunks File SHA Flags Name");
sw.WriteLine($"Total number of files : {numFiles}");
sw.WriteLine($"Total number of chunks : {numChunks}");
sw.WriteLine($"Total bytes on disk : {uncompressedSize}");
sw.WriteLine($"Total bytes compressed : {compressedSize}");
sw.WriteLine();
sw.WriteLine(" Size Chunks File SHA Flags Name");
foreach (var file in manifest.Files)
{
var sha1Hash = BitConverter.ToString(file.FileHash).Replace("-", "");
sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}");
}
foreach (var file in manifest.Files)
{
var sha1Hash = BitConverter.ToString(file.FileHash).Replace("-", "");
sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}");
}
}
}
DepotDownloader/DepotConfigStore.cs
+8 -8
View File
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
@@ -16,7 +16,7 @@ namespace DepotDownloader
DepotConfigStore()
{
InstalledManifestIDs = new Dictionary<uint, ulong>();
InstalledManifestIDs = [];
}
static bool Loaded
@@ -33,9 +33,9 @@ namespace DepotDownloader
if (File.Exists(filename))
{
using (var fs = File.Open(filename, FileMode.Open))
using (var ds = new DeflateStream(fs, CompressionMode.Decompress))
Instance = Serializer.Deserialize<DepotConfigStore>(ds);
using var fs = File.Open(filename, FileMode.Open);
using var ds = new DeflateStream(fs, CompressionMode.Decompress);
Instance = Serializer.Deserialize<DepotConfigStore>(ds);
}
else
{
@@ -50,9 +50,9 @@ namespace DepotDownloader
if (!Loaded)
throw new Exception("Saved config before loading");
using (var fs = File.Open(Instance.FileName, FileMode.Create))
using (var ds = new DeflateStream(fs, CompressionMode.Compress))
Serializer.Serialize(ds, Instance);
using var fs = File.Open(Instance.FileName, FileMode.Create);
using var ds = new DeflateStream(fs, CompressionMode.Compress);
Serializer.Serialize(ds, Instance);
}
}
}
DepotDownloader/HttpClientFactory.cs
+5 -3
View File
@@ -1,4 +1,4 @@
using System.IO;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Sockets;
@@ -29,8 +29,10 @@ namespace DepotDownloader
// By default, we create dual-mode sockets:
// Socket socket = new Socket(SocketType.Stream, ProtocolType.Tcp);
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.NoDelay = true;
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)
{
NoDelay = true
};
try
{
DepotDownloader/Program.cs
+12 -8
View File
@@ -16,6 +16,8 @@ namespace DepotDownloader
static int Main(string[] args)
=> MainAsync(args).GetAwaiter().GetResult();
internal static readonly char[] newLineCharacters = ['\n', '\r'];
static async Task<int> MainAsync(string[] args)
{
if (args.Length == 0)
@@ -63,20 +65,22 @@ namespace DepotDownloader
if (fileList != null)
{
const string RegexPrefix = "regex:";
try
{
var fileListData = await File.ReadAllTextAsync(fileList);
var files = fileListData.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
var files = fileListData.Split(newLineCharacters, StringSplitOptions.RemoveEmptyEntries);
ContentDownloader.Config.UsingFileList = true;
ContentDownloader.Config.FilesToDownload = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
ContentDownloader.Config.FilesToDownloadRegex = new List<Regex>();
ContentDownloader.Config.FilesToDownloadRegex = [];
foreach (var fileEntry in files)
{
if (fileEntry.StartsWith("regex:"))
if (fileEntry.StartsWith(RegexPrefix))
{
var rgx = new Regex(fileEntry.Substring(6), RegexOptions.Compiled | RegexOptions.IgnoreCase);
var rgx = new Regex(fileEntry[RegexPrefix.Length..], RegexOptions.Compiled | RegexOptions.IgnoreCase);
ContentDownloader.Config.FilesToDownloadRegex.Add(rgx);
}
else
@@ -192,7 +196,7 @@ namespace DepotDownloader
ContentDownloader.Config.DownloadAllPlatforms = HasParameter(args, "-all-platforms");
var os = GetParameter<string>(args, "-os");
if (ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty(os))
if (ContentDownloader.Config.DownloadAllPlatforms && !string.IsNullOrEmpty(os))
{
Console.WriteLine("Error: Cannot specify -os when -all-platforms is specified.");
return 1;
@@ -203,7 +207,7 @@ namespace DepotDownloader
ContentDownloader.Config.DownloadAllLanguages = HasParameter(args, "-all-languages");
var language = GetParameter<string>(args, "-language");
if (ContentDownloader.Config.DownloadAllLanguages && !String.IsNullOrEmpty(language))
if (ContentDownloader.Config.DownloadAllLanguages && !string.IsNullOrEmpty(language))
{
Console.WriteLine("Error: Cannot specify -language when -all-languages is specified.");
return 1;
@@ -314,7 +318,7 @@ namespace DepotDownloader
return IndexOfParam(args, param) > -1;
}
static T GetParameter<T>(string[] args, string param, T defaultValue = default(T))
static T GetParameter<T>(string[] args, string param, T defaultValue = default)
{
var index = IndexOfParam(args, param);
@@ -329,7 +333,7 @@ namespace DepotDownloader
return (T)converter.ConvertFromString(strParam);
}
return default(T);
return default;
}
static List<T> GetParameterList<T>(string[] args, string param)
DepotDownloader/ProtoManifest.cs
+16 -20
View File
@@ -14,7 +14,7 @@ namespace DepotDownloader
// Proto ctor
private ProtoManifest()
{
Files = new List<FileData>();
Files = [];
}
public ProtoManifest(DepotManifest sourceManifest, ulong id) : this()
@@ -30,7 +30,7 @@ namespace DepotDownloader
// Proto ctor
private FileData()
{
Chunks = new List<ChunkData>();
Chunks = [];
}
public FileData(DepotManifest.FileData sourceData) : this()
@@ -130,33 +130,29 @@ namespace DepotDownloader
return null;
}
using (var ms = new MemoryStream())
{
using (var fs = File.Open(filename, FileMode.Open))
using (var ds = new DeflateStream(fs, CompressionMode.Decompress))
ds.CopyTo(ms);
using var ms = new MemoryStream();
using (var fs = File.Open(filename, FileMode.Open))
using (var ds = new DeflateStream(fs, CompressionMode.Decompress))
ds.CopyTo(ms);
checksum = SHA1.HashData(ms.ToArray());
checksum = SHA1.HashData(ms.ToArray());
ms.Seek(0, SeekOrigin.Begin);
return Serializer.Deserialize<ProtoManifest>(ms);
}
ms.Seek(0, SeekOrigin.Begin);
return Serializer.Deserialize<ProtoManifest>(ms);
}
public void SaveToFile(string filename, out byte[] checksum)
{
using (var ms = new MemoryStream())
{
Serializer.Serialize(ms, this);
using var ms = new MemoryStream();
Serializer.Serialize(ms, this);
checksum = SHA1.HashData(ms.ToArray());
checksum = SHA1.HashData(ms.ToArray());
ms.Seek(0, SeekOrigin.Begin);
ms.Seek(0, SeekOrigin.Begin);
using (var fs = File.Open(filename, FileMode.Create))
using (var ds = new DeflateStream(fs, CompressionMode.Compress))
ms.CopyTo(ds);
}
using var fs = File.Open(filename, FileMode.Create);
using var ds = new DeflateStream(fs, CompressionMode.Compress);
ms.CopyTo(ds);
}
}
}
DepotDownloader/Steam3Session.cs
+14 -51
View File
@@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
@@ -14,16 +13,7 @@ namespace DepotDownloader
{
class Steam3Session
{
public class Credentials
{
public bool LoggedOn { get; set; }
public ulong SessionToken { get; set; }
public bool IsValid
{
get { return LoggedOn; }
}
}
public bool IsLoggedOn { get; private set; }
public ReadOnlyCollection<SteamApps.LicenseListCallback.License> Licenses
{
@@ -31,13 +21,12 @@ namespace DepotDownloader
private set;
}
public Dictionary<uint, ulong> AppTokens { get; private set; }
public Dictionary<uint, ulong> PackageTokens { get; private set; }
public Dictionary<uint, byte[]> DepotKeys { get; private set; }
public ConcurrentDictionary<string, TaskCompletionSource<SteamApps.CDNAuthTokenCallback>> CDNAuthTokens { get; private set; }
public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> AppInfo { get; private set; }
public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> PackageInfo { get; private set; }
public Dictionary<string, byte[]> AppBetaPasswords { get; private set; }
public Dictionary<uint, ulong> AppTokens { get; } = [];
public Dictionary<uint, ulong> PackageTokens { get; } = [];
public Dictionary<uint, byte[]> DepotKeys { get; } = [];
public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> AppInfo { get; } = [];
public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> PackageInfo { get; } = [];
public Dictionary<string, byte[]> AppBetaPasswords { get; } = [];
public SteamClient steamClient;
public SteamUser steamUser;
@@ -63,32 +52,13 @@ namespace DepotDownloader
// input
readonly SteamUser.LogOnDetails logonDetails;
// output
readonly Credentials credentials;
static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds(30);
public Steam3Session(SteamUser.LogOnDetails details)
{
this.logonDetails = details;
this.authenticatedUser = details.Username != null || ContentDownloader.Config.UseQrCode;
this.credentials = new Credentials();
this.bConnected = false;
this.bConnecting = false;
this.bAborted = false;
this.bExpectingDisconnectRemote = false;
this.bDidDisconnect = false;
this.seq = 0;
this.AppTokens = new Dictionary<uint, ulong>();
this.PackageTokens = new Dictionary<uint, ulong>();
this.DepotKeys = new Dictionary<uint, byte[]>();
this.CDNAuthTokens = new ConcurrentDictionary<string, TaskCompletionSource<SteamApps.CDNAuthTokenCallback>>();
this.AppInfo = new Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo>();
this.PackageInfo = new Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo>();
this.AppBetaPasswords = new Dictionary<string, byte[]>();
var clientConfiguration = SteamConfiguration.Create(config =>
config
@@ -109,7 +79,6 @@ namespace DepotDownloader
this.callbacks.Subscribe<SteamClient.ConnectedCallback>(ConnectedCallback);
this.callbacks.Subscribe<SteamClient.DisconnectedCallback>(DisconnectedCallback);
this.callbacks.Subscribe<SteamUser.LoggedOnCallback>(LogOnCallback);
this.callbacks.Subscribe<SteamUser.SessionTokenCallback>(SessionTokenCallback);
this.callbacks.Subscribe<SteamApps.LicenseListCallback>(LicenseListCallback);
Console.Write("Connecting to Steam3...");
@@ -118,7 +87,7 @@ namespace DepotDownloader
public delegate bool WaitCondition();
private readonly object steamLock = new object();
private readonly object steamLock = new();
public bool WaitUntilCallback(Action submitter, WaitCondition waiter)
{
@@ -142,14 +111,14 @@ namespace DepotDownloader
return bAborted;
}
public Credentials WaitForCredentials()
public bool WaitForCredentials()
{
if (credentials.IsValid || bAborted)
return credentials;
if (IsLoggedOn || bAborted)
return IsLoggedOn;
WaitUntilCallback(() => { }, () => { return credentials.IsValid; });
WaitUntilCallback(() => { }, () => IsLoggedOn);
return credentials;
return IsLoggedOn;
}
public void RequestAppInfo(uint appId, bool bForce = false)
@@ -708,7 +677,7 @@ namespace DepotDownloader
Console.WriteLine(" Done!");
this.seq++;
credentials.LoggedOn = true;
IsLoggedOn = true;
if (ContentDownloader.Config.CellID == 0)
{
@@ -717,12 +686,6 @@ namespace DepotDownloader
}
}
private void SessionTokenCallback(SteamUser.SessionTokenCallback sessionToken)
{
Console.WriteLine("Got session token!");
credentials.SessionToken = sessionToken.SessionToken;
}
private void LicenseListCallback(SteamApps.LicenseListCallback licenseList)
{
if (licenseList.Result != EResult.OK)