mirror of
https://github.com/SteamRE/DepotDownloader.git
synced 2026-10-20 21:35:51 +00:00
Compare commits
90
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6396afe2e3 | ||
|
|
16a95c1bfd | ||
|
|
989f37b1fb | ||
|
|
8275f16d3e | ||
|
|
e7474cb9ee | ||
|
|
c124842e86 | ||
|
|
b1557786b2 | ||
|
|
ae192f6448 | ||
|
|
48b0f51c53 | ||
|
|
1e8e20c72c | ||
|
|
c5fc766986 | ||
|
|
d178832b38 | ||
|
|
b2b7e975ad | ||
|
|
91d623419a | ||
|
|
282b9eae81 | ||
|
|
66c63c6dad | ||
|
|
3dc34b50bd | ||
|
|
4aac37401c | ||
|
|
58794cbbb3 | ||
|
|
81cf3f6b3d | ||
|
|
4d56522e44 | ||
|
|
c220fc5b88 | ||
|
|
f5318b9ed6 | ||
|
|
370b800967 | ||
|
|
027b4367fc | ||
|
|
43a31b2713 | ||
|
|
68089f6e7c | ||
|
|
e8e3d1bbfc | ||
|
|
f5c7f2d428 | ||
|
|
5baa3f7625 | ||
|
|
8f8d0c9adf | ||
|
|
b8ca1d44f1 | ||
|
|
007857c13f | ||
|
|
77f9067eaa | ||
|
|
9a2bafeb05 | ||
|
|
748fe11132 | ||
|
|
97653ac481 | ||
|
|
bfeeeb5b40 | ||
|
|
d5afc41b97 | ||
|
|
3d8b7dd696 | ||
|
|
560e48a8b5 | ||
|
|
54b0af377b | ||
|
|
5164a9f963 | ||
|
|
2d2113a8e9 | ||
|
|
5d6950afd9 | ||
|
|
b042b055c4 | ||
|
|
02a525acdb | ||
|
|
c553ef4d60 | ||
|
|
9e203e0d7c | ||
|
|
76038d83e8 | ||
|
|
c21489d304 | ||
|
|
0d37091adf | ||
|
|
0d66cf09ac | ||
|
|
8d875579c5 | ||
|
|
ff9c709787 | ||
|
|
bce88e4d32 | ||
|
|
08542bd09f | ||
|
|
665f83983b | ||
|
|
272f5b646a | ||
|
|
f078581947 | ||
|
|
4896ac0788 | ||
|
|
401d086191 | ||
|
|
8d8c6fc59a | ||
|
|
19df5910c3 | ||
|
|
1e72a47846 | ||
|
|
2c07abb015 | ||
|
|
2682d44684 | ||
|
|
be3682cd4b | ||
|
|
644e3f1ebc | ||
|
|
56822a831f | ||
|
|
001f5303a7 | ||
|
|
0150b7eff4 | ||
|
|
14c6a6dafa | ||
|
|
0617974ac0 | ||
|
|
3249d284ba | ||
|
|
7b3cf4c6c7 | ||
|
|
b6fc6d5c6f | ||
|
|
4dbe7ede14 | ||
|
|
24d7f0b02a | ||
|
|
3bd2cf52ba | ||
|
|
f660b95d3f | ||
|
|
17d3996588 | ||
|
|
5d0c4fb050 | ||
|
|
d087678628 | ||
|
|
deac83600c | ||
|
|
f567a63d84 | ||
|
|
e9a5abecb7 | ||
|
|
5fe8a827d8 | ||
|
|
682bfbac27 | ||
|
|
1c97b42124 |
No files matched your search
+12
-12
@@ -25,16 +25,16 @@ jobs:
|
||||
env:
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Setup .NET Core
|
||||
uses: actions/setup-dotnet@v4
|
||||
uses: actions/setup-dotnet@v6
|
||||
|
||||
- name: Build
|
||||
run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts /p:ContinuousIntegrationBuild=true
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'windows-latest'
|
||||
with:
|
||||
name: DepotDownloader-framework
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
run: dotnet publish DepotDownloader/DepotDownloader.csproj --configuration Release -p:PublishSingleFile=true -p:DebugType=embedded --self-contained --runtime osx-arm64 --output selfcontained-osx-arm64
|
||||
|
||||
- name: Upload Windows-x64
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'windows-latest'
|
||||
with:
|
||||
name: DepotDownloader-windows-x64
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload Windows-arm64
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'windows-latest'
|
||||
with:
|
||||
name: DepotDownloader-windows-arm64
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload Linux-x64
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'ubuntu-latest'
|
||||
with:
|
||||
name: DepotDownloader-linux-x64
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload Linux-arm
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'ubuntu-latest'
|
||||
with:
|
||||
name: DepotDownloader-linux-arm
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload Linux-arm64
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'ubuntu-latest'
|
||||
with:
|
||||
name: DepotDownloader-linux-arm64
|
||||
@@ -110,7 +110,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload macOS-x64
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'macos-latest'
|
||||
with:
|
||||
name: DepotDownloader-macos-x64
|
||||
@@ -118,7 +118,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload macOS-arm64
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release' && matrix.runs-on == 'macos-14'
|
||||
with:
|
||||
name: DepotDownloader-macos-arm64
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
working-directory: release
|
||||
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
draft: true
|
||||
files: release/*
|
||||
|
||||
@@ -17,10 +17,10 @@ jobs:
|
||||
env:
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Setup .NET Core
|
||||
uses: actions/setup-dotnet@v4
|
||||
uses: actions/setup-dotnet@v6
|
||||
|
||||
- name: Configure NuGet
|
||||
run: |
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts /p:ContinuousIntegrationBuild=true
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: matrix.configuration == 'Release'
|
||||
with:
|
||||
name: DepotDownloader-${{ matrix.runs-on }}
|
||||
|
||||
@@ -14,7 +14,11 @@ jobs:
|
||||
run: |
|
||||
$wingetPackage = "SteamRE.DepotDownloader"
|
||||
|
||||
$github = Invoke-RestMethod -uri "https://api.github.com/repos/SteamRE/DepotDownloader/releases"
|
||||
$headers = @{
|
||||
Authorization = "Bearer ${{ secrets.GITHUB_TOKEN }}"
|
||||
}
|
||||
|
||||
$github = Invoke-RestMethod -uri "https://api.github.com/repos/SteamRE/DepotDownloader/releases" -Headers $headers
|
||||
|
||||
$targetRelease = $github | Where-Object -Property name -match '^DepotDownloader' | Select -First 1
|
||||
$assets = $targetRelease | Select -ExpandProperty assets -First 1
|
||||
|
||||
+3
-2
@@ -60,7 +60,7 @@ _ReSharper*
|
||||
*.ncrunch*
|
||||
.*crunch*.local.xml
|
||||
|
||||
# Installshield output folder
|
||||
# Installshield output folder
|
||||
[Ee]xpress
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
@@ -117,4 +117,5 @@ protobuf/
|
||||
cryptopp/
|
||||
|
||||
# misc
|
||||
Thumbs.db
|
||||
Thumbs.db
|
||||
launchSettings.json
|
||||
@@ -32,8 +32,8 @@ namespace DepotDownloader
|
||||
AccountSettingsStore()
|
||||
{
|
||||
ContentServerPenalty = new ConcurrentDictionary<string, int>();
|
||||
LoginTokens = [];
|
||||
GuardData = [];
|
||||
LoginTokens = new(StringComparer.OrdinalIgnoreCase);
|
||||
GuardData = new(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
static bool Loaded
|
||||
|
||||
@@ -2,10 +2,8 @@
|
||||
// in file 'LICENSE', which is part of this source code package.
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SteamKit2.CDN;
|
||||
|
||||
@@ -16,139 +14,80 @@ namespace DepotDownloader
|
||||
/// </summary>
|
||||
class CDNClientPool
|
||||
{
|
||||
private const int ServerEndpointMinimumSize = 8;
|
||||
|
||||
private readonly Steam3Session steamSession;
|
||||
private readonly uint appId;
|
||||
public Client CDNClient { get; }
|
||||
public Server ProxyServer { get; private set; }
|
||||
|
||||
private readonly ConcurrentStack<Server> activeConnectionPool = [];
|
||||
private readonly BlockingCollection<Server> availableServerEndpoints = [];
|
||||
|
||||
private readonly AutoResetEvent populatePoolEvent = new(true);
|
||||
private readonly Task monitorTask;
|
||||
private readonly CancellationTokenSource shutdownToken = new();
|
||||
public CancellationTokenSource ExhaustedToken { get; set; }
|
||||
private readonly List<Server> servers = [];
|
||||
private int nextServer;
|
||||
|
||||
public CDNClientPool(Steam3Session steamSession, uint appId)
|
||||
{
|
||||
this.steamSession = steamSession;
|
||||
this.appId = appId;
|
||||
CDNClient = new Client(steamSession.steamClient);
|
||||
|
||||
monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap();
|
||||
}
|
||||
|
||||
public void Shutdown()
|
||||
public async Task UpdateServerList()
|
||||
{
|
||||
shutdownToken.Cancel();
|
||||
monitorTask.Wait();
|
||||
}
|
||||
var servers = await this.steamSession.steamContent.GetServersForSteamPipe();
|
||||
|
||||
private async Task<IReadOnlyCollection<Server>> FetchBootstrapServerListAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var cdnServers = await this.steamSession.steamContent.GetServersForSteamPipe();
|
||||
if (cdnServers != null)
|
||||
ProxyServer = servers.Where(x => x.UseAsProxy).FirstOrDefault();
|
||||
|
||||
var weightedCdnServers = servers
|
||||
.Where(server =>
|
||||
{
|
||||
return cdnServers;
|
||||
var isEligibleForApp = server.AllowedAppIds.Length == 0 || server.AllowedAppIds.Contains(appId);
|
||||
return isEligibleForApp && (server.Type == "SteamCache" || server.Type == "CDN");
|
||||
})
|
||||
.Select(server =>
|
||||
{
|
||||
AccountSettingsStore.Instance.ContentServerPenalty.TryGetValue(server.Host, out var penalty);
|
||||
|
||||
return (server, penalty);
|
||||
})
|
||||
.OrderBy(pair => pair.penalty).ThenBy(pair => pair.server.WeightedLoad);
|
||||
|
||||
foreach (var (server, weight) in weightedCdnServers)
|
||||
{
|
||||
for (var i = 0; i < server.NumEntries; i++)
|
||||
{
|
||||
this.servers.Add(server);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
if (this.servers.Count == 0)
|
||||
{
|
||||
Console.WriteLine("Failed to retrieve content server list: {0}", ex.Message);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task ConnectionPoolMonitorAsync()
|
||||
{
|
||||
var didPopulate = false;
|
||||
|
||||
while (!shutdownToken.IsCancellationRequested)
|
||||
{
|
||||
populatePoolEvent.WaitOne(TimeSpan.FromSeconds(1));
|
||||
|
||||
// We want the Steam session so we can take the CellID from the session and pass it through to the ContentServer Directory Service
|
||||
if (availableServerEndpoints.Count < ServerEndpointMinimumSize && steamSession.steamClient.IsConnected)
|
||||
{
|
||||
var servers = await FetchBootstrapServerListAsync().ConfigureAwait(false);
|
||||
|
||||
if (servers == null || servers.Count == 0)
|
||||
{
|
||||
ExhaustedToken?.Cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
ProxyServer = servers.Where(x => x.UseAsProxy).FirstOrDefault();
|
||||
|
||||
var weightedCdnServers = servers
|
||||
.Where(server =>
|
||||
{
|
||||
var isEligibleForApp = server.AllowedAppIds.Length == 0 || server.AllowedAppIds.Contains(appId);
|
||||
return isEligibleForApp && (server.Type == "SteamCache" || server.Type == "CDN");
|
||||
})
|
||||
.Select(server =>
|
||||
{
|
||||
AccountSettingsStore.Instance.ContentServerPenalty.TryGetValue(server.Host, out var penalty);
|
||||
|
||||
return (server, penalty);
|
||||
})
|
||||
.OrderBy(pair => pair.penalty).ThenBy(pair => pair.server.WeightedLoad);
|
||||
|
||||
foreach (var (server, weight) in weightedCdnServers)
|
||||
{
|
||||
for (var i = 0; i < server.NumEntries; i++)
|
||||
{
|
||||
availableServerEndpoints.Add(server);
|
||||
}
|
||||
}
|
||||
|
||||
didPopulate = true;
|
||||
}
|
||||
else if (availableServerEndpoints.Count == 0 && !steamSession.steamClient.IsConnected && didPopulate)
|
||||
{
|
||||
ExhaustedToken?.Cancel();
|
||||
return;
|
||||
}
|
||||
throw new Exception("Failed to retrieve any download servers.");
|
||||
}
|
||||
}
|
||||
|
||||
private Server BuildConnection(CancellationToken token)
|
||||
public Server GetConnection()
|
||||
{
|
||||
if (availableServerEndpoints.Count < ServerEndpointMinimumSize)
|
||||
{
|
||||
populatePoolEvent.Set();
|
||||
}
|
||||
|
||||
return availableServerEndpoints.Take(token);
|
||||
}
|
||||
|
||||
public Server GetConnection(CancellationToken token)
|
||||
{
|
||||
if (!activeConnectionPool.TryPop(out var connection))
|
||||
{
|
||||
connection = BuildConnection(token);
|
||||
}
|
||||
|
||||
return connection;
|
||||
return servers[nextServer % servers.Count];
|
||||
}
|
||||
|
||||
public void ReturnConnection(Server server)
|
||||
{
|
||||
if (server == null) return;
|
||||
|
||||
activeConnectionPool.Push(server);
|
||||
// nothing to do, maybe remove from ContentServerPenalty?
|
||||
}
|
||||
|
||||
public void ReturnBrokenConnection(Server server)
|
||||
{
|
||||
if (server == null) return;
|
||||
|
||||
// Broken connections are not returned to the pool
|
||||
lock (servers)
|
||||
{
|
||||
if (servers[nextServer % servers.Count] == server)
|
||||
{
|
||||
nextServer++;
|
||||
|
||||
// TODO: Add server to ContentServerPenalty
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// This file is subject to the terms and conditions defined
|
||||
// in file 'LICENSE', which is part of this source code package.
|
||||
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using SteamKit2.Authentication;
|
||||
|
||||
namespace DepotDownloader
|
||||
{
|
||||
// This is practically copied from https://github.com/SteamRE/SteamKit/blob/master/SteamKit2/SteamKit2/Steam/Authentication/UserConsoleAuthenticator.cs
|
||||
internal class ConsoleAuthenticator : IAuthenticator
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<string> GetDeviceCodeAsync(bool previousCodeWasIncorrect)
|
||||
{
|
||||
if (previousCodeWasIncorrect)
|
||||
{
|
||||
Console.Error.WriteLine("The previous 2-factor auth code you have provided is incorrect.");
|
||||
}
|
||||
|
||||
string code;
|
||||
|
||||
do
|
||||
{
|
||||
Console.Error.Write("STEAM GUARD! Please enter your 2-factor auth code from your authenticator app: ");
|
||||
code = Console.ReadLine()?.Trim();
|
||||
|
||||
if (code == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
while (string.IsNullOrEmpty(code));
|
||||
|
||||
return Task.FromResult(code!);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<string> GetEmailCodeAsync(string email, bool previousCodeWasIncorrect)
|
||||
{
|
||||
if (previousCodeWasIncorrect)
|
||||
{
|
||||
Console.Error.WriteLine("The previous 2-factor auth code you have provided is incorrect.");
|
||||
}
|
||||
|
||||
string code;
|
||||
|
||||
do
|
||||
{
|
||||
Console.Error.Write($"STEAM GUARD! Please enter the auth code sent to the email at {email}: ");
|
||||
code = Console.ReadLine()?.Trim();
|
||||
|
||||
if (code == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
while (string.IsNullOrEmpty(code));
|
||||
|
||||
return Task.FromResult(code!);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<bool> AcceptDeviceConfirmationAsync()
|
||||
{
|
||||
if (ContentDownloader.Config.SkipAppConfirmation)
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
Console.Error.WriteLine("STEAM GUARD! Use the Steam Mobile App to confirm your sign in...");
|
||||
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
@@ -35,6 +36,16 @@ namespace DepotDownloader
|
||||
private const string CONFIG_DIR = ".DepotDownloader";
|
||||
private static readonly string STAGING_DIR = Path.Combine(CONFIG_DIR, "staging");
|
||||
|
||||
private static readonly FrozenSet<EWorkshopFileType> SupportedWorkshopFileTypes = FrozenSet.ToFrozenSet(new[]
|
||||
{
|
||||
EWorkshopFileType.Community,
|
||||
EWorkshopFileType.Art,
|
||||
EWorkshopFileType.Screenshot,
|
||||
EWorkshopFileType.Merch,
|
||||
EWorkshopFileType.IntegratedGuide,
|
||||
EWorkshopFileType.ControllerBinding,
|
||||
});
|
||||
|
||||
private sealed class DepotDownloadInfo(
|
||||
uint depotid, uint appId, ulong manifestId, string branch,
|
||||
string installDir, byte[] depotKey)
|
||||
@@ -189,6 +200,20 @@ namespace DepotDownloader
|
||||
return uint.Parse(buildid.Value);
|
||||
}
|
||||
|
||||
static uint GetSteam3DepotProxyAppId(uint depotId, uint appId)
|
||||
{
|
||||
var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots);
|
||||
var depotChild = depots[depotId.ToString()];
|
||||
|
||||
if (depotChild == KeyValue.Invalid)
|
||||
return INVALID_APP_ID;
|
||||
|
||||
if (depotChild["depotfromapp"] == KeyValue.Invalid)
|
||||
return INVALID_APP_ID;
|
||||
|
||||
return depotChild["depotfromapp"].AsUnsignedInteger();
|
||||
}
|
||||
|
||||
static async Task<ulong> GetSteam3DepotManifest(uint depotId, uint appId, string branch)
|
||||
{
|
||||
var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots);
|
||||
@@ -217,60 +242,56 @@ namespace DepotDownloader
|
||||
}
|
||||
|
||||
var manifests = depotChild["manifests"];
|
||||
var manifests_encrypted = depotChild["encryptedmanifests"];
|
||||
|
||||
if (manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0)
|
||||
if (manifests.Children.Count == 0)
|
||||
return INVALID_MANIFEST_ID;
|
||||
|
||||
var node = manifests[branch]["gid"];
|
||||
|
||||
if (node == KeyValue.Invalid && !string.Equals(branch, DEFAULT_BRANCH, StringComparison.OrdinalIgnoreCase))
|
||||
// Non passworded branch, found the manifest
|
||||
if (node.Value != null)
|
||||
return ulong.Parse(node.Value);
|
||||
|
||||
// If we requested public branch and it had no manifest, nothing to do
|
||||
if (string.Equals(branch, DEFAULT_BRANCH, StringComparison.OrdinalIgnoreCase))
|
||||
return INVALID_MANIFEST_ID;
|
||||
|
||||
// Either the branch just doesn't exist, or it has a password
|
||||
if (string.IsNullOrEmpty(Config.BetaPassword))
|
||||
{
|
||||
var node_encrypted = manifests_encrypted[branch];
|
||||
if (node_encrypted != KeyValue.Invalid)
|
||||
{
|
||||
var password = Config.BetaPassword;
|
||||
while (string.IsNullOrEmpty(password))
|
||||
{
|
||||
Console.Write("Please enter the password for branch {0}: ", branch);
|
||||
Config.BetaPassword = password = Console.ReadLine();
|
||||
}
|
||||
|
||||
var encrypted_gid = node_encrypted["gid"];
|
||||
|
||||
if (encrypted_gid != KeyValue.Invalid)
|
||||
{
|
||||
// Submit the password to Steam now to get encryption keys
|
||||
await steam3.CheckAppBetaPassword(appId, Config.BetaPassword);
|
||||
|
||||
if (!steam3.AppBetaPasswords.TryGetValue(branch, out var appBetaPassword))
|
||||
{
|
||||
Console.WriteLine("Password was invalid for branch {0}", branch);
|
||||
return INVALID_MANIFEST_ID;
|
||||
}
|
||||
|
||||
var input = Util.DecodeHexString(encrypted_gid.Value);
|
||||
byte[] manifest_bytes;
|
||||
try
|
||||
{
|
||||
manifest_bytes = Util.SymmetricDecryptECB(input, appBetaPassword);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("Failed to decrypt branch {0}: {1}", branch, e.Message);
|
||||
return INVALID_MANIFEST_ID;
|
||||
}
|
||||
|
||||
return BitConverter.ToUInt64(manifest_bytes, 0);
|
||||
}
|
||||
|
||||
Console.WriteLine("Unhandled depot encryption for depotId {0}", depotId);
|
||||
return INVALID_MANIFEST_ID;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Branch {branch} for depot {depotId} was not found, either it does not exist or it has a password.");
|
||||
return INVALID_MANIFEST_ID;
|
||||
}
|
||||
|
||||
if (!steam3.AppBetaPasswords.ContainsKey(branch))
|
||||
{
|
||||
// Submit the password to Steam now to get encryption keys
|
||||
await steam3.CheckAppBetaPassword(appId, Config.BetaPassword);
|
||||
|
||||
if (!steam3.AppBetaPasswords.ContainsKey(branch))
|
||||
{
|
||||
Console.WriteLine($"Error: Password was invalid for branch {branch} (or the branch does not exist)");
|
||||
return INVALID_MANIFEST_ID;
|
||||
}
|
||||
}
|
||||
|
||||
// Got the password, request private depot section
|
||||
// TODO: We're probably repeating this request for every depot?
|
||||
var privateDepotSection = await steam3.GetPrivateBetaDepotSection(appId, branch);
|
||||
|
||||
// Now repeat the same code to get the manifest gid from depot section
|
||||
depotChild = privateDepotSection[depotId.ToString()];
|
||||
|
||||
if (depotChild == KeyValue.Invalid)
|
||||
return INVALID_MANIFEST_ID;
|
||||
|
||||
manifests = depotChild["manifests"];
|
||||
|
||||
if (manifests.Children.Count == 0)
|
||||
return INVALID_MANIFEST_ID;
|
||||
|
||||
node = manifests[branch]["gid"];
|
||||
|
||||
if (node.Value == null)
|
||||
return INVALID_MANIFEST_ID;
|
||||
|
||||
@@ -319,33 +340,60 @@ namespace DepotDownloader
|
||||
|
||||
public static void ShutdownSteam3()
|
||||
{
|
||||
if (cdnPool != null)
|
||||
{
|
||||
cdnPool.Shutdown();
|
||||
cdnPool = null;
|
||||
}
|
||||
|
||||
if (steam3 == null)
|
||||
return;
|
||||
|
||||
steam3.Disconnect();
|
||||
}
|
||||
|
||||
public static async Task DownloadPubfileAsync(uint appId, ulong publishedFileId)
|
||||
private static async Task ProcessPublishedFileAsync(uint appId, ulong publishedFileId, List<ValueTuple<string, string>> fileUrls, List<ulong> contentFileIds)
|
||||
{
|
||||
var details = await steam3.GetPublishedFileDetails(appId, publishedFileId);
|
||||
var fileType = (EWorkshopFileType)details.file_type;
|
||||
|
||||
if (!string.IsNullOrEmpty(details?.file_url))
|
||||
if (fileType == EWorkshopFileType.Collection)
|
||||
{
|
||||
await DownloadWebFile(appId, details.filename, details.file_url);
|
||||
foreach (var child in details.children)
|
||||
{
|
||||
await ProcessPublishedFileAsync(appId, child.publishedfileid, fileUrls, contentFileIds);
|
||||
}
|
||||
}
|
||||
else if (details?.hcontent_file > 0)
|
||||
else if (SupportedWorkshopFileTypes.Contains(fileType))
|
||||
{
|
||||
await DownloadAppAsync(appId, new List<(uint, ulong)> { (appId, details.hcontent_file) }, DEFAULT_BRANCH, null, null, null, false, true);
|
||||
if (!string.IsNullOrEmpty(details?.file_url))
|
||||
{
|
||||
fileUrls.Add((details.filename, details.file_url));
|
||||
}
|
||||
else if (details?.hcontent_file > 0)
|
||||
{
|
||||
contentFileIds.Add(details.hcontent_file);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("Unable to locate manifest ID for published file {0}", publishedFileId);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("Unable to locate manifest ID for published file {0}", publishedFileId);
|
||||
Console.WriteLine("Published file {0} has unsupported file type {1}. Skipping file", publishedFileId, fileType);
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task DownloadPubfileAsync(uint appId, ulong publishedFileId)
|
||||
{
|
||||
List<ValueTuple<string, string>> fileUrls = new();
|
||||
List<ulong> contentFileIds = new();
|
||||
|
||||
await ProcessPublishedFileAsync(appId, publishedFileId, fileUrls, contentFileIds);
|
||||
|
||||
foreach (var item in fileUrls)
|
||||
{
|
||||
await DownloadWebFile(appId, item.Item1, item.Item2);
|
||||
}
|
||||
|
||||
if (contentFileIds.Count > 0)
|
||||
{
|
||||
var depotManifestIds = contentFileIds.Select(id => (appId, id)).ToList();
|
||||
await DownloadAppAsync(appId, depotManifestIds, DEFAULT_BRANCH, null, null, null, false, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -421,7 +469,7 @@ namespace DepotDownloader
|
||||
|
||||
if (!await AccountHasAccess(appId, appId))
|
||||
{
|
||||
if (await steam3.RequestFreeAppLicense(appId))
|
||||
if (steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser && await steam3.RequestFreeAppLicense(appId))
|
||||
{
|
||||
Console.WriteLine("Obtained FreeOnDemand license for app {0}", appId);
|
||||
|
||||
@@ -538,6 +586,8 @@ namespace DepotDownloader
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
|
||||
try
|
||||
{
|
||||
await DownloadSteam3Async(infos).ConfigureAwait(false);
|
||||
@@ -595,13 +645,25 @@ namespace DepotDownloader
|
||||
return null;
|
||||
}
|
||||
|
||||
return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey);
|
||||
// For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload
|
||||
var containingAppId = appId;
|
||||
var proxyAppId = GetSteam3DepotProxyAppId(depotId, appId);
|
||||
if (proxyAppId != INVALID_APP_ID)
|
||||
{
|
||||
var common = GetSteam3AppSection(appId, EAppInfoSection.Common);
|
||||
if (common == null || !common["FreeToDownload"].AsBoolean())
|
||||
{
|
||||
containingAppId = proxyAppId;
|
||||
}
|
||||
}
|
||||
|
||||
return new DepotDownloadInfo(depotId, containingAppId, manifestId, branch, installDir, depotKey);
|
||||
}
|
||||
|
||||
private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
|
||||
private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
|
||||
{
|
||||
public ProtoManifest.ChunkData OldChunk { get; } = oldChunk;
|
||||
public ProtoManifest.ChunkData NewChunk { get; } = newChunk;
|
||||
public DepotManifest.ChunkData OldChunk { get; } = oldChunk;
|
||||
public DepotManifest.ChunkData NewChunk { get; } = newChunk;
|
||||
}
|
||||
|
||||
private class DepotFilesData
|
||||
@@ -609,9 +671,9 @@ namespace DepotDownloader
|
||||
public DepotDownloadInfo depotDownloadInfo;
|
||||
public DepotDownloadCounter depotCounter;
|
||||
public string stagingDir;
|
||||
public ProtoManifest manifest;
|
||||
public ProtoManifest previousManifest;
|
||||
public List<ProtoManifest.FileData> filteredFiles;
|
||||
public DepotManifest manifest;
|
||||
public DepotManifest previousManifest;
|
||||
public List<DepotManifest.FileData> filteredFiles;
|
||||
public HashSet<string> allFileNames;
|
||||
}
|
||||
|
||||
@@ -641,9 +703,9 @@ namespace DepotDownloader
|
||||
{
|
||||
Ansi.Progress(Ansi.ProgressState.Indeterminate);
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
cdnPool.ExhaustedToken = cts;
|
||||
await cdnPool.UpdateServerList();
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var downloadCounter = new GlobalDownloadCounter();
|
||||
var depotsToDownload = new List<DepotFilesData>(depots.Count);
|
||||
var allFileNamesAllDepots = new HashSet<string>();
|
||||
@@ -694,8 +756,8 @@ namespace DepotDownloader
|
||||
|
||||
Console.WriteLine("Processing depot {0}", depot.DepotId);
|
||||
|
||||
ProtoManifest oldProtoManifest = null;
|
||||
ProtoManifest newProtoManifest = null;
|
||||
DepotManifest oldManifest = null;
|
||||
DepotManifest newManifest = null;
|
||||
var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);
|
||||
|
||||
var lastManifestId = INVALID_MANIFEST_ID;
|
||||
@@ -707,72 +769,28 @@ namespace DepotDownloader
|
||||
|
||||
if (lastManifestId != INVALID_MANIFEST_ID)
|
||||
{
|
||||
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId));
|
||||
|
||||
if (File.Exists(oldManifestFileName))
|
||||
{
|
||||
byte[] expectedChecksum;
|
||||
|
||||
try
|
||||
{
|
||||
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
expectedChecksum = null;
|
||||
}
|
||||
|
||||
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum);
|
||||
|
||||
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
|
||||
{
|
||||
// We only have to show this warning if the old manifest ID was different
|
||||
if (lastManifestId != depot.ManifestId)
|
||||
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
|
||||
oldProtoManifest = null;
|
||||
}
|
||||
}
|
||||
// We only have to show this warning if the old manifest ID was different
|
||||
var badHashWarning = (lastManifestId != depot.ManifestId);
|
||||
oldManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, lastManifestId, badHashWarning);
|
||||
}
|
||||
|
||||
if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
|
||||
if (lastManifestId == depot.ManifestId && oldManifest != null)
|
||||
{
|
||||
newProtoManifest = oldProtoManifest;
|
||||
newManifest = oldManifest;
|
||||
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
|
||||
}
|
||||
else
|
||||
{
|
||||
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId));
|
||||
if (newManifestFileName != null)
|
||||
{
|
||||
byte[] expectedChecksum;
|
||||
newManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, depot.ManifestId, true);
|
||||
|
||||
try
|
||||
{
|
||||
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
expectedChecksum = null;
|
||||
}
|
||||
|
||||
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum);
|
||||
|
||||
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
|
||||
{
|
||||
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
|
||||
newProtoManifest = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (newProtoManifest != null)
|
||||
if (newManifest != null)
|
||||
{
|
||||
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.Write("Downloading depot manifest... ");
|
||||
Console.WriteLine($"Downloading depot {depot.DepotId} manifest");
|
||||
|
||||
DepotManifest depotManifest = null;
|
||||
ulong manifestRequestCode = 0;
|
||||
var manifestRequestCodeExpiration = DateTime.MinValue;
|
||||
|
||||
@@ -784,7 +802,7 @@ namespace DepotDownloader
|
||||
|
||||
try
|
||||
{
|
||||
connection = cdnPool.GetConnection(cts.Token);
|
||||
connection = cdnPool.GetConnection();
|
||||
|
||||
string cdnToken = null;
|
||||
if (steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise))
|
||||
@@ -810,7 +828,6 @@ namespace DepotDownloader
|
||||
// If we could not get the manifest code, this is a fatal error
|
||||
if (manifestRequestCode == 0)
|
||||
{
|
||||
Console.WriteLine("No manifest request code was returned for {0} {1}", depot.DepotId, depot.ManifestId);
|
||||
cts.Cancel();
|
||||
}
|
||||
}
|
||||
@@ -820,7 +837,7 @@ namespace DepotDownloader
|
||||
depot.ManifestId,
|
||||
connection,
|
||||
cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
|
||||
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(
|
||||
newManifest = await cdnPool.CDNClient.DownloadManifestAsync(
|
||||
depot.DepotId,
|
||||
depot.ManifestId,
|
||||
manifestRequestCode,
|
||||
@@ -872,9 +889,9 @@ namespace DepotDownloader
|
||||
cdnPool.ReturnBrokenConnection(connection);
|
||||
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
|
||||
}
|
||||
} while (depotManifest == null);
|
||||
} while (newManifest == null);
|
||||
|
||||
if (depotManifest == null)
|
||||
if (newManifest == null)
|
||||
{
|
||||
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId);
|
||||
cts.Cancel();
|
||||
@@ -883,28 +900,21 @@ namespace DepotDownloader
|
||||
// Throw the cancellation exception if requested so that this task is marked failed
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
|
||||
|
||||
newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
|
||||
newProtoManifest.SaveToFile(newManifestFileName, out var checksum);
|
||||
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
|
||||
|
||||
Console.WriteLine(" Done!");
|
||||
Util.SaveManifestToFile(configDir, newManifest);
|
||||
}
|
||||
}
|
||||
|
||||
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
|
||||
|
||||
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
|
||||
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime);
|
||||
|
||||
if (Config.DownloadManifestOnly)
|
||||
{
|
||||
DumpManifestToTextFile(depot, newProtoManifest);
|
||||
DumpManifestToTextFile(depot, newManifest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);
|
||||
|
||||
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
|
||||
var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
|
||||
var allFileNames = new HashSet<string>(filesAfterExclusions.Count);
|
||||
|
||||
// Pre-process
|
||||
@@ -936,8 +946,8 @@ namespace DepotDownloader
|
||||
depotDownloadInfo = depot,
|
||||
depotCounter = depotCounter,
|
||||
stagingDir = stagingDir,
|
||||
manifest = newProtoManifest,
|
||||
previousManifest = oldProtoManifest,
|
||||
manifest = newManifest,
|
||||
previousManifest = oldManifest,
|
||||
filteredFiles = filesAfterExclusions,
|
||||
allFileNames = allFileNames
|
||||
};
|
||||
@@ -952,20 +962,27 @@ namespace DepotDownloader
|
||||
Console.WriteLine("Downloading depot {0}", depot.DepotId);
|
||||
|
||||
var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
|
||||
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
|
||||
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>();
|
||||
|
||||
await Util.InvokeAsync(
|
||||
files.Select(file => new Func<Task>(async () =>
|
||||
await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, downloadCounter, depotFilesData, file, networkChunkQueue)))),
|
||||
maxDegreeOfParallelism: Config.MaxDownloads
|
||||
);
|
||||
var parallelOptions = new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Config.MaxDownloads,
|
||||
CancellationToken = cts.Token
|
||||
};
|
||||
|
||||
await Util.InvokeAsync(
|
||||
networkChunkQueue.Select(q => new Func<Task>(async () =>
|
||||
await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, downloadCounter, depotFilesData,
|
||||
q.fileData, q.fileStreamData, q.chunk)))),
|
||||
maxDegreeOfParallelism: Config.MaxDownloads
|
||||
);
|
||||
await Parallel.ForEachAsync(files, parallelOptions, async (file, cancellationToken) =>
|
||||
{
|
||||
await Task.Yield();
|
||||
DownloadSteam3AsyncDepotFile(cts, downloadCounter, depotFilesData, file, networkChunkQueue);
|
||||
});
|
||||
|
||||
await Parallel.ForEachAsync(networkChunkQueue, parallelOptions, async (q, cancellationToken) =>
|
||||
{
|
||||
await DownloadSteam3AsyncDepotFileChunk(
|
||||
cts, downloadCounter, depotFilesData,
|
||||
q.fileData, q.fileStreamData, q.chunk
|
||||
);
|
||||
});
|
||||
|
||||
// Check for deleted files if updating the depot.
|
||||
if (depotFilesData.previousManifest != null)
|
||||
@@ -1006,8 +1023,8 @@ namespace DepotDownloader
|
||||
CancellationTokenSource cts,
|
||||
GlobalDownloadCounter downloadCounter,
|
||||
DepotFilesData depotFilesData,
|
||||
ProtoManifest.FileData file,
|
||||
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
|
||||
DepotManifest.FileData file,
|
||||
ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue)
|
||||
{
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
|
||||
@@ -1015,7 +1032,7 @@ namespace DepotDownloader
|
||||
var stagingDir = depotFilesData.stagingDir;
|
||||
var depotDownloadCounter = depotFilesData.depotCounter;
|
||||
var oldProtoManifest = depotFilesData.previousManifest;
|
||||
ProtoManifest.FileData oldManifestFile = null;
|
||||
DepotManifest.FileData oldManifestFile = null;
|
||||
if (oldProtoManifest != null)
|
||||
{
|
||||
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
|
||||
@@ -1030,7 +1047,7 @@ namespace DepotDownloader
|
||||
File.Delete(fileStagingPath);
|
||||
}
|
||||
|
||||
List<ProtoManifest.ChunkData> neededChunks;
|
||||
List<DepotManifest.ChunkData> neededChunks;
|
||||
var fi = new FileInfo(fileFinalPath);
|
||||
var fileDidExist = fi.Exists;
|
||||
if (!fileDidExist)
|
||||
@@ -1048,7 +1065,7 @@ namespace DepotDownloader
|
||||
throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
|
||||
}
|
||||
|
||||
neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
|
||||
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -1092,7 +1109,7 @@ namespace DepotDownloader
|
||||
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
|
||||
|
||||
var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength);
|
||||
if (!adler.SequenceEqual(match.OldChunk.Checksum))
|
||||
if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum)))
|
||||
{
|
||||
neededChunks.Add(match.NewChunk);
|
||||
}
|
||||
@@ -1211,9 +1228,9 @@ namespace DepotDownloader
|
||||
CancellationTokenSource cts,
|
||||
GlobalDownloadCounter downloadCounter,
|
||||
DepotFilesData depotFilesData,
|
||||
ProtoManifest.FileData file,
|
||||
DepotManifest.FileData file,
|
||||
FileStreamData fileStreamData,
|
||||
ProtoManifest.ChunkData chunk)
|
||||
DepotManifest.ChunkData chunk)
|
||||
{
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
|
||||
@@ -1222,17 +1239,8 @@ namespace DepotDownloader
|
||||
|
||||
var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant();
|
||||
|
||||
var data = new DepotManifest.ChunkData
|
||||
{
|
||||
ChunkID = chunk.ChunkID,
|
||||
Checksum = BitConverter.ToUInt32(chunk.Checksum),
|
||||
Offset = chunk.Offset,
|
||||
CompressedLength = chunk.CompressedLength,
|
||||
UncompressedLength = chunk.UncompressedLength
|
||||
};
|
||||
|
||||
var written = 0;
|
||||
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)data.UncompressedLength);
|
||||
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)chunk.UncompressedLength);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -1244,7 +1252,7 @@ namespace DepotDownloader
|
||||
|
||||
try
|
||||
{
|
||||
connection = cdnPool.GetConnection(cts.Token);
|
||||
connection = cdnPool.GetConnection();
|
||||
|
||||
string cdnToken = null;
|
||||
if (steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise))
|
||||
@@ -1256,7 +1264,7 @@ namespace DepotDownloader
|
||||
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
|
||||
written = await cdnPool.CDNClient.DownloadDepotChunkAsync(
|
||||
depot.DepotId,
|
||||
data,
|
||||
chunk,
|
||||
connection,
|
||||
chunkBuffer,
|
||||
depot.DepotKey,
|
||||
@@ -1270,6 +1278,7 @@ namespace DepotDownloader
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
|
||||
cdnPool.ReturnBrokenConnection(connection);
|
||||
}
|
||||
catch (SteamKitWebRequestException e)
|
||||
{
|
||||
@@ -1325,7 +1334,7 @@ namespace DepotDownloader
|
||||
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
|
||||
}
|
||||
|
||||
fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin);
|
||||
fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin);
|
||||
await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token);
|
||||
}
|
||||
finally
|
||||
@@ -1369,44 +1378,55 @@ namespace DepotDownloader
|
||||
}
|
||||
}
|
||||
|
||||
static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest)
|
||||
class ChunkIdComparer : IEqualityComparer<byte[]>
|
||||
{
|
||||
public bool Equals(byte[] x, byte[] y)
|
||||
{
|
||||
if (ReferenceEquals(x, y)) return true;
|
||||
if (x == null || y == null) return false;
|
||||
return x.SequenceEqual(y);
|
||||
}
|
||||
|
||||
public int GetHashCode(byte[] obj)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(obj);
|
||||
|
||||
// ChunkID is SHA-1, so we can just use the first 4 bytes
|
||||
return BitConverter.ToInt32(obj, 0);
|
||||
}
|
||||
}
|
||||
|
||||
static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest)
|
||||
{
|
||||
var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt");
|
||||
using var sw = new StreamWriter(txtManifest);
|
||||
|
||||
sw.WriteLine($"Content Manifest for Depot {depot.DepotId}");
|
||||
sw.WriteLine($"Content Manifest for Depot {depot.DepotId} ");
|
||||
sw.WriteLine();
|
||||
sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime}");
|
||||
sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime} ");
|
||||
|
||||
int numFiles = 0, numChunks = 0;
|
||||
ulong uncompressedSize = 0, compressedSize = 0;
|
||||
var uniqueChunks = new HashSet<byte[]>(new ChunkIdComparer());
|
||||
|
||||
foreach (var file in manifest.Files)
|
||||
{
|
||||
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
|
||||
continue;
|
||||
|
||||
numFiles++;
|
||||
numChunks += file.Chunks.Count;
|
||||
|
||||
foreach (var chunk in file.Chunks)
|
||||
{
|
||||
uncompressedSize += chunk.UncompressedLength;
|
||||
compressedSize += chunk.CompressedLength;
|
||||
uniqueChunks.Add(chunk.ChunkID);
|
||||
}
|
||||
}
|
||||
|
||||
sw.WriteLine($"Total number of files : {numFiles}");
|
||||
sw.WriteLine($"Total number of chunks : {numChunks}");
|
||||
sw.WriteLine($"Total bytes on disk : {uncompressedSize}");
|
||||
sw.WriteLine($"Total bytes compressed : {compressedSize}");
|
||||
sw.WriteLine($"Total number of files : {manifest.Files.Count} ");
|
||||
sw.WriteLine($"Total number of chunks : {uniqueChunks.Count} ");
|
||||
sw.WriteLine($"Total bytes on disk : {manifest.TotalUncompressedSize} ");
|
||||
sw.WriteLine($"Total bytes compressed : {manifest.TotalCompressedSize} ");
|
||||
sw.WriteLine();
|
||||
sw.WriteLine();
|
||||
sw.WriteLine(" Size Chunks File SHA Flags Name");
|
||||
|
||||
foreach (var file in manifest.Files)
|
||||
{
|
||||
var sha1Hash = Convert.ToHexString(file.FileHash);
|
||||
sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}");
|
||||
var sha1Hash = Convert.ToHexString(file.FileHash).ToLower();
|
||||
sw.WriteLine($"{file.TotalSize,14:d} {file.Chunks.Count,6:d} {sha1Hash} {(int)file.Flags,5:x} {file.FileName}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
<RollForward>LatestMajor</RollForward>
|
||||
<Version>2.7.3</Version>
|
||||
<Version>3.4.0</Version>
|
||||
<Description>Steam Downloading Utility</Description>
|
||||
<Authors>SteamRE Team</Authors>
|
||||
<Copyright>Copyright © SteamRE Team 2024</Copyright>
|
||||
<Copyright>Copyright © SteamRE Team 2025</Copyright>
|
||||
<ApplicationIcon>..\Icon\DepotDownloader.ico</ApplicationIcon>
|
||||
<Deterministic>true</Deterministic>
|
||||
<TreatWarningsAsErrors Condition="'$(ContinuousIntegrationBuild)' == 'true'">true</TreatWarningsAsErrors>
|
||||
@@ -21,12 +21,12 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Windows.CsWin32" Version="0.3.106">
|
||||
<PackageReference Include="Microsoft.Windows.CsWin32" Version="0.3.363">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="protobuf-net" Version="3.2.45" />
|
||||
<PackageReference Include="QRCoder" Version="1.6.0" />
|
||||
<PackageReference Include="SteamKit2" Version="3.0.0" />
|
||||
<PackageReference Include="protobuf-net" Version="3.4.30" />
|
||||
<PackageReference Include="QRCoder" Version="1.8.1" />
|
||||
<PackageReference Include="SteamKit2" Version="3.4.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -23,7 +23,6 @@ namespace DepotDownloader
|
||||
|
||||
public bool VerifyAll { get; set; }
|
||||
|
||||
public int MaxServers { get; set; }
|
||||
public int MaxDownloads { get; set; }
|
||||
|
||||
public bool RememberPassword { get; set; }
|
||||
@@ -32,5 +31,6 @@ namespace DepotDownloader
|
||||
public uint? LoginID { get; set; }
|
||||
|
||||
public bool UseQrCode { get; set; }
|
||||
public bool SkipAppConfirmation { get; set; }
|
||||
}
|
||||
}
|
||||
+103
-6
@@ -11,11 +11,14 @@ using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using SteamKit2;
|
||||
using SteamKit2.CDN;
|
||||
|
||||
namespace DepotDownloader
|
||||
{
|
||||
class Program
|
||||
{
|
||||
private static bool[] consumedArgs;
|
||||
|
||||
static async Task<int> Main(string[] args)
|
||||
{
|
||||
if (args.Length == 0)
|
||||
@@ -46,6 +49,8 @@ namespace DepotDownloader
|
||||
return 0;
|
||||
}
|
||||
|
||||
consumedArgs = new bool[args.Length];
|
||||
|
||||
if (HasParameter(args, "-debug"))
|
||||
{
|
||||
PrintVersion(true);
|
||||
@@ -63,6 +68,21 @@ namespace DepotDownloader
|
||||
var password = GetParameter<string>(args, "-password") ?? GetParameter<string>(args, "-pass");
|
||||
ContentDownloader.Config.RememberPassword = HasParameter(args, "-remember-password");
|
||||
ContentDownloader.Config.UseQrCode = HasParameter(args, "-qr");
|
||||
ContentDownloader.Config.SkipAppConfirmation = HasParameter(args, "-no-mobile");
|
||||
|
||||
if (username == null)
|
||||
{
|
||||
if (ContentDownloader.Config.RememberPassword && !ContentDownloader.Config.UseQrCode)
|
||||
{
|
||||
Console.WriteLine("Error: -remember-password can not be used without -username or -qr.");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
else if (ContentDownloader.Config.UseQrCode)
|
||||
{
|
||||
Console.WriteLine("Error: -qr can not be used with -username.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
ContentDownloader.Config.DownloadManifestOnly = HasParameter(args, "-manifest-only");
|
||||
|
||||
@@ -117,9 +137,24 @@ namespace DepotDownloader
|
||||
ContentDownloader.Config.InstallDirectory = GetParameter<string>(args, "-dir");
|
||||
|
||||
ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate");
|
||||
ContentDownloader.Config.MaxServers = GetParameter(args, "-max-servers", 20);
|
||||
|
||||
if (HasParameter(args, "-use-lancache"))
|
||||
{
|
||||
await Client.DetectLancacheServerAsync();
|
||||
if (Client.UseLancacheServer)
|
||||
{
|
||||
Console.WriteLine("Detected Lancache server! Downloads will be directed through the Lancache.");
|
||||
|
||||
// Increasing the number of concurrent downloads when the cache is detected since the downloads will likely
|
||||
// be served much faster than over the internet. Steam internally has this behavior as well.
|
||||
if (!HasParameter(args, "-max-downloads"))
|
||||
{
|
||||
ContentDownloader.Config.MaxDownloads = 25;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ContentDownloader.Config.MaxDownloads = GetParameter(args, "-max-downloads", 8);
|
||||
ContentDownloader.Config.MaxServers = Math.Max(ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads);
|
||||
ContentDownloader.Config.LoginID = HasParameter(args, "-loginid") ? GetParameter<uint>(args, "-loginid") : null;
|
||||
|
||||
#endregion
|
||||
@@ -137,6 +172,8 @@ namespace DepotDownloader
|
||||
{
|
||||
#region Pubfile Downloading
|
||||
|
||||
PrintUnconsumedArgs(args);
|
||||
|
||||
if (InitializeSteam(username, password))
|
||||
{
|
||||
try
|
||||
@@ -172,6 +209,8 @@ namespace DepotDownloader
|
||||
{
|
||||
#region UGC Downloading
|
||||
|
||||
PrintUnconsumedArgs(args);
|
||||
|
||||
if (InitializeSteam(username, password))
|
||||
{
|
||||
try
|
||||
@@ -208,7 +247,13 @@ namespace DepotDownloader
|
||||
#region App downloading
|
||||
|
||||
var branch = GetParameter<string>(args, "-branch") ?? GetParameter<string>(args, "-beta") ?? ContentDownloader.DEFAULT_BRANCH;
|
||||
ContentDownloader.Config.BetaPassword = GetParameter<string>(args, "-betapassword");
|
||||
ContentDownloader.Config.BetaPassword = GetParameter<string>(args, "-branchpassword") ?? GetParameter<string>(args, "-betapassword");
|
||||
|
||||
if (!string.IsNullOrEmpty(ContentDownloader.Config.BetaPassword) && string.IsNullOrEmpty(branch))
|
||||
{
|
||||
Console.WriteLine("Error: Cannot specify -branchpassword when -branch is not specified.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
ContentDownloader.Config.DownloadAllPlatforms = HasParameter(args, "-all-platforms");
|
||||
|
||||
@@ -262,6 +307,8 @@ namespace DepotDownloader
|
||||
depotManifestIds.AddRange(depotIdList.Select(depotId => (depotId, ContentDownloader.INVALID_MANIFEST_ID)));
|
||||
}
|
||||
|
||||
PrintUnconsumedArgs(args);
|
||||
|
||||
if (InitializeSteam(username, password))
|
||||
{
|
||||
try
|
||||
@@ -303,6 +350,11 @@ namespace DepotDownloader
|
||||
{
|
||||
if (username != null && password == null && (!ContentDownloader.Config.RememberPassword || !AccountSettingsStore.Instance.LoginTokens.ContainsKey(username)))
|
||||
{
|
||||
if (AccountSettingsStore.Instance.LoginTokens.ContainsKey(username))
|
||||
{
|
||||
Console.WriteLine($"Account \"{username}\" has stored credentials. Did you forget to specify -remember-password?");
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
Console.Write("Enter account password for \"{0}\": ", username);
|
||||
@@ -325,6 +377,21 @@ namespace DepotDownloader
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(password))
|
||||
{
|
||||
const int MAX_PASSWORD_SIZE = 64;
|
||||
|
||||
if (password.Length > MAX_PASSWORD_SIZE)
|
||||
{
|
||||
Console.Error.WriteLine($"Warning: Password is longer than {MAX_PASSWORD_SIZE} characters, which is not supported by Steam.");
|
||||
}
|
||||
|
||||
if (!password.All(char.IsAscii))
|
||||
{
|
||||
Console.Error.WriteLine("Warning: Password contains non-ASCII characters, which is not supported by Steam.");
|
||||
}
|
||||
}
|
||||
|
||||
return ContentDownloader.InitializeSteam3(username, password);
|
||||
}
|
||||
|
||||
@@ -333,7 +400,10 @@ namespace DepotDownloader
|
||||
for (var x = 0; x < args.Length; ++x)
|
||||
{
|
||||
if (args[x].Equals(param, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
consumedArgs[x] = true;
|
||||
return x;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
@@ -356,6 +426,7 @@ namespace DepotDownloader
|
||||
var converter = TypeDescriptor.GetConverter(typeof(T));
|
||||
if (converter != null)
|
||||
{
|
||||
consumedArgs[index + 1] = true;
|
||||
return (T)converter.ConvertFromString(strParam);
|
||||
}
|
||||
|
||||
@@ -381,6 +452,7 @@ namespace DepotDownloader
|
||||
var converter = TypeDescriptor.GetConverter(typeof(T));
|
||||
if (converter != null)
|
||||
{
|
||||
consumedArgs[index] = true;
|
||||
list.Add((T)converter.ConvertFromString(strParam));
|
||||
}
|
||||
|
||||
@@ -390,6 +462,26 @@ namespace DepotDownloader
|
||||
return list;
|
||||
}
|
||||
|
||||
static void PrintUnconsumedArgs(string[] args)
|
||||
{
|
||||
var printError = false;
|
||||
|
||||
for (var index = 0; index < consumedArgs.Length; index++)
|
||||
{
|
||||
if (!consumedArgs[index])
|
||||
{
|
||||
printError = true;
|
||||
Console.Error.WriteLine($"Argument #{index + 1} {args[index]} was not used.");
|
||||
}
|
||||
}
|
||||
|
||||
if (printError)
|
||||
{
|
||||
Console.Error.WriteLine("Make sure you specified the arguments correctly. Check --help for correct arguments.");
|
||||
Console.Error.WriteLine();
|
||||
}
|
||||
}
|
||||
|
||||
static void PrintUsage()
|
||||
{
|
||||
// Do not use tabs to align parameters here because tab size may differ
|
||||
@@ -407,8 +499,8 @@ namespace DepotDownloader
|
||||
Console.WriteLine(" -app <#> - the AppID to download.");
|
||||
Console.WriteLine(" -depot <#> - the DepotID to download.");
|
||||
Console.WriteLine(" -manifest <id> - manifest id of content to download (requires -depot, default: current for branch).");
|
||||
Console.WriteLine($" -beta <branchname> - download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH}).");
|
||||
Console.WriteLine(" -betapassword <pass> - branch password if applicable.");
|
||||
Console.WriteLine($" -branch <branchname> - download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH}).");
|
||||
Console.WriteLine(" -branchpassword <pass> - branch password if applicable.");
|
||||
Console.WriteLine(" -all-platforms - downloads all platform-specific depots when -app is used.");
|
||||
Console.WriteLine(" -all-archs - download all architecture-specific depots when -app is used.");
|
||||
Console.WriteLine(" -os <os> - the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)");
|
||||
@@ -424,6 +516,8 @@ namespace DepotDownloader
|
||||
Console.WriteLine(" -password <pass> - the password of the account to login to for restricted content.");
|
||||
Console.WriteLine(" -remember-password - if set, remember the password for subsequent logins of this user.");
|
||||
Console.WriteLine(" use -username <username> -remember-password as login credentials.");
|
||||
Console.WriteLine(" -qr - display a login QR code to be scanned with the Steam mobile app");
|
||||
Console.WriteLine(" -no-mobile - prefer entering a 2FA code instead of prompting to accept in the Steam mobile app");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(" -dir <installdir> - the directory in which to place downloaded files.");
|
||||
Console.WriteLine(" -filelist <file.txt> - the name of a local file that contains a list of files to download (from the manifest).");
|
||||
@@ -432,9 +526,12 @@ namespace DepotDownloader
|
||||
Console.WriteLine(" -validate - include checksum verification of files already downloaded");
|
||||
Console.WriteLine(" -manifest-only - downloads a human readable manifest for any depots that would be downloaded.");
|
||||
Console.WriteLine(" -cellid <#> - the overridden CellID of the content server to download from.");
|
||||
Console.WriteLine(" -max-servers <#> - maximum number of content servers to use. (default: 20).");
|
||||
Console.WriteLine(" -max-downloads <#> - maximum number of chunks to download concurrently. (default: 8).");
|
||||
Console.WriteLine(" -loginid <#> - a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.");
|
||||
Console.WriteLine(" -use-lancache - forces downloads over the local network via a Lancache instance.");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(" -debug - enable verbose debug logging.");
|
||||
Console.WriteLine(" -V or --version - print version and runtime.");
|
||||
}
|
||||
|
||||
static void PrintVersion(bool printExtra = false)
|
||||
|
||||
@@ -6,6 +6,7 @@ using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using ProtoBuf;
|
||||
using SteamKit2;
|
||||
|
||||
@@ -157,5 +158,39 @@ namespace DepotDownloader
|
||||
using var ds = new DeflateStream(fs, CompressionMode.Compress);
|
||||
ms.CopyTo(ds);
|
||||
}
|
||||
|
||||
public DepotManifest ConvertToSteamManifest(uint depotId)
|
||||
{
|
||||
ulong uncompressedSize = 0, compressedSize = 0;
|
||||
var newManifest = new DepotManifest();
|
||||
newManifest.Files = new List<DepotManifest.FileData>(Files.Count);
|
||||
|
||||
foreach (var file in Files)
|
||||
{
|
||||
var fileNameHash = SHA1.HashData(Encoding.UTF8.GetBytes(file.FileName.Replace('/', '\\').ToLowerInvariant()));
|
||||
var newFile = new DepotManifest.FileData(file.FileName, fileNameHash, file.Flags, file.TotalSize, file.FileHash, null, false, file.Chunks.Count);
|
||||
|
||||
foreach (var chunk in file.Chunks)
|
||||
{
|
||||
var newChunk = new DepotManifest.ChunkData(chunk.ChunkID, BitConverter.ToUInt32(chunk.Checksum, 0), chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength);
|
||||
newFile.Chunks.Add(newChunk);
|
||||
|
||||
uncompressedSize += chunk.UncompressedLength;
|
||||
compressedSize += chunk.CompressedLength;
|
||||
}
|
||||
|
||||
newManifest.Files.Add(newFile);
|
||||
}
|
||||
|
||||
newManifest.FilenamesEncrypted = false;
|
||||
newManifest.DepotID = depotId;
|
||||
newManifest.ManifestGID = ID;
|
||||
newManifest.CreationTime = CreationTime;
|
||||
newManifest.TotalUncompressedSize = uncompressedSize;
|
||||
newManifest.TotalCompressedSize = compressedSize;
|
||||
newManifest.EncryptedCRC = 0;
|
||||
|
||||
return newManifest;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ namespace DepotDownloader
|
||||
|
||||
var clientConfiguration = SteamConfiguration.Create(config =>
|
||||
config
|
||||
.WithHttpClientFactory(HttpClientFactory.CreateHttpClient)
|
||||
.WithHttpClientFactory(static purpose => HttpClientFactory.CreateHttpClient())
|
||||
);
|
||||
|
||||
this.steamClient = new SteamClient(clientConfiguration);
|
||||
@@ -248,7 +248,6 @@ namespace DepotDownloader
|
||||
|
||||
if (depotKey.Result != EResult.OK)
|
||||
{
|
||||
Abort();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -263,9 +262,19 @@ namespace DepotDownloader
|
||||
|
||||
var requestCode = await steamContent.GetManifestRequestCode(depotId, appId, manifestId, branch);
|
||||
|
||||
Console.WriteLine("Got manifest request code for {0} {1} result: {2}",
|
||||
depotId, manifestId,
|
||||
requestCode);
|
||||
if (requestCode == 0)
|
||||
{
|
||||
Console.WriteLine($"No manifest request code was returned for depot {depotId} from app {appId}, manifest {manifestId}");
|
||||
|
||||
if (!authenticatedUser)
|
||||
{
|
||||
Console.WriteLine("Suggestion: Try logging in with -username as old manifests may not be available for anonymous accounts.");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"Got manifest request code for depot {depotId} from app {appId}, manifest {manifestId}, result: {requestCode}");
|
||||
}
|
||||
|
||||
return requestCode;
|
||||
}
|
||||
@@ -306,9 +315,29 @@ namespace DepotDownloader
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<KeyValue> GetPrivateBetaDepotSection(uint appid, string branch)
|
||||
{
|
||||
if (!AppBetaPasswords.TryGetValue(branch, out var branchPassword)) // Should be filled by CheckAppBetaPassword
|
||||
{
|
||||
return new KeyValue();
|
||||
}
|
||||
|
||||
AppTokens.TryGetValue(appid, out var accessToken); // Should be filled by RequestAppInfo
|
||||
|
||||
var privateBeta = await steamApps.PICSGetPrivateBeta(appid, accessToken, branch, branchPassword);
|
||||
|
||||
Console.WriteLine($"Retrieved private beta depot section for {appid} with result: {privateBeta.Result}");
|
||||
|
||||
return privateBeta.DepotSection;
|
||||
}
|
||||
|
||||
public async Task<PublishedFileDetails> GetPublishedFileDetails(uint appId, PublishedFileID pubFile)
|
||||
{
|
||||
var pubFileRequest = new CPublishedFile_GetDetails_Request { appid = appId };
|
||||
var pubFileRequest = new CPublishedFile_GetDetails_Request
|
||||
{
|
||||
appid = appId,
|
||||
includechildren = true,
|
||||
};
|
||||
pubFileRequest.publishedfileids.Add(pubFile);
|
||||
|
||||
var details = await steamPublishedFile.GetDetails(pubFileRequest);
|
||||
@@ -417,13 +446,14 @@ namespace DepotDownloader
|
||||
try
|
||||
{
|
||||
_ = AccountSettingsStore.Instance.GuardData.TryGetValue(logonDetails.Username, out var guarddata);
|
||||
authSession = await steamClient.Authentication.BeginAuthSessionViaCredentialsAsync(new SteamKit2.Authentication.AuthSessionDetails
|
||||
authSession = await steamClient.Authentication.BeginAuthSessionViaCredentialsAsync(new AuthSessionDetails
|
||||
{
|
||||
DeviceFriendlyName = nameof(DepotDownloader),
|
||||
Username = logonDetails.Username,
|
||||
Password = logonDetails.Password,
|
||||
IsPersistentSession = ContentDownloader.Config.RememberPassword,
|
||||
GuardData = guarddata,
|
||||
Authenticator = new UserConsoleAuthenticator(),
|
||||
Authenticator = new ConsoleAuthenticator(),
|
||||
});
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
@@ -445,8 +475,8 @@ namespace DepotDownloader
|
||||
{
|
||||
var session = await steamClient.Authentication.BeginAuthSessionViaQRAsync(new AuthSessionDetails
|
||||
{
|
||||
DeviceFriendlyName = nameof(DepotDownloader),
|
||||
IsPersistentSession = ContentDownloader.Config.RememberPassword,
|
||||
Authenticator = new UserConsoleAuthenticator(),
|
||||
});
|
||||
|
||||
authSession = session;
|
||||
@@ -489,11 +519,17 @@ namespace DepotDownloader
|
||||
if (result.NewGuardData != null)
|
||||
{
|
||||
AccountSettingsStore.Instance.GuardData[result.AccountName] = result.NewGuardData;
|
||||
|
||||
if (ContentDownloader.Config.UseQrCode)
|
||||
{
|
||||
Console.WriteLine($"Success! Next time you can login with -username {result.AccountName} -remember-password instead of -qr.");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
AccountSettingsStore.Instance.GuardData.Remove(result.AccountName);
|
||||
}
|
||||
|
||||
AccountSettingsStore.Instance.LoginTokens[result.AccountName] = result.RefreshToken;
|
||||
AccountSettingsStore.Save();
|
||||
}
|
||||
@@ -674,10 +710,14 @@ namespace DepotDownloader
|
||||
using var qrGenerator = new QRCodeGenerator();
|
||||
var qrCodeData = qrGenerator.CreateQrCode(challengeUrl, QRCodeGenerator.ECCLevel.L);
|
||||
using var qrCode = new AsciiQRCode(qrCodeData);
|
||||
var qrCodeAsAsciiArt = qrCode.GetGraphic(1, drawQuietZones: false);
|
||||
var qrCodeAsAsciiArt = qrCode.GetLineByLineGraphic(1, drawQuietZones: true);
|
||||
|
||||
Console.WriteLine("Use the Steam Mobile App to sign in with this QR code:");
|
||||
Console.WriteLine(qrCodeAsAsciiArt);
|
||||
|
||||
foreach (var line in qrCodeAsAsciiArt)
|
||||
{
|
||||
Console.WriteLine(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+98
-35
@@ -9,6 +9,7 @@ using System.Runtime.InteropServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using SteamKit2;
|
||||
|
||||
namespace DepotDownloader
|
||||
{
|
||||
@@ -78,16 +79,16 @@ namespace DepotDownloader
|
||||
}
|
||||
|
||||
// Validate a file against Steam3 Chunk data
|
||||
public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata)
|
||||
public static List<DepotManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata)
|
||||
{
|
||||
var neededChunks = new List<ProtoManifest.ChunkData>();
|
||||
var neededChunks = new List<DepotManifest.ChunkData>();
|
||||
|
||||
foreach (var data in chunkdata)
|
||||
{
|
||||
fs.Seek((long)data.Offset, SeekOrigin.Begin);
|
||||
|
||||
var adler = AdlerHash(fs, (int)data.UncompressedLength);
|
||||
if (!adler.SequenceEqual(data.Checksum))
|
||||
if (!adler.SequenceEqual(BitConverter.GetBytes(data.Checksum)))
|
||||
{
|
||||
neededChunks.Add(data);
|
||||
}
|
||||
@@ -110,6 +111,100 @@ namespace DepotDownloader
|
||||
return BitConverter.GetBytes(a | (b << 16));
|
||||
}
|
||||
|
||||
public static byte[] FileSHAHash(string filename)
|
||||
{
|
||||
using (var fs = File.Open(filename, FileMode.Open))
|
||||
using (var sha = SHA1.Create())
|
||||
{
|
||||
var output = sha.ComputeHash(fs);
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
|
||||
public static DepotManifest LoadManifestFromFile(string directory, uint depotId, ulong manifestId, bool badHashWarning)
|
||||
{
|
||||
// Try loading Steam format manifest first.
|
||||
var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", depotId, manifestId));
|
||||
|
||||
if (File.Exists(filename))
|
||||
{
|
||||
byte[] expectedChecksum;
|
||||
|
||||
try
|
||||
{
|
||||
expectedChecksum = File.ReadAllBytes(filename + ".sha");
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
expectedChecksum = null;
|
||||
}
|
||||
|
||||
var currentChecksum = FileSHAHash(filename);
|
||||
|
||||
if (expectedChecksum != null && expectedChecksum.SequenceEqual(currentChecksum))
|
||||
{
|
||||
return DepotManifest.LoadFromFile(filename);
|
||||
}
|
||||
else if (badHashWarning)
|
||||
{
|
||||
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId);
|
||||
}
|
||||
}
|
||||
|
||||
// Try converting legacy manifest format.
|
||||
filename = Path.Combine(directory, string.Format("{0}_{1}.bin", depotId, manifestId));
|
||||
|
||||
if (File.Exists(filename))
|
||||
{
|
||||
byte[] expectedChecksum;
|
||||
|
||||
try
|
||||
{
|
||||
expectedChecksum = File.ReadAllBytes(filename + ".sha");
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
expectedChecksum = null;
|
||||
}
|
||||
|
||||
byte[] currentChecksum;
|
||||
var oldManifest = ProtoManifest.LoadFromFile(filename, out currentChecksum);
|
||||
|
||||
if (oldManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
|
||||
{
|
||||
oldManifest = null;
|
||||
|
||||
if (badHashWarning)
|
||||
{
|
||||
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId);
|
||||
}
|
||||
}
|
||||
|
||||
if (oldManifest != null)
|
||||
{
|
||||
return oldManifest.ConvertToSteamManifest(depotId);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public static bool SaveManifestToFile(string directory, DepotManifest manifest)
|
||||
{
|
||||
try
|
||||
{
|
||||
var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", manifest.DepotID, manifest.ManifestGID));
|
||||
manifest.SaveToFile(filename);
|
||||
File.WriteAllBytes(filename + ".sha", FileSHAHash(filename));
|
||||
return true; // If serialization completes without throwing an exception, return true
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false; // Return false if an error occurs
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] DecodeHexString(string hex)
|
||||
{
|
||||
if (hex == null)
|
||||
@@ -140,37 +235,5 @@ namespace DepotDownloader
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
public static async Task InvokeAsync(IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(taskFactories);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(maxDegreeOfParallelism, 0);
|
||||
|
||||
var queue = taskFactories.ToArray();
|
||||
|
||||
if (queue.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var tasksInFlight = new List<Task>(maxDegreeOfParallelism);
|
||||
var index = 0;
|
||||
|
||||
do
|
||||
{
|
||||
while (tasksInFlight.Count < maxDegreeOfParallelism && index < queue.Length)
|
||||
{
|
||||
var taskFactory = queue[index++];
|
||||
|
||||
tasksInFlight.Add(taskFactory());
|
||||
}
|
||||
|
||||
var completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false);
|
||||
|
||||
await completedTask.ConfigureAwait(false);
|
||||
|
||||
tasksInFlight.Remove(completedTask);
|
||||
} while (index < queue.Length || tasksInFlight.Count != 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,34 +61,54 @@ For example: `./DepotDownloader -app 730 -ugc 770604181014286929`
|
||||
|
||||
## Parameters
|
||||
|
||||
#### Authentication
|
||||
|
||||
Parameter | Description
|
||||
----------------------- | -----------
|
||||
`-app <#>` | the AppID to download.
|
||||
`-depot <#>` | the DepotID to download.
|
||||
`-manifest <id>` | manifest id of content to download (requires `-depot`, default: current for branch).
|
||||
`-ugc <#>` | the UGC ID to download.
|
||||
`-beta <branchname>` | download from specified branch if available (default: Public).
|
||||
`-betapassword <pass>` | branch password if applicable.
|
||||
`-all-platforms` | downloads all platform-specific depots when `-app` is used.
|
||||
`-os <os>` | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)
|
||||
`-osarch <arch>` | the architecture for which to download the game (32 or 64, default: the host's architecture)
|
||||
`-all-archs` | download all architecture-specific depots when `-app` is used.
|
||||
`-all-languages` | download all language-specific depots when `-app` is used.
|
||||
`-language <lang>` | the language for which to download the game (default: english)
|
||||
`-lowviolence` | download low violence depots when `-app` is used.
|
||||
`-pubfile <#>` | the PublishedFileId to download. (Will automatically resolve to UGC id)
|
||||
`-username <user>` | the username of the account to login to for restricted content.
|
||||
`-password <pass>` | the password of the account to login to for restricted content.
|
||||
`-remember-password` | if set, remember the password for subsequent logins of this user. (Use `-username <username> -remember-password` as login credentials)
|
||||
`-username <user>` | the username of the account to login to for restricted content.
|
||||
`-password <pass>` | the password of the account to login to for restricted content.
|
||||
`-remember-password` | if set, remember the password for subsequent logins of this user. (Use `-username <username> -remember-password` as login credentials)
|
||||
`-qr` | display a login QR code to be scanned with the Steam mobile app
|
||||
`-no-mobile` | prefer entering a 2FA code instead of prompting to accept in the Steam mobile app.
|
||||
`-loginid <#>` | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.
|
||||
|
||||
#### Downloading
|
||||
|
||||
Parameter | Description
|
||||
------------------------ | -----------
|
||||
`-app <#>` | the AppID to download.
|
||||
`-depot <#>` | the DepotID to download.
|
||||
`-manifest <id>` | manifest id of content to download (requires `-depot`, default: current for branch).
|
||||
`-ugc <#>` | the UGC ID to download.
|
||||
`-pubfile <#>` | the PublishedFileId to download. (Will automatically resolve to UGC id)
|
||||
`-branch <branchname>` | download from specified branch if available (default: Public).
|
||||
`-branchpassword <pass>` | branch password if applicable.
|
||||
|
||||
#### Download configuration
|
||||
|
||||
Parameter | Description
|
||||
----------------------- | -----------
|
||||
`-all-platforms` | downloads all platform-specific depots when `-app` is used.
|
||||
`-os <os>` | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)
|
||||
`-osarch <arch>` | the architecture for which to download the game (32 or 64, default: the host's architecture)
|
||||
`-all-archs` | download all architecture-specific depots when `-app` is used.
|
||||
`-all-languages` | download all language-specific depots when `-app` is used.
|
||||
`-language <lang>` | the language for which to download the game (default: english)
|
||||
`-lowviolence` | download low violence depots when `-app` is used.
|
||||
`-dir <installdir>` | the directory in which to place downloaded files.
|
||||
`-filelist <file.txt>` | the name of a local file that contains a list of files to download (from the manifest). prefix file path with `regex:` if you want to match with regex. each file path should be on their own line.
|
||||
`-validate` | Include checksum verification of files already downloaded
|
||||
`-manifest-only` | downloads a human readable manifest for any depots that would be downloaded.
|
||||
`-cellid <#>` | the overridden CellID of the content server to download from.
|
||||
`-max-servers <#>` | maximum number of content servers to use. (default: 20).
|
||||
`-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8).
|
||||
`-loginid <#>` | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.
|
||||
`-V` or `--version` | print version and runtime
|
||||
`-filelist <file.txt>` | the name of a local file that contains a list of files to download (from the manifest). prefix file path with `regex:` if you want to match with regex. each file path should be on their own line.
|
||||
`-validate` | include checksum verification of files already downloaded.
|
||||
`-manifest-only` | downloads a human readable manifest for any depots that would be downloaded.
|
||||
`-cellid <#>` | the overridden CellID of the content server to download from.
|
||||
`-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8).
|
||||
`-use-lancache` | forces downloads over the local network via a Lancache instance.
|
||||
|
||||
#### Other
|
||||
|
||||
Parameter | Description
|
||||
----------------------- | -----------
|
||||
`-debug` | enable verbose debug logging.
|
||||
`-V` or `--version` | print version and runtime.
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
@@ -100,3 +120,12 @@ Any connection to Steam will be closed if they share a LoginID. You can specify
|
||||
|
||||
### Why doesn't my password containing special characters work? Do I have to specify the password on the command line?
|
||||
If you pass the `-password` parameter with a password that contains special characters, you will need to escape the command appropriately for the shell you are using. You do not have to include the `-password` parameter on the command line as long as you include a `-username`. You will be prompted to enter your password interactively.
|
||||
|
||||
### I am getting error 401 or no manifest code returned for old manifest ids
|
||||
Try logging in with a Steam account, this may happen when using anonymous account.
|
||||
|
||||
Steam allows developers to block downloading old manifests, in which case no manifest code is returned even when parameters appear correct.
|
||||
|
||||
### Why am I getting slow download speeds and frequent connection timeouts?
|
||||
When downloading old builds, cache server may not have the chunks readily available which makes downloading slower.
|
||||
Try increasing `-max-downloads` to saturate the network more.
|
||||
Reference in new issue
Block a user