From c41d0ba9766479d3a0de48fb6c83cb34828cb8a6 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 04:00:22 -0600 Subject: [PATCH 1/5] New remap command for remapping system paths --- Parallel.Cli/Commands/RemapCommand.cs | 74 +++++++++++++++++++ Parallel.Cli/Commands/SyncCommand.cs | 8 ++ Parallel.Cli/Commands/VaultsCommand.cs | 7 ++ Parallel.Cli/Commands/ZipCommand.cs | 4 +- .../Database/Contexts/SqliteContext.cs | 16 ++++ Parallel.Core/Database/IDatabase.cs | 3 + Parallel.Core/Models/SystemFile.cs | 2 +- 7 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 Parallel.Cli/Commands/SyncCommand.cs diff --git a/Parallel.Cli/Commands/RemapCommand.cs b/Parallel.Cli/Commands/RemapCommand.cs index 459193d..930af3a 100644 --- a/Parallel.Cli/Commands/RemapCommand.cs +++ b/Parallel.Cli/Commands/RemapCommand.cs @@ -1,13 +1,87 @@ // Copyright 2025 Kyle Ebbinga using System.CommandLine; +using System.ComponentModel.DataAnnotations.Schema; +using System.Diagnostics; +using Parallel.Cli.Utils; +using Parallel.Core.IO.Syncing; +using Parallel.Core.Models; +using Parallel.Core.Security; +using Parallel.Core.Settings; +using SQLitePCL; namespace Parallel.Cli.Commands { public class RemapCommand : Command { + private readonly Argument _sourceArg = new("source", "The source path to change."); + private readonly Argument _targetArg = new("target", "The target path to change to."); + private readonly Option _optionOpt = new("config", "The vault configuration to use."); + + private Stopwatch _sw = new Stopwatch(); + public RemapCommand() : base("remap", "Remaps paths in the vault.") { + this.AddArgument(_sourceArg); + this.AddArgument(_targetArg); + this.SetHandler(async (config, source, target) => + { + _sw = Stopwatch.StartNew(); + LocalVaultConfig? vault = ParallelConfig.Load().Vaults.FirstOrDefault(); + if (!string.IsNullOrEmpty(config)) vault = ParallelConfig.GetVault(config); + if (vault == null) + { + CommandLine.WriteLine($"Unable to find vault with name: '{vault}'", ConsoleColor.Yellow); + return; + } + + if (string.IsNullOrEmpty(source) || string.IsNullOrEmpty(target)) + { + CommandLine.WriteLine("The source and target paths must be specified!", ConsoleColor.Yellow); + return; + } + + await RemapPathAsync(vault, source, target); + }, _optionOpt, _sourceArg, _targetArg); + + } + + private async Task RemapPathAsync(LocalVaultConfig vault, string source, string target) + { + ISyncManager syncManager = SyncManager.CreateNew(vault); + if (!await syncManager.ConnectAsync()) + { + CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); + return; + } + + CommandLine.WriteLine($"Locating files for remapping...", ConsoleColor.DarkGray); + IEnumerable files = await syncManager.Database.GetFilesAsync(source); + if (!files.Any()) + { + CommandLine.WriteLine(vault, "No files were found!", ConsoleColor.Yellow); + return; + } + + int progress = 0; + int total = files.Count(); + await System.Threading.Tasks.Parallel.ForEachAsync(files, async (file, ct) => + { + CommandLine.ProgressBar(progress++, total, _sw.Elapsed, ConsoleColor.DarkGray); + + string newPath = file.LocalPath.Replace(source, target); + string newId = HashGenerator.CreateSHA1(newPath); + await syncManager.Database.RemapObjectsAsync(file.Id, newId); + + //CommandLine.WriteLine(vault, $"Remapping '{file.LocalPath}' to '{newPath}'"); + await syncManager.Database.RemoveFileAsync(file); + + file.Id = newId; + file.LocalPath = newPath; + await syncManager.Database.AddFileAsync(file); + }); + + CommandLine.WriteLine(vault, $"Successfully remapped {files.Count():N0} files to '{target}'.", ConsoleColor.Green); } } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/SyncCommand.cs b/Parallel.Cli/Commands/SyncCommand.cs new file mode 100644 index 0000000..0d1cd5b --- /dev/null +++ b/Parallel.Cli/Commands/SyncCommand.cs @@ -0,0 +1,8 @@ +// Copyright 2025 Kyle Ebbinga + +namespace Parallel.Cli.Commands +{ + public class SyncCommand + { + } +} \ No newline at end of file diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index 8ad97f2..de0d753 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -18,6 +18,7 @@ public class VaultsCommand : Command private Command addCmd = new("add", "Adds a new vault configuration."); private Command editCmd = new("edit", "Edits a vault configuration."); + private readonly Command findCmd = new("find", "Finds vault configurations in a location."); private Command viewCmd = new("view", "Shows the vault configuration."); private Command setCmd = new("set", "Sets a new vault configuration."); private Command delCmd = new("delete", "Deletes a vault configuration."); @@ -69,6 +70,12 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") CommandLine.WriteLine($"Saved new storage vault: '{localVault.Name}' ({localVault.Id})"); }); + this.AddCommand(findCmd); + findCmd.SetHandler(() => + { + + }); + this.AddCommand(viewCmd); viewCmd.AddArgument(configArg); viewCmd.SetHandler(async (vault) => diff --git a/Parallel.Cli/Commands/ZipCommand.cs b/Parallel.Cli/Commands/ZipCommand.cs index 83722f7..ef6b354 100644 --- a/Parallel.Cli/Commands/ZipCommand.cs +++ b/Parallel.Cli/Commands/ZipCommand.cs @@ -33,12 +33,12 @@ public ZipCommand() : base("zip", "Zips files in a directory.") return; } - CommandLine.WriteLine($"Zipping {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); + CommandLine.WriteLine($"Zipping {files.Length:N0} files...", ConsoleColor.DarkGray); _totalTasks = files.Length; _tasks.AddRange(files.Select(file => Task.Run(() => CompressFile(file, keep)))); await Task.WhenAll(_tasks); - CommandLine.WriteLine($"Successfully zipped {files.Length.ToString("N0")} files in {_sw.Elapsed}.", ConsoleColor.Green); + CommandLine.WriteLine($"Successfully zipped {files.Length:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); }, sourceArg, keepOpt); } diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 1ab16e8..10c9c67 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -59,6 +59,14 @@ public async Task AddFileAsync(SystemFile file) return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }) > 0; } + /// + public async Task RemoveFileAsync(SystemFile file) + { + using IDbConnection connection = CreateConnection(); + string sql = $"DELETE FROM files WHERE id = @Id;"; + await connection.ExecuteAsync(sql, new { file.Id }); + } + public async Task GetLocalSizeAsync() { using IDbConnection connection = CreateConnection(); @@ -148,6 +156,14 @@ public async Task> GetObjectsAsync(string id) return await connection.QueryAsync(sql, new { id }); } + /// + public async Task RemapObjectsAsync(string oldId, string newId) + { + using IDbConnection connection = CreateConnection(); + string sql = "UPDATE objects SET id = @newId WHERE id = @oldId;"; + return await connection.ExecuteAsync(sql, new { oldId, newId }) > 0; + } + #endregion } } \ No newline at end of file diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index fddf31b..2bd0fba 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -75,6 +75,8 @@ public interface IDatabase /// True if successful, false otherwise Task AddFileAsync(SystemFile file); + Task RemoveFileAsync(SystemFile file); + Task> GetFilesAsync(string path); Task> GetFilesAsync(string path, bool deleted); Task GetFileAsync(string path); @@ -108,5 +110,6 @@ public interface IDatabase #endregion + Task RemapObjectsAsync(string oldId, string newId); } } \ No newline at end of file diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 6410b0e..0749743 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -16,7 +16,7 @@ public class SystemFile /// /// The unique identifier of the file. /// - public string Id { get; } = string.Empty; + public string Id { get; set; } = string.Empty; /// /// The name of the file. From edaed6fcaca134edf270ce3302d922cba73ae3cf Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 04:59:26 -0600 Subject: [PATCH 2/5] Various clean ups --- Parallel.Cli/Commands/PullCommand.cs | 3 ++- Parallel.Cli/Commands/RemapCommand.cs | 9 +++++---- Parallel.Cli/Commands/UnzipCommand.cs | 2 +- Parallel.Cli/Commands/ZipCommand.cs | 2 +- Parallel.Cli/Utils/CommandLine.cs | 8 +++----- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 2 -- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index 0b2c895..6f6c1c9 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -54,6 +54,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force return; } + CommandLine.WriteLine(vault, $"Scanning for files in {path}...", ConsoleColor.DarkGray); IEnumerable files = await syncManager.Database.GetFilesAsync(fullPath); if (!files.Any()) { @@ -69,7 +70,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force }); Log.Debug($"Pulling {pullFiles.Count} files..."); - await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressLogger()); + await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressReport(vault, files.Count())); CommandLine.WriteLine(vault, $"Successfully pulled {pullFiles.Count:N0} files from '{vault.FileSystem.RootDirectory}'.", ConsoleColor.Green); } diff --git a/Parallel.Cli/Commands/RemapCommand.cs b/Parallel.Cli/Commands/RemapCommand.cs index 930af3a..8641ae5 100644 --- a/Parallel.Cli/Commands/RemapCommand.cs +++ b/Parallel.Cli/Commands/RemapCommand.cs @@ -55,7 +55,7 @@ private async Task RemapPathAsync(LocalVaultConfig vault, string source, string return; } - CommandLine.WriteLine($"Locating files for remapping...", ConsoleColor.DarkGray); + CommandLine.WriteLine(vault, $"Scanning for files in {source}...", ConsoleColor.DarkGray); IEnumerable files = await syncManager.Database.GetFilesAsync(source); if (!files.Any()) { @@ -65,15 +65,15 @@ private async Task RemapPathAsync(LocalVaultConfig vault, string source, string int progress = 0; int total = files.Count(); + + CommandLine.WriteLine(vault, $"Remapping '{source}' to '{target}'..."); await System.Threading.Tasks.Parallel.ForEachAsync(files, async (file, ct) => { - CommandLine.ProgressBar(progress++, total, _sw.Elapsed, ConsoleColor.DarkGray); + CommandLine.ProgressBar(progress++, total, _sw.Elapsed); string newPath = file.LocalPath.Replace(source, target); string newId = HashGenerator.CreateSHA1(newPath); await syncManager.Database.RemapObjectsAsync(file.Id, newId); - - //CommandLine.WriteLine(vault, $"Remapping '{file.LocalPath}' to '{newPath}'"); await syncManager.Database.RemoveFileAsync(file); file.Id = newId; @@ -81,6 +81,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, async (file, ct) => await syncManager.Database.AddFileAsync(file); }); + await syncManager.DisconnectAsync(); CommandLine.WriteLine(vault, $"Successfully remapped {files.Count():N0} files to '{target}'.", ConsoleColor.Green); } } diff --git a/Parallel.Cli/Commands/UnzipCommand.cs b/Parallel.Cli/Commands/UnzipCommand.cs index 396a646..db0aae5 100644 --- a/Parallel.Cli/Commands/UnzipCommand.cs +++ b/Parallel.Cli/Commands/UnzipCommand.cs @@ -58,7 +58,7 @@ private void DecompressFile(string path, bool keep) } } - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw?.Elapsed ?? TimeSpan.Zero, ConsoleColor.DarkGray); + CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw?.Elapsed ?? TimeSpan.Zero); } } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/ZipCommand.cs b/Parallel.Cli/Commands/ZipCommand.cs index ef6b354..6d065f9 100644 --- a/Parallel.Cli/Commands/ZipCommand.cs +++ b/Parallel.Cli/Commands/ZipCommand.cs @@ -60,7 +60,7 @@ private void CompressFile(string path, bool keep) } } - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed, ConsoleColor.DarkGray); + CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed); } } } \ No newline at end of file diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index 3f16a0c..0e01b11 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -127,17 +127,15 @@ public static void WriteLine(object value, ConsoleColor color = ConsoleColor.Gra } } - public static void ProgressBar(double part, double total, TimeSpan elapsed, ConsoleColor color = ConsoleColor.Gray) + public static void ProgressBar(double part, double total, TimeSpan elapsed) { double percent = part / total; string percentStr = $"> Progress: {Convert.ToInt32(percent * 100).ToString("D2")}%"; TimeSpan remaining; double remainingMs = elapsed.TotalMilliseconds * (total - part) / part; - if (remainingMs <= TimeSpan.MaxValue.TotalMilliseconds) - remaining = TimeSpan.FromMilliseconds(remainingMs); - else - remaining = TimeSpan.MaxValue; + if (remainingMs <= TimeSpan.MaxValue.TotalMilliseconds) remaining = TimeSpan.FromMilliseconds(remainingMs); + else remaining = TimeSpan.MaxValue; string remainingStr = $"{remaining.Hours:00}:{remaining.Minutes:00}:{remaining.Seconds:00} remaining"; int barWidth = Console.WindowWidth - percentStr.Length - remainingStr.Length - 4; diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index 3c6f417..ade52c5 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -83,10 +83,8 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await FileSystem.ExistsAsync(remotePath)) { - Log.Debug($"Downloading object: {hash}"); await FileSystem.DownloadStreamAsync(fs, remotePath); } } From 6e89b4da851114e85d7d63838ee55f533ba7e42d Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 06:03:22 -0600 Subject: [PATCH 3/5] Cleaned up command line messages and reworked disk command to show new object storage savings --- Parallel.Cli/Commands/DiskCommand.cs | 13 ++- .../Database/Contexts/SqliteContext.cs | 8 ++ Parallel.Core/Database/IDatabase.cs | 3 +- .../Diagnostics/IProgressReporter.cs | 4 +- .../IO/FileSystem/DotNetFileSystem.cs | 1 - Parallel.Core/IO/FileSystem/SftpFileSystem.cs | 1 - Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 101 +++++++++++------- 7 files changed, 82 insertions(+), 49 deletions(-) diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index 4753ae6..2ed79a1 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -45,9 +45,15 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) IDatabase db = syncManager.Database; long localSize = await db.GetLocalSizeAsync(); - long remoteSize = await db.GetRemoteSizeAsync(); long totalLocalFiles = await db.GetTotalFilesAsync(false); long totalDeletedFiles = await db.GetTotalFilesAsync(true); + long totalObjects = await db.GetTotalObjectsAsync(); + + int chunkSize = ((ObjectSyncManager)syncManager).ChunkSize; + double expectedChunks = (double)localSize / chunkSize; + long expectedBytes = (long)(expectedChunks * chunkSize); + long actualBytes = totalObjects * chunkSize; + long savedBytes = expectedBytes - actualBytes; CommandLine.WriteLine($"Using vault '{vault.Name}' ({vault.Id}):"); CommandLine.WriteLine($"Service Type: {vault.FileSystem.Service}"); @@ -56,15 +62,14 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) CommandLine.WriteLine($"Local Files: {totalLocalFiles:N0}"); CommandLine.WriteLine($"Deleted Files: {totalDeletedFiles:N0}"); CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); - CommandLine.WriteLine($"Remote Size: {Formatter.FromBytes(remoteSize)}"); - CommandLine.WriteLine($"Space Saved: {Math.Round((localSize - remoteSize) / (double)localSize * 100, 2)}%"); + CommandLine.WriteLine($"Total Objects: {totalObjects:N0}"); + CommandLine.WriteLine($"Space Saved: {Formatter.FromBytes(savedBytes)} ({Math.Round(savedBytes / (double)expectedBytes * 100, 1)}%)"); if (vault.FileSystem.Service.Equals(FileService.Local)) { DriveInfo drive = new(vault.FileSystem.RootDirectory); long diskUsage = drive.TotalSize - drive.TotalFreeSpace; CommandLine.WriteLine($"Total Usage: {Formatter.FromBytes(diskUsage)} ({Math.Round(diskUsage / (double)drive.TotalSize * 100, 1)}%)"); - CommandLine.WriteLine($"Disk Usage: {Formatter.FromBytes(diskUsage - remoteSize)} ({Math.Round((diskUsage - remoteSize) / (double)drive.TotalSize * 100, 1)}%)"); CommandLine.WriteLine($"Disk Free: {Formatter.FromBytes(drive.TotalFreeSpace)} ({Math.Round(drive.TotalFreeSpace / (double)drive.TotalSize * 100, 1)}%)"); CommandLine.WriteLine($"Disk Total: {Formatter.FromBytes(drive.TotalSize)}"); } diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 10c9c67..f3292cc 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -156,6 +156,14 @@ public async Task> GetObjectsAsync(string id) return await connection.QueryAsync(sql, new { id }); } + /// + public async Task GetTotalObjectsAsync() + { + using IDbConnection connection = CreateConnection(); + string sql = $"SELECT COUNT(*) FROM objects;"; + return await connection.QuerySingleOrDefaultAsync(sql); + } + /// public async Task RemapObjectsAsync(string oldId, string newId) { diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 2bd0fba..643c8ef 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -82,7 +82,6 @@ public interface IDatabase Task GetFileAsync(string path); Task GetLocalSizeAsync(); - Task GetRemoteSizeAsync(); Task GetTotalFilesAsync(bool deleted); #endregion @@ -108,6 +107,8 @@ public interface IDatabase Task AddObjectAsync(string id, string hash, int index); Task> GetObjectsAsync(string id); + Task GetTotalObjectsAsync(); + #endregion Task RemapObjectsAsync(string oldId, string newId); diff --git a/Parallel.Core/Diagnostics/IProgressReporter.cs b/Parallel.Core/Diagnostics/IProgressReporter.cs index 943e0da..be7934f 100644 --- a/Parallel.Core/Diagnostics/IProgressReporter.cs +++ b/Parallel.Core/Diagnostics/IProgressReporter.cs @@ -7,8 +7,8 @@ namespace Parallel.Core.Diagnostics public enum ProgressOperation { Archiving, - Downloading, - Uploading, + Pulling, + Pushing, Compressing, Decompressing, Syncing diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index 05d6f6d..31035fc 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -110,7 +110,6 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string? parent = Path.GetDirectoryName(file.RemotePath); if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); - progress.Report(ProgressOperation.Uploading, file); await using FileStream openStream = File.OpenRead(file.LocalPath); await using FileStream createStream = File.Create(file.RemotePath); await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs index da67727..9841c27 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs @@ -119,7 +119,6 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres try { Stopwatch sw = new Stopwatch(); - progress.Report(ProgressOperation.Uploading, file); if (await _client.ExistsAsync(file.RemotePath)) _client.ChangePermissions(file.RemotePath, 644); string[] subDirs = file.RemotePath.Split('/'); diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index ade52c5..7b3bfdb 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -1,5 +1,6 @@ // Copyright 2025 Kyle Ebbinga +using System.Collections.Concurrent; using System.Reflection.Metadata; using Newtonsoft.Json.Linq; using Parallel.Core.Database; @@ -16,11 +17,12 @@ namespace Parallel.Core.IO.Syncing /// public class ObjectSyncManager : BaseSyncManager { + private static readonly ConcurrentDictionary _locks = new(); + /// /// The size, in bytes, to use for chunks of a file. /// - private static readonly int ChunkSize = 4194304; - + public readonly int ChunkSize = 4194304; /// /// Initializes a new instance of the class. @@ -33,39 +35,49 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - if (file.Deleted) - { - progress.Report(ProgressOperation.Archiving, file); - await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); - await Database.AddFileAsync(file); - } - else + try { - int bytesRead = 0; - byte[] buffer = new byte[ChunkSize]; - await using FileStream fs = File.OpenRead(file.LocalPath); - - int index = 0; - progress.Report(ProgressOperation.Uploading, file); - await Database.AddFileAsync(file); - - while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) + if (file.Deleted) { - await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); - string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); - await Database.AddObjectAsync(file.Id, hash, index); - index++; + progress.Report(ProgressOperation.Archiving, file); + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); + await Database.AddFileAsync(file); + } + else + { + int bytesRead = 0; + byte[] buffer = new byte[ChunkSize]; + await using FileStream fs = File.OpenRead(file.LocalPath); - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); - string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - if (!await FileSystem.ExistsAsync(remotePath)) + int index = 0; + progress.Report(ProgressOperation.Pushing, file); + await Database.AddFileAsync(file); + + while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) { - if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); - await FileSystem.UploadStreamAsync(ms, remotePath); + await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); + string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); + await Database.AddObjectAsync(file.Id, hash, index); + index++; + + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); + string remotePath = PathBuilder.Combine(parentDir, hash[4..]); + if (!await FileSystem.ExistsAsync(remotePath)) + { + if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); + await FileSystem.UploadStreamAsync(ms, remotePath); + } } + + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); } } + catch (Exception ex) + { + Log.Error(ex.GetBaseException().ToString()); + progress.Failed(ex, file); + } }); } @@ -74,22 +86,31 @@ public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - progress.Report(ProgressOperation.Downloading, file); - string? parentDir = Path.GetDirectoryName(file.LocalPath); - if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); - - await using FileStream fs = File.Create(file.LocalPath); - foreach (string hash in await Database.GetObjectsAsync(file.Id)) + try { - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await FileSystem.ExistsAsync(remotePath)) + progress.Report(ProgressOperation.Pulling, file); + string? parentDir = Path.GetDirectoryName(file.LocalPath); + if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); + + await using FileStream fs = File.Create(file.LocalPath); + foreach (string hash in await Database.GetObjectsAsync(file.Id)) { - await FileSystem.DownloadStreamAsync(fs, remotePath); + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); + if (await FileSystem.ExistsAsync(remotePath)) + { + await FileSystem.DownloadStreamAsync(fs, remotePath); + } } - } - await fs.FlushAsync(ct); + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pulled); + await fs.FlushAsync(ct); + } + catch (Exception ex) + { + Log.Error(ex.GetBaseException().ToString()); + progress.Failed(ex, file); + } }); } } From ac4f1ed9b6421e5bb6b3e892a4c0ef7154f9ab34 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 07:15:13 -0600 Subject: [PATCH 4/5] To avoid confusion between the IFileSystem and the FileSystemCredentials, IFileSystem is now IStorageProvider --- Parallel.Cli/Commands/DiskCommand.cs | 9 +----- Parallel.Core/Database/IDatabase.cs | 4 +-- Parallel.Core/IO/Blobs/BlobStorage.cs | 2 +- Parallel.Core/IO/PathBuilder.cs | 2 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 20 ++++++------ Parallel.Core/IO/Syncing/FileSyncManager.cs | 6 ++-- Parallel.Core/IO/Syncing/ISyncManager.cs | 4 +-- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 10 +++--- .../Settings/FileSystemCredentials.cs | 2 +- Parallel.Core/Settings/LocalVaultConfig.cs | 2 +- .../IStorageProvider.cs} | 3 +- .../LocalStorageProvider.cs} | 17 +++++----- .../SshStorageProvider.cs} | 31 +++---------------- .../StorageProvider.cs} | 8 ++--- Parallel.Core/Utils/Formatter.cs | 9 ++++-- 15 files changed, 51 insertions(+), 78 deletions(-) rename Parallel.Core/{IO/FileSystem/IFileSystem.cs => Storage/IStorageProvider.cs} (95%) rename Parallel.Core/{IO/FileSystem/DotNetFileSystem.cs => Storage/LocalStorageProvider.cs} (88%) rename Parallel.Core/{IO/FileSystem/SftpFileSystem.cs => Storage/SshStorageProvider.cs} (80%) rename Parallel.Core/{IO/FileSystem/FileSystemManager.cs => Storage/StorageProvider.cs} (81%) diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index 2ed79a1..ba713cd 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -49,21 +49,14 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) long totalDeletedFiles = await db.GetTotalFilesAsync(true); long totalObjects = await db.GetTotalObjectsAsync(); - int chunkSize = ((ObjectSyncManager)syncManager).ChunkSize; - double expectedChunks = (double)localSize / chunkSize; - long expectedBytes = (long)(expectedChunks * chunkSize); - long actualBytes = totalObjects * chunkSize; - long savedBytes = expectedBytes - actualBytes; - CommandLine.WriteLine($"Using vault '{vault.Name}' ({vault.Id}):"); CommandLine.WriteLine($"Service Type: {vault.FileSystem.Service}"); CommandLine.WriteLine($"Root Directory: {vault.FileSystem.RootDirectory}"); CommandLine.WriteLine($"Managed Files: {(totalLocalFiles + totalDeletedFiles):N0}"); CommandLine.WriteLine($"Local Files: {totalLocalFiles:N0}"); CommandLine.WriteLine($"Deleted Files: {totalDeletedFiles:N0}"); - CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); CommandLine.WriteLine($"Total Objects: {totalObjects:N0}"); - CommandLine.WriteLine($"Space Saved: {Formatter.FromBytes(savedBytes)} ({Math.Round(savedBytes / (double)expectedBytes * 100, 1)}%)"); + CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); if (vault.FileSystem.Service.Equals(FileService.Local)) { diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 643c8ef..2ef3b69 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -109,8 +109,8 @@ public interface IDatabase Task GetTotalObjectsAsync(); - #endregion - Task RemapObjectsAsync(string oldId, string newId); + + #endregion } } \ No newline at end of file diff --git a/Parallel.Core/IO/Blobs/BlobStorage.cs b/Parallel.Core/IO/Blobs/BlobStorage.cs index 668a8a3..f34413f 100644 --- a/Parallel.Core/IO/Blobs/BlobStorage.cs +++ b/Parallel.Core/IO/Blobs/BlobStorage.cs @@ -23,7 +23,7 @@ public abstract class BlobStorage /// The temp directory to send chunked objects to. /// /// - public static async Task CreateManifestAsync(IFileSystem fileSystem, string sourcePath, string tempObjDir, IProgressReporter progress) + public static async Task CreateManifestAsync(IStorageProvider fileSystem, string sourcePath, string tempObjDir, IProgressReporter progress) { List chunkHashes = new List(); await using FileStream fs = File.OpenRead(sourcePath); diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 55ad1c3..fff8c25 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -142,7 +142,7 @@ public static string GetDatabaseFile(LocalVaultConfig localVault) } /// - /// Builds the path on the remote . + /// Builds the path on the remote . /// /// /// diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 4280bdf..f368ab2 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -27,7 +27,7 @@ public abstract class BaseSyncManager : ISyncManager public IDatabase Database { get; set; } /// - public IFileSystem FileSystem { get; set; } + public IStorageProvider Storage { get; set; } /// /// @@ -35,7 +35,7 @@ public abstract class BaseSyncManager : ISyncManager /// public BaseSyncManager(LocalVaultConfig localVault) { - FileSystem = FileSystemManager.CreateNew(localVault); + Storage = StorageProvider.CreateNew(localVault); LocalVault = localVault; } @@ -43,13 +43,13 @@ public BaseSyncManager(LocalVaultConfig localVault) public async Task ConnectAsync() { string root = PathBuilder.GetRootDirectory(LocalVault); - if (!await FileSystem.ExistsAsync(root)) + if (!await Storage.ExistsAsync(root)) { - await FileSystem.CreateDirectoryAsync(root); + await Storage.CreateDirectoryAsync(root); Log.Debug($"Created root directory: {root}"); } - if (!await FileSystem.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) + if (!await Storage.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) { RemoteVault = new RemoteVaultConfig(LocalVault); RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); @@ -59,7 +59,7 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); + await Storage.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); if(config == null) return false; RemoteVault = config; @@ -67,7 +67,7 @@ public async Task ConnectAsync() Log.Debug($"Downloaded config file: {TempConfigFile}"); } - if (!await FileSystem.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) + if (!await Storage.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) { Database = new SqliteContext(TempDbFile); await Database.InitializeAsync(); @@ -76,7 +76,7 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); + await Storage.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); Database = new SqliteContext(TempDbFile); Log.Debug($"Downloaded db file: {TempDbFile}"); @@ -92,8 +92,8 @@ public async Task DisconnectAsync() Log.Debug($"Uploaded db file: {TempDbFile}"); SystemFile[] tempFiles = [new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault)), new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))]; - await FileSystem.UploadFilesAsync(tempFiles, new NullProgressReporter()); - FileSystem.Dispose(); + await Storage.UploadFilesAsync(tempFiles, new NullProgressReporter()); + Storage.Dispose(); } /// diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index d069ad6..7cfc03f 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -24,7 +24,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter if (!files.Any()) return; SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); Log.Information($"Backing up {backupFiles.Length} files..."); - await FileSystem.UploadFilesAsync(backupFiles, progress); + await Storage.UploadFilesAsync(backupFiles, progress); progress.Reset(); await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => @@ -38,7 +38,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options else { progress.Report(ProgressOperation.Syncing, file); - SystemFile? remote = await FileSystem.GetFileAsync(file.RemotePath); + SystemFile? remote = await Storage.GetFileAsync(file.RemotePath); if (remote is not null) { file.RemoteSize = remote.RemoteSize; @@ -52,7 +52,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options /// public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) { - await FileSystem.DownloadFilesAsync(files, progress); + await Storage.DownloadFilesAsync(files, progress); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 15d1395..80f93f6 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -31,10 +31,10 @@ public interface ISyncManager /// /// The associated file system connection. /// - IFileSystem FileSystem { get; set; } + public IStorageProvider Storage { get; set; } /// - /// Establishes a connection to the associated and downloads the needed files. + /// Establishes a connection to the associated and downloads the needed files. /// Task ConnectAsync(); diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index 7b3bfdb..7d464cb 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -63,10 +63,10 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - if (!await FileSystem.ExistsAsync(remotePath)) + if (!await Storage.ExistsAsync(remotePath)) { - if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); - await FileSystem.UploadStreamAsync(ms, remotePath); + if (!await Storage.ExistsAsync(parentDir)) await Storage.CreateDirectoryAsync(parentDir); + await Storage.UploadStreamAsync(ms, remotePath); } } @@ -97,9 +97,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await FileSystem.ExistsAsync(remotePath)) + if (await Storage.ExistsAsync(remotePath)) { - await FileSystem.DownloadStreamAsync(fs, remotePath); + await Storage.DownloadStreamAsync(fs, remotePath); } } diff --git a/Parallel.Core/Settings/FileSystemCredentials.cs b/Parallel.Core/Settings/FileSystemCredentials.cs index 8000bf9..ac7d4f9 100644 --- a/Parallel.Core/Settings/FileSystemCredentials.cs +++ b/Parallel.Core/Settings/FileSystemCredentials.cs @@ -7,7 +7,7 @@ namespace Parallel.Core.Settings { /// - /// Represents credentials used to gain access to various s. + /// Represents credentials used to gain access to various s. /// public class FileSystemCredentials { diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index 76f5652..82b0077 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -21,7 +21,7 @@ public class LocalVaultConfig public string Name { get; set; } /// - /// The credentials needed to log in to the associated . + /// The credentials needed to log in to the associated . /// public FileSystemCredentials FileSystem { get; } diff --git a/Parallel.Core/IO/FileSystem/IFileSystem.cs b/Parallel.Core/Storage/IStorageProvider.cs similarity index 95% rename from Parallel.Core/IO/FileSystem/IFileSystem.cs rename to Parallel.Core/Storage/IStorageProvider.cs index 74e6f8e..7c6ce3f 100644 --- a/Parallel.Core/IO/FileSystem/IFileSystem.cs +++ b/Parallel.Core/Storage/IStorageProvider.cs @@ -14,7 +14,7 @@ namespace Parallel.Core.IO.FileSystem /// /// Defines the way for communicating with a file system. /// - public interface IFileSystem : IDisposable + public interface IStorageProvider : IDisposable { /// /// Creates all directories and subdirectories in the specified path unless they already exist. @@ -67,6 +67,7 @@ public interface IFileSystem : IDisposable /// /// /// + /// The size, in bytes, of the uploaded stream. Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress); /// diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/Storage/LocalStorageProvider.cs similarity index 88% rename from Parallel.Core/IO/FileSystem/DotNetFileSystem.cs rename to Parallel.Core/Storage/LocalStorageProvider.cs index 31035fc..15f87e1 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/Storage/LocalStorageProvider.cs @@ -16,15 +16,15 @@ namespace Parallel.Core.IO.FileSystem /// /// Represents the wrapper for a default dotnet file system. /// - public class DotNetFileSystem : IFileSystem + public class LocalStorageProvider : IStorageProvider { private readonly LocalVaultConfig _vaultConfig; /// - /// Represents an for interacting with physical machine hardware. + /// Represents an for interacting with physical machine hardware. /// /// The vault to use. - public DotNetFileSystem(LocalVaultConfig vaultConfig) + public LocalStorageProvider(LocalVaultConfig vaultConfig) { _vaultConfig = vaultConfig; } @@ -49,12 +49,9 @@ public Task DeleteDirectoryAsync(string path) /// public Task DeleteFileAsync(string path) { - if (File.Exists(path)) - { - File.SetAttributes(path, ~FileAttributes.ReadOnly & File.GetAttributes(path)); - Microsoft.VisualBasic.FileIO.FileSystem.DeleteFile(path, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin); - } - + if (!File.Exists(path)) return Task.CompletedTask; + File.SetAttributes(path, ~FileAttributes.ReadOnly & File.GetAttributes(path)); + File.Delete(path); return Task.CompletedTask; } @@ -131,7 +128,7 @@ public async Task UploadStreamAsync(Stream input, string remotePath) await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); await input.CopyToAsync(gzipStream); - //File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); + File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/Storage/SshStorageProvider.cs similarity index 80% rename from Parallel.Core/IO/FileSystem/SftpFileSystem.cs rename to Parallel.Core/Storage/SshStorageProvider.cs index 9841c27..28847c2 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/Storage/SshStorageProvider.cs @@ -17,16 +17,16 @@ namespace Parallel.Core.IO.FileSystem /// /// Represents the wrapper for an SFTP file system through SSH. /// - public class SftpFileSystem : IFileSystem + public class SshStorageProvider : IStorageProvider { private readonly ConnectionInfo _connectionInfo; private readonly SftpClient _client; /// - /// Represents an for interacting with an SSH server. + /// Represents an for interacting with an SSH server. /// /// The credentials to log in with. - public SftpFileSystem(LocalVaultConfig localVault) + public SshStorageProvider(LocalVaultConfig localVault) { _connectionInfo = new ConnectionInfo(localVault.FileSystem.Address, localVault.FileSystem.Username, new PasswordAuthenticationMethod(localVault.FileSystem.Username, Encryption.Decode(localVault.FileSystem.Password))); _client = new SftpClient(_connectionInfo); @@ -144,31 +144,10 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres public async Task UploadStreamAsync(Stream input, string remotePath) { await using SftpFileStream createStream = _client.Create(remotePath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await using GZipStream gzipStream = new(createStream, CompressionLevel.SmallestSize); await input.CopyToAsync(gzipStream); - //_client.ChangePermissions(remotePath, 444); - } - - /// - public async Task UploadFileAsync(string sourcePath, string destinationPath) - { - if (await ExistsAsync(destinationPath)) _client.ChangePermissions(destinationPath, 644); - - string parentDir = string.Empty; - foreach (string subPath in destinationPath.Split('/')) - { - parentDir += $"/{subPath}"; - if (!await _client.ExistsAsync(parentDir)) - { - await _client.CreateDirectoryAsync(parentDir); - } - } - await using SftpFileStream createStream = _client.Create(destinationPath); - await using FileStream openStream = File.OpenRead(sourcePath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); - _client.ChangePermissions(destinationPath, 444); + _client.ChangePermissions(remotePath, 444); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/FileSystemManager.cs b/Parallel.Core/Storage/StorageProvider.cs similarity index 81% rename from Parallel.Core/IO/FileSystem/FileSystemManager.cs rename to Parallel.Core/Storage/StorageProvider.cs index adb410f..9ad1f64 100644 --- a/Parallel.Core/IO/FileSystem/FileSystemManager.cs +++ b/Parallel.Core/Storage/StorageProvider.cs @@ -28,18 +28,18 @@ public enum FileService /// /// Represents the way to connect to different file system associations. This class cannot be inherited. /// - public static class FileSystemManager + public static class StorageProvider { /// /// Creates a new file system association. /// /// The vault needed for the associated file system. - public static IFileSystem CreateNew(LocalVaultConfig vaultConfig) + public static IStorageProvider CreateNew(LocalVaultConfig vaultConfig) { return vaultConfig.FileSystem.Service switch { - FileService.Local => new DotNetFileSystem(vaultConfig), - FileService.Remote => new SftpFileSystem(vaultConfig), + FileService.Local => new LocalStorageProvider(vaultConfig), + FileService.Remote => new SshStorageProvider(vaultConfig), //FileService.Cloud => new AmazonS3FileSystem(credentials), _ => null }; diff --git a/Parallel.Core/Utils/Formatter.cs b/Parallel.Core/Utils/Formatter.cs index 411d22d..daed03b 100644 --- a/Parallel.Core/Utils/Formatter.cs +++ b/Parallel.Core/Utils/Formatter.cs @@ -12,11 +12,11 @@ public class Formatter /// /// The bytes to convert. /// The bytes formatted as a string. - public static string FromBytes(long bytes) + public static string FromBytes(double bytes) { string[] sizeSuffixes = { "Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" }; int sizeIndex = 0; - double size = bytes; + double size = Math.Abs(bytes); // use absolute for scaling while (size >= 1000 && sizeIndex < sizeSuffixes.Length - 1) { @@ -24,9 +24,12 @@ public static string FromBytes(long bytes) size /= 1000; } - return $"{size:N2} {sizeSuffixes[sizeIndex]}"; + // add the sign back + string sign = bytes < 0 ? "-" : ""; + return $"{sign}{size:N2} {sizeSuffixes[sizeIndex]}"; } + /// /// Formats a with the corresponding data volume. /// From c54c59836fc188b2a5bbfd24ec5d95173a333e60 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Wed, 10 Dec 2025 01:02:03 -0600 Subject: [PATCH 5/5] Added enabled/disable option --- Parallel.Core/Settings/LocalVaultConfig.cs | 5 +++++ Parallel.Core/Settings/ParallelConfig.cs | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index 82b0077..82d8203 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -20,6 +20,11 @@ public class LocalVaultConfig /// public string Name { get; set; } + /// + /// If the current vault config is enabled. + /// + public bool Enabled { get; set; } = true; + /// /// The credentials needed to log in to the associated . /// diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index 17d382a..dc95650 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -26,6 +26,8 @@ public class ParallelConfig MaxDegreeOfParallelism = Load().MaxConcurrentProcesses }; + public static int MaxUploads { get; } = Load().MaxConcurrentUploads; + // /// // /// The address that will accept incoming commands. // /// Default: 127.0.0.1 @@ -46,9 +48,15 @@ public class ParallelConfig /// /// Gets or sets the maximum number of concurrent processes that can run. - /// Default: Half the processor count. + /// Default: The processor count. + /// + public int MaxConcurrentProcesses { get; set; } = Environment.ProcessorCount; + + /// + /// Gets or sets the maximum number of concurrent processes that can run. + /// Default: 4 /// - public int MaxConcurrentProcesses { get; set; } = Math.Clamp(Environment.ProcessorCount / 2, 1, Environment.ProcessorCount); + public int MaxConcurrentUploads { get; set; } = 4; /// /// The amount of time, in days, to hold a file before it can be cleaned. @@ -118,7 +126,7 @@ public async Task ForEachVaultAsync(Func actionAsync, Ca CancellationToken = cancellationToken }; - await System.Threading.Tasks.Parallel.ForEachAsync(Vaults, options, async (vault, ct) => + await System.Threading.Tasks.Parallel.ForEachAsync(Vaults.Where(v => v.Enabled), options, async (vault, ct) => { await actionAsync(vault); });