Feature/blobs

Parallel.Cli/Commands/CleanCommand.cs

@@ -2,7 +2,6 @@
 
 using System.CommandLine;
 using Parallel.Cli.Utils;
-using Parallel.Core.IO.Backup;
 using Parallel.Core.IO.Scanning;
 using Parallel.Core.IO.Syncing;
 using Parallel.Core.Settings;
@@ -13,6 +12,8 @@ namespace Parallel.Cli.Commands
     public class CleanCommand : Command
     {
         private long _freedBytes = 0;
+        private int _filesCount = 0;
+        private int _dirsCount = 0;
 
         private readonly Option<string> _sourceOpt = new(["--path", "-p"], "The source path to clean.");
         private readonly Option<int> _daysOpt = new(["--days", "-d"], "The amount of days to hang onto files.");
@@ -39,6 +40,7 @@ public CleanCommand() : base("clean", "Cleans up the file system by removing old
                     await CleanDirectoryAsync(config, path, days, recursive, verbose);
                 }
 
+                CommandLine.WriteLine($"Successfully cleaned {_filesCount:N0} files and {_dirsCount:N0} directories, ({Formatter.FromBytes(_freedBytes)} removed)", ConsoleColor.Green);
             }, _sourceOpt, _daysOpt, _recursiveOpt, _verboseOpt);
         }
 
@@ -62,7 +64,6 @@ private async Task CleanDirectoryAsync(ParallelConfig config, string path, int d
             UnixTime minTime = UnixTime.FromMilliseconds(UnixTime.Now.TotalMilliseconds - (days * UnixTime.Day));
             IEnumerable<FileInfo> cleanableFiles = FileScanner.GetCleanableFiles(path, minTime, recursive);
             if (!cleanableFiles.Any()) CommandLine.WriteLine($"No cleanable files were found in the provided path.", ConsoleColor.Green);
-            int filesCount = cleanableFiles.Count();
 
             await System.Threading.Tasks.Parallel.ForEachAsync(cleanableFiles, ParallelConfig.Options, (fi, ct) =>
             {
@@ -71,6 +72,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(cleanableFiles, ParallelConfi
                     try
                     {
                         _freedBytes += fi.Length;
+                        _filesCount++;
                         fi.Delete();
                     }
                     catch (Exception ex)
@@ -85,11 +87,10 @@ await System.Threading.Tasks.Parallel.ForEachAsync(cleanableFiles, ParallelConfi
 
             IEnumerable<DirectoryInfo> directories = FileScanner.GetEmptyDirectories(path, recursive);
             if (!directories.Any()) CommandLine.WriteLine($"No empty directories were found in the provided path.", ConsoleColor.Green);
-            int directoriesCount = directories.Count();
 
             await System.Threading.Tasks.Parallel.ForEachAsync(directories, ParallelConfig.Options, (di, ct) =>
             {
-                if (di.Exists && di.EnumerateFiles().Any())
+                if (di.Exists && !di.EnumerateFiles().Any())
                 {
                     try
                     {
@@ -98,7 +99,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(directories, ParallelConfig.O
                     }
                     catch (Exception ex)
                     {
-                        Log.Warning($"{ex.GetBaseException().Message}");
+                        Log.Error($"{ex.GetBaseException().Message}");
                     }
                 }
 
@@ -113,15 +114,13 @@ await System.Threading.Tasks.Parallel.ForEachAsync(directories, ParallelConfig.O
                 {
                     Log.Debug($"Removing empty directory: {currentDir.FullName}");
                     currentDir.Delete(true);
-                    directoriesCount++;
+                    _dirsCount++;
                 }
                 catch (Exception ex)
                 {
                     Log.Warning($"{ex.GetBaseException().Message}");
                 }
             }
-
-            if(filesCount > 0 || directoriesCount > 0) CommandLine.WriteLine($"Successfully cleaned {filesCount:N0} files and {directoriesCount:N0} directories, ({Formatter.FromBytes(_freedBytes)} removed)", ConsoleColor.Green);
         }
     }
 }

Parallel.Cli/Commands/DiskCommand.cs

@@ -5,7 +5,6 @@
 using Newtonsoft.Json.Linq;
 using Parallel.Cli.Utils;
 using Parallel.Core.Database;
-using Parallel.Core.IO.Backup;
 using Parallel.Core.IO.FileSystem;
 using Parallel.Core.IO.Syncing;
 using Parallel.Core.Settings;
@@ -36,7 +35,7 @@ public DiskCommand() : base("disk", "Shows the current disk usage.")
 
         private async Task DisplayDiskInformationAsync(LocalVaultConfig vault)
         {
-            ISyncManager syncManager = new FileSyncManager(vault);
+            ISyncManager syncManager = SyncManager.CreateNew(vault);
             if (!await syncManager.ConnectAsync())
             {
                 CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red);

Parallel.Cli/Commands/DuplicatesCommand.cs

@@ -2,7 +2,6 @@
 
 using System.CommandLine;
 using Parallel.Cli.Utils;
-using Parallel.Core.IO.Backup;
 using Parallel.Core.IO.Scanning;
 using Parallel.Core.Models;
 using Parallel.Core.Settings;

Parallel.Cli/Commands/PullCommand.cs

@@ -1,9 +1,93 @@
 // Copyright 2025 Kyle Ebbinga
 
+using System.CommandLine;
+using Parallel.Cli.Utils;
+using Parallel.Core.Diagnostics;
+using Parallel.Core.IO;
+using Parallel.Core.IO.Scanning;
+using Parallel.Core.IO.Syncing;
+using Parallel.Core.Models;
+using Parallel.Core.Settings;
+
 namespace Parallel.Cli.Commands
 {
-    public class PullCommand
+    public class PullCommand : Command
     {
-
+        private readonly Option<string> _sourceArg = new(["--path", "-p"], "The source path to sync.");
+        private readonly Option<string> _configOpt = new(["--config", "-c"], "The vault configuration to use.");
+        private readonly Option<bool> _forceOpt = new(["--force", "-f"], "Forces the pull overwriting any files.");
+
+        public PullCommand() : base("pull", "Pulls changes from a vault.")
+        {
+            this.AddOption(_sourceArg);
+            this.AddOption(_configOpt);
+            this.AddOption(_forceOpt);
+            this.SetHandler(async (path, config, force) =>
+            {
+                LocalVaultConfig? vault = ParallelConfig.GetVault(config);
+                if (vault == null)
+                {
+                    CommandLine.WriteLine($"Unable to find vault with name: '{vault}'", ConsoleColor.Yellow);
+                    return;
+                }
+
+                await PullPathAsync(vault, path, force);
+            }, _sourceArg, _configOpt, _forceOpt);
+        }
+
+        private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force)
+        {
+            ISyncManager syncManager = SyncManager.CreateNew(vault);
+            if (!await syncManager.ConnectAsync())
+            {
+                CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red);
+                return;
+            }
+
+            string fullPath = Path.GetFullPath(path);
+            if (PathBuilder.IsFile(fullPath))
+            {
+                await PullFileAsync(syncManager, fullPath, force);
+                return;
+            }
+
+            IEnumerable<SystemFile> files = await syncManager.Database.GetFilesAsync(fullPath);
+            if (!files.Any())
+            {
+                CommandLine.WriteLine("The provided directory has not been pushed!", ConsoleColor.Yellow);
+                return;
+            }
+
+            List<SystemFile> pullFiles = new List<SystemFile>();
+            await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) =>
+            {
+                if (!File.Exists(file.LocalPath) || FileScanner.HasChanged(file, new SystemFile(file.LocalPath)) || force) pullFiles.Add(file);
+            });
+
+            Log.Debug($"Pulling {pullFiles.Count} files...");
+            await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressLogger());
+            CommandLine.WriteLine(vault, $"Successfully pulled {pullFiles.Count:N0} files from '{vault.FileSystem.RootDirectory}'.", ConsoleColor.Green);
+        }
+
+        private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool force)
+        {
+            SystemFile? remoteFile = await syncManager.Database.GetFileAsync(fullPath);
+            if (remoteFile == null)
+            {
+                CommandLine.WriteLine("The provided file has not been pushed!", ConsoleColor.Yellow);
+                return;
+            }
+
+            SystemFile localFile = new SystemFile(fullPath);
+            if (!(FileScanner.HasChanged(localFile, remoteFile) || force))
+            {
+                CommandLine.WriteLine("Cannot overwrite an existing file!", ConsoleColor.Yellow);
+                return;
+            }
+
+            Log.Debug($"Pulling '{fullPath}'");
+            await syncManager.PullFilesAsync([remoteFile], new ProgressLogger());
+            CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pulled file from '{syncManager.RemoteVault.FileSystem.RootDirectory}'.", ConsoleColor.Green);
+        }
     }
 }

Parallel.Cli/Commands/PushCommand.cs

@@ -4,7 +4,6 @@
 using Parallel.Cli.Utils;
 using Parallel.Core.Diagnostics;
 using Parallel.Core.IO;
-using Parallel.Core.IO.Backup;
 using Parallel.Core.IO.Scanning;
 using Parallel.Core.IO.Syncing;
 using Parallel.Core.Models;
@@ -22,7 +21,7 @@ public class PushCommand : Command
         private readonly Option<string> _configOpt = new(["--config", "-c"], "The vault configuration to use.");
         private readonly Option<bool> _verboseOpt = new(["--verbose", "-v"], "Shows verbose output.");
 
-        public PushCommand() : base("push", "Pushes changed files to one vault or multiple.")
+        public PushCommand() : base("push", "Pushes changed files to vaults.")
         {
             this.AddOption(_sourceArg);
             this.AddOption(_configOpt);
@@ -50,7 +49,7 @@ private async Task SyncPathAsync(string path)
         {
             await Program.Settings.ForEachVaultAsync(async vault =>
             {
-                ISyncManager syncManager = new FileSyncManager(vault);
+                ISyncManager syncManager = SyncManager.CreateNew(vault);
                 if (!await syncManager.ConnectAsync())
                 {
                     CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red);

Parallel.Cli/Commands/VaultsCommand.cs

@@ -3,7 +3,6 @@
 using System.CommandLine;
 using Parallel.Cli.Utils;
 using Parallel.Core.Database;
-using Parallel.Core.IO.Backup;
 using Parallel.Core.IO.FileSystem;
 using Parallel.Core.IO.Syncing;
 using Parallel.Core.Security;
@@ -82,7 +81,7 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.")
                     return;
                 }
 
-                ISyncManager syncManager = new FileSyncManager(config);
+                ISyncManager syncManager = SyncManager.CreateNew(config);
                 if (!await syncManager.ConnectAsync())
                 {
                     CommandLine.WriteLine(config, $"Failed to connect to vault '{config.Name}'!", ConsoleColor.Red);

Parallel.Cli/Program.cs

@@ -16,8 +16,11 @@ public static async Task Main(string[] args)
             Settings = ParallelConfig.Load();
             string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", "latest.txt");
             if (File.Exists(logFile)) File.Delete(logFile);
+            #if DEBUG
+            Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.Console().CreateLogger();
+            #else
             Log.Logger = new LoggerConfiguration().MinimumLevel.Warning().WriteTo.File(logFile).CreateLogger();
-            //Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.Console().CreateLogger();
+            #endif
 
             AssemblyName assembly = Assembly.GetExecutingAssembly().GetName();
             Log.Information($"{assembly.Name} [Version {assembly.Version}]");

Parallel.Core/Database/Contexts/SqliteContext.cs

@@ -82,6 +82,14 @@ public async Task<long> GetTotalFilesAsync(bool deleted)
             return await connection.QuerySingleOrDefaultAsync<long>(sql, new { deleted });
         }
 
+        /// <inheritdoc />
+        public async Task<IEnumerable<SystemFile>> GetFilesAsync(string path)
+        {
+            using IDbConnection connection = CreateConnection();
+            string sql = $"SELECT * FROM files WHERE localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC";
+            return await connection.QueryAsync<SystemFile>(sql);
+        }
+
         /// <inheritdoc />
         public async Task<IEnumerable<SystemFile>> GetFilesAsync(string path, bool deleted)
         {

Parallel.Core/Database/IDatabase.cs

@@ -96,6 +96,7 @@ public interface IDatabase : IDisposable
 
         #endregion
 
+        Task<IEnumerable<SystemFile>> GetFilesAsync(string path);
         Task<IEnumerable<SystemFile>> GetFilesAsync(string path, bool deleted);
         Task<SystemFile?> GetFileAsync(string path);
     }

Parallel.Core/IO/Blobs/BlobStorage.cs

@@ -0,0 +1,81 @@
+// Copyright 2025 Kyle Ebbinga
+
+using Parallel.Core.Diagnostics;
+using Parallel.Core.Security;
+
+namespace Parallel.Core.IO.Blobs
+{
+    /// <summary>
+    /// Represents the way to chunk files into blobs for syncing.
+    /// </summary>
+    public class BlobStorage
+    {
+        /// <summary>
+        /// The size, in bytes, to use for chunks of a file.
+        /// </summary>
+        public int ChunkSize { get; set; }
+
+        /// <summary>
+        /// Gets the temp directory for storing blobs.
+        /// </summary>
+        public string TempDirectory { get; set; }
+
+        public BlobStorage(string tempDir, int chunkSize = 4194304)
+        {
+            TempDirectory = tempDir;
+            ChunkSize = chunkSize;
+        }
+
+        /// <summary>
+        /// Chunks a file into hashes for blob storage.
+        /// </summary>
+        /// <param name="sourcePath">The source path of the file.</param>
+        /// <param name="destPath">The destination to send chunked objects to.</param>
+        /// <param name="progress"></param>
+        /// <returns></returns>
+        public async Task<IEnumerable<string>> ChunkFileAsync(string sourcePath, string destPath, IProgressReporter progress)
+        {
+            List<string> chunkHashes = new List<string>();
+            await using FileStream fs = File.OpenRead(sourcePath);
+            byte[] buffer = new byte[ChunkSize];
+            int bytesRead = 0;
+
+            while ((bytesRead = await fs.ReadAsync(buffer)) > 0)
+            {
+                byte[] chunkData = new byte[bytesRead];
+                Buffer.BlockCopy(buffer, 0, chunkData, 0, bytesRead);
+
+                string hash = HashGenerator.CreateSHA256(chunkData);
+                string chunkPath = PathBuilder.GetObjectPath(destPath, hash);
+
+                if(!File.Exists(chunkPath)) await File.WriteAllBytesAsync(chunkPath, chunkData);
+                chunkHashes.Add(hash);
+            }
+
+            Log.Debug($"Wrote {chunkHashes.Count} hashes to {destPath}");
+            return chunkHashes;
+        }
+
+        /// <summary>
+        /// Assembles a file from the chunked hashes.
+        /// </summary>
+        /// <param name="hashes"></param>
+        /// <param name="sourcePath">The path to the chunked objects' folder.</param>
+        /// <param name="createFilePath"></param>
+        public async Task AssembleFileAsync(IEnumerable<string> chunkHashes, string sourcePath, string createFilePath)
+        {
+            Log.Debug($"Assembling '{createFilePath}' from {chunkHashes.Count()} hashes.");
+            await using FileStream createStream = File.Create(createFilePath);
+            foreach (string hash in chunkHashes)
+            {
+                string chunkPath = PathBuilder.GetObjectPath(sourcePath, hash);
+                if(!File.Exists(chunkPath)) throw new FileNotFoundException($"Missing chunk for hash: {hash}");
+
+                await using FileStream chunkStream = File.OpenRead(chunkPath);
+                await chunkStream.CopyToAsync(createStream);
+            }
+
+            await createStream.FlushAsync();
+        }
+    }
+}

Parallel.Core/IO/PathBuilder.cs

@@ -167,5 +167,11 @@ public static bool IsFile(string path)
         {
             return !Directory.Exists(path) && File.Exists(path);
         }
+
+        public static string GetObjectPath(string basePath, string hash)
+        {
+            if (hash.Length < 8) throw new ArgumentException("Hash too short for sharding", nameof(hash));
+            return Path.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash.Substring(4, 2), hash.Substring(6, 2), hash);
+        }
     }
 }