Differences
This shows you the differences between the selected revisions of the page.
| 2017-10-19 | 2017-10-19 | ||
| syntax (martin) | upload (martin) | ||
| Line 1: | Line 1: | ||
| - | ====== Automating download in parallel connections over SFTP/FTP protocol ====== | + | ====== Automating transfers in parallel connections over SFTP/FTP protocol ====== |
| - | ===== [[csharp]] C# ===== | + | ===== [[csharp]] Download (C#) ===== |
| The example opens by default three parallel connections and uses them to download remote file tree to local folder in parallel. | The example opens by default three parallel connections and uses them to download remote file tree to local folder in parallel. | ||
| Line 119: | Line 119: | ||
| </code> | </code> | ||
| - | ===== PowerShell ===== | + | ===== Download (PowerShell) ===== |
| The code is not equivalent to the C# example above. The PowerShell code does not download subdirectories. It also split the files to batches by their count only, instead of using a queue like the C# code. | The code is not equivalent to the C# example above. The PowerShell code does not download subdirectories. It also split the files to batches by their count only, instead of using a queue like the C# code. | ||
| Line 278: | Line 278: | ||
| </code> | </code> | ||
| + | ===== [[upload_csharp]] Upload (C#) ===== | ||
| + | The example opens by default three parallel connections and uses them to upload locale file tree to remote folder in parallel. | ||
| + | |||
| + | <code csharp> | ||
| + | using System; | ||
| + | using System.Collections.Generic; | ||
| + | using System.IO; | ||
| + | using System.Threading.Tasks; | ||
| + | using WinSCP; | ||
| + | |||
| + | class Example | ||
| + | { | ||
| + | static int Main() | ||
| + | { | ||
| + | try | ||
| + | { | ||
| + | // Setup session options | ||
| + | SessionOptions sessionOptions = new SessionOptions | ||
| + | { | ||
| + | Protocol = Protocol.Sftp, | ||
| + | HostName = "example.com", | ||
| + | UserName = "user", | ||
| + | Password = "password", | ||
| + | SshHostKeyFingerprint = "ssh-rsa 2048 xx:xx:xx:xx:xx:xx:xx:xx..." | ||
| + | }; | ||
| + | |||
| + | const string localPath = "C:\\local\\path"; | ||
| + | const string remotePath = "/remote/path"; | ||
| + | const int batches = 3; | ||
| + | |||
| + | DateTime started = DateTime.Now; | ||
| + | int count = 0; | ||
| + | Int64 bytes = 0; | ||
| + | |||
| + | Console.WriteLine("Starting files enumeration..."); | ||
| + | IEnumerable<string> files = | ||
| + | Directory.EnumerateFiles(localPath, "*.*", SearchOption.AllDirectories); | ||
| + | IEnumerator<string> filesEnumerator = files.GetEnumerator(); | ||
| + | |||
| + | List<Task> tasks = new List<Task>(); | ||
| + | |||
| + | HashSet<string> existingRemotePaths = new HashSet<string>(); | ||
| + | |||
| + | for (int i = 1; i <= batches; i++) | ||
| + | { | ||
| + | int no = i; | ||
| + | |||
| + | Task task = new Task(() => | ||
| + | { | ||
| + | using (Session uploadSession = new Session()) | ||
| + | { | ||
| + | while (true) | ||
| + | { | ||
| + | string localFilePath; | ||
| + | lock (filesEnumerator) | ||
| + | { | ||
| + | if (!filesEnumerator.MoveNext()) | ||
| + | { | ||
| + | break; | ||
| + | } | ||
| + | |||
| + | localFilePath = filesEnumerator.Current; | ||
| + | bytes += new FileInfo(localFilePath).Length; | ||
| + | count++; | ||
| + | } | ||
| + | |||
| + | if (!uploadSession.Opened) | ||
| + | { | ||
| + | Console.WriteLine("Starting upload {0}...", no); | ||
| + | uploadSession.Open(sessionOptions); | ||
| + | } | ||
| + | |||
| + | string remoteFilePath = | ||
| + | uploadSession.TranslateLocalPathToRemote( | ||
| + | localFilePath, localPath, remotePath); | ||
| + | Console.WriteLine( | ||
| + | "Uploading {0} to {1} in {2}...", | ||
| + | localFilePath, remoteFilePath, no); | ||
| + | |||
| + | string path = | ||
| + | remoteFilePath.Substring(0, remoteFilePath.LastIndexOf('/')); | ||
| + | string current = ""; | ||
| + | |||
| + | if (path.Substring(0, 1) == "/") | ||
| + | { | ||
| + | path = path.Substring(1); | ||
| + | } | ||
| + | |||
| + | while (!string.IsNullOrEmpty(path)) | ||
| + | { | ||
| + | int p = path.IndexOf('/'); | ||
| + | current += '/'; | ||
| + | if (p >= 0) | ||
| + | { | ||
| + | current += path.Substring(0, p); | ||
| + | path = path.Substring(p + 1); | ||
| + | } | ||
| + | else | ||
| + | { | ||
| + | current += path; | ||
| + | path = ""; | ||
| + | } | ||
| + | |||
| + | lock (existingRemotePaths) | ||
| + | { | ||
| + | if (!existingRemotePaths.Contains(current)) // optimization | ||
| + | { | ||
| + | if (!uploadSession.FileExists(current)) | ||
| + | { | ||
| + | Console.WriteLine("Creating {0}...", current); | ||
| + | uploadSession.CreateDirectory(current); | ||
| + | } | ||
| + | existingRemotePaths.Add(current); | ||
| + | } | ||
| + | } | ||
| + | } | ||
| + | |||
| + | uploadSession.PutFiles( | ||
| + | localFilePath, uploadSession.EscapeFileMask(remoteFilePath)). | ||
| + | Check(); | ||
| + | } | ||
| + | |||
| + | if (uploadSession.Opened) | ||
| + | { | ||
| + | Console.WriteLine("Upload {0} done", no); | ||
| + | } | ||
| + | else | ||
| + | { | ||
| + | Console.WriteLine("Upload {0} had nothing to do", no); | ||
| + | } | ||
| + | } | ||
| + | |||
| + | }); | ||
| + | |||
| + | tasks.Add(task); | ||
| + | task.Start(); | ||
| + | } | ||
| + | |||
| + | Console.WriteLine("Waiting for uploads to complete..."); | ||
| + | Task.WaitAll(tasks.ToArray()); | ||
| + | |||
| + | Console.WriteLine("Done"); | ||
| + | |||
| + | DateTime ended = DateTime.Now; | ||
| + | Console.WriteLine("Took {0}", (ended - started)); | ||
| + | Console.WriteLine("Uploaded {0} files, totaling {1:N0} bytes", count, bytes); | ||
| + | |||
| + | return 0; | ||
| + | } | ||
| + | catch (Exception e) | ||
| + | { | ||
| + | Console.WriteLine("Error: {0}", e); | ||
| + | return 1; | ||
| + | } | ||
| + | } | ||
| + | } | ||
| + | </code> | ||