Differences
This shows you the differences between the selected revisions of the page.
| 2016-06-30 | 2017-02-10 | ||
| explictly cast the $files to array, so that it is array even when there is a single file only, see t=23251 (martin) | c# (martin) | ||
| Line 1: | Line 1: | ||
| ====== Automating download in parallel connections over SFTP/FTP protocol ====== | ====== Automating download in parallel connections over SFTP/FTP protocol ====== | ||
| - | The following example uses [[library|WinSCP .NET assembly]] from a [[library_powershell|PowerShell]] script. If you have another preferred language, you can easily translate it. | + | ===== C# ==== |
| - | The example opens by default three parallel connections and evenly splits the download of files from specified remote directory among these. | + | The example opens by default three parallel connections and uses them to download remote file tree to local folder in parallel. |
| + | |||
| + | <code csharp> | ||
| + | using System; | ||
| + | using System.Collections.Generic; | ||
| + | using System.IO; | ||
| + | using System.Threading.Tasks; | ||
| + | using WinSCP; | ||
| + | |||
| + | class Example | ||
| + | { | ||
| + | public static int Main() | ||
| + | { | ||
| + | try | ||
| + | { | ||
| + | // Setup session options | ||
| + | SessionOptions sessionOptions = new SessionOptions | ||
| + | { | ||
| + | Protocol = Protocol.Sftp, | ||
| + | HostName = "example.com", | ||
| + | UserName = "user", | ||
| + | Password = "mypassword", | ||
| + | SshHostKeyFingerprint = "ssh-rsa 2048 xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx" | ||
| + | }; | ||
| + | |||
| + | const string localPath = "C:\\local\\path"; | ||
| + | const string remotePath = "/remote/path"; | ||
| + | const int batches = 3; | ||
| + | |||
| + | DateTime started = DateTime.Now; | ||
| + | int count = 0; | ||
| + | Int64 bytes = 0; | ||
| + | |||
| + | using (Session session = new Session()) | ||
| + | { | ||
| + | Console.WriteLine("Connecting..."); | ||
| + | session.Open(sessionOptions); | ||
| + | |||
| + | Console.WriteLine("Starting files enumeration..."); | ||
| + | IEnumerable<RemoteFileInfo> files = session.EnumerateRemoteFiles(remotePath, null, EnumerationOptions.AllDirectories); | ||
| + | IEnumerator<RemoteFileInfo> filesEnumerator = files.GetEnumerator(); | ||
| + | |||
| + | List<Task> tasks = new List<Task>(); | ||
| + | |||
| + | for (int i = 1; i <= batches; i++) | ||
| + | { | ||
| + | int no = i; | ||
| + | |||
| + | Task task = new Task(() => | ||
| + | { | ||
| + | using (Session downloadSession = new Session()) | ||
| + | { | ||
| + | Console.WriteLine("Starting download {0}...", no); | ||
| + | downloadSession.Open(sessionOptions); | ||
| + | |||
| + | while (true) | ||
| + | { | ||
| + | string remoteFilePath; | ||
| + | lock (filesEnumerator) | ||
| + | { | ||
| + | if (!filesEnumerator.MoveNext()) | ||
| + | { | ||
| + | break; | ||
| + | } | ||
| + | |||
| + | RemoteFileInfo file = filesEnumerator.Current; | ||
| + | bytes += file.Length; | ||
| + | count++; | ||
| + | remoteFilePath = file.FullName; | ||
| + | } | ||
| + | |||
| + | string localFilePath = session.TranslateRemotePathToLocal(remoteFilePath, remotePath, localPath); | ||
| + | Console.WriteLine("Downloading {0} to {1} in {2}...", remoteFilePath, localFilePath, no); | ||
| + | Directory.CreateDirectory(Path.GetDirectoryName(localFilePath)); | ||
| + | downloadSession.GetFiles(session.EscapeFileMask(remoteFilePath), localFilePath).Check(); | ||
| + | } | ||
| + | |||
| + | Console.WriteLine("Download {0} done", no); | ||
| + | } | ||
| + | |||
| + | }); | ||
| + | |||
| + | tasks.Add(task); | ||
| + | task.Start(); | ||
| + | } | ||
| + | |||
| + | Console.WriteLine("Waiting for downloads to complete..."); | ||
| + | Task.WaitAll(tasks.ToArray()); | ||
| + | } | ||
| + | |||
| + | Console.WriteLine("Done"); | ||
| + | |||
| + | DateTime ended = DateTime.Now; | ||
| + | Console.WriteLine("Took {0}", (ended - started)); | ||
| + | Console.WriteLine("Downloaded {0} files, totaling {1:N0} bytes", count, bytes); | ||
| + | |||
| + | return 0; | ||
| + | } | ||
| + | catch (Exception e) | ||
| + | { | ||
| + | Console.WriteLine("Error: {0}", e); | ||
| + | return 1; | ||
| + | } | ||
| + | } | ||
| + | } | ||
| + | </code> | ||
| + | |||
| + | ===== PowerShell ==== | ||
| + | |||
| + | The code is not equivalent to the C# example above. The PowerShell code does not download subdirectories. It also split the files to batches by their count only, instead of using a queue like the C# code. | ||
| <code powershell> | <code powershell> | ||