Differences
This shows you the differences between the selected revisions of the page.
| library_example_parallel_transfers 2021-06-04 | library_example_parallel_transfers 2023-11-27 (current) | ||
| Line 1: | Line 1: | ||
| - | ====== Automating transfers in parallel connections over SFTP/FTP protocol ====== | + | ====== Automating transfers or synchronization in parallel connections over SFTP/FTP protocol ====== |
| ===== Download ===== | ===== Download ===== | ||
| Line 27: | Line 27: | ||
| UserName = "user", | UserName = "user", | ||
| Password = "mypassword", | Password = "mypassword", | ||
| - | SshHostKeyFingerprint = "ssh-rsa 2048 xxxxxxxxxxx...=" | + | SshHostKeyFingerprint = "ssh-rsa 2048 xxxxxxxxxxx..." |
| }; | }; | ||
| Line 119: | Line 119: | ||
| ==== [[powershell]] PowerShell ==== | ==== [[powershell]] PowerShell ==== | ||
| - | The following code uses [[ps>threadjob/start-threadjob|''Start-ThreadJob'' cmdlet]] from ''ThreadJob'' module. It is a part of PowerShell 6 and newer. In PowerShell 5, it can be installed using ''Install-Module ThreadJob''. (*Use $files.GetEnumerator()*) &beta | + | The following code uses [[ps>threadjob/start-threadjob|''Start-ThreadJob'' cmdlet]] from ''ThreadJob'' module. It is a part of PowerShell 6 and newer. In PowerShell 5, it can be installed using ''Install-Module ThreadJob''. |
| <code powershell> | <code powershell> | ||
| param ( | param ( | ||
| - | $sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...=@example.com/", | + | $sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...@example.com/", |
| $remotePath = "/remote/path/", | $remotePath = "/remote/path/", | ||
| $localPath = "c:\local\path\", | $localPath = "c:\local\path\", | ||
| Line 157: | Line 157: | ||
| $session.EnumerateRemoteFiles( | $session.EnumerateRemoteFiles( | ||
| $remotePath, $Null, [WinSCP.EnumerationOptions]::AllDirectories) | $remotePath, $Null, [WinSCP.EnumerationOptions]::AllDirectories) | ||
| - | # An explicit implementation of IEnumerable cannot be accessed directly in PowerShell. | + | $filesEnumerator = $files.GetEnumerator() |
| - | # WinSCP 5.18.4 has an implicit implementation, so direct $files.GetEnumerator() works. | + | |
| - | ········$filesEnumerator = | + | |
| - | ···········[System.Collections.IEnumerable].GetMethod("GetEnumerator").Invoke($files, $Null) | + | |
| for ($i = 1; $i -le $batches; $i++) | for ($i = 1; $i -le $batches; $i++) | ||
| Line 214: | Line 211: | ||
| Write-Host "Waiting for downloads to complete..." | Write-Host "Waiting for downloads to complete..." | ||
| - | Get-Job | Receive-Job -Wait | + | Get-Job | Receive-Job -Wait -ErrorAction Stop |
| Write-Host "Done" | Write-Host "Done" | ||
| Line 264: | Line 261: | ||
| UserName = "user", | UserName = "user", | ||
| Password = "password", | Password = "password", | ||
| - | SshHostKeyFingerprint = "ssh-rsa 2048 xxxxxxxxxxx...=" | + | SshHostKeyFingerprint = "ssh-rsa 2048 xxxxxxxxxxx..." |
| }; | }; | ||
| Line 398: | Line 395: | ||
| } | } | ||
| </code> | </code> | ||
| + | |||
| + | ===== [[synchronization]] Synchronization ===== | ||
| + | |||
| + | ==== [[synchronization_powershell]] PowerShell ==== | ||
| + | |||
| + | //Regarding ''Start-ThreadJob'' cmdlet, see the comment in [[#powershell|Download section]].// | ||
| + | |||
| + | <code powershell> | ||
| + | param ( | ||
| + | $sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...@example.com/", | ||
| + | $remotePath = "/remote/path/", | ||
| + | $localPath = "c:\local\path\", | ||
| + | $removeFiles = $False, | ||
| + | $connections = 3 | ||
| + | ) | ||
| + | |||
| + | try | ||
| + | { | ||
| + | $assemblyFilePath = "WinSCPnet.dll" | ||
| + | # Load WinSCP .NET assembly | ||
| + | Add-Type -Path $assemblyFilePath | ||
| + | |||
| + | # Setup session options | ||
| + | $sessionOptions = New-Object WinSCP.SessionOptions | ||
| + | $sessionOptions.ParseUrl($sessionUrl) | ||
| + | |||
| + | $started = Get-Date | ||
| + | # Plain variables cannot be modified in job threads | ||
| + | $stats = @{ | ||
| + | count = 0 | ||
| + | } | ||
| + | |||
| + | try | ||
| + | { | ||
| + | # Connect | ||
| + | Write-Host "Connecting..." | ||
| + | $session = New-Object WinSCP.Session | ||
| + | $session.Open($sessionOptions) | ||
| + | |||
| + | Write-Host "Comparing directories..." | ||
| + | $differences = | ||
| + | $session.CompareDirectories( | ||
| + | [WinSCP.SynchronizationMode]::Both, $localPath, $remotePath, $removeFiles) | ||
| + | if ($differences.Count -eq 0) | ||
| + | { | ||
| + | Write-Host "No changes found." | ||
| + | } | ||
| + | else | ||
| + | { | ||
| + | if ($differences.Count -lt $connections) | ||
| + | { | ||
| + | $connections = $differences.Count; | ||
| + | } | ||
| + | $differenceEnumerator = $differences.GetEnumerator() | ||
| + | |||
| + | for ($i = 1; $i -le $connections; $i++) | ||
| + | { | ||
| + | Start-ThreadJob -Name "Connection $i" -ArgumentList $i { | ||
| + | param ($no) | ||
| + | |||
| + | try | ||
| + | { | ||
| + | Write-Host "Starting connection $no..." | ||
| + | |||
| + | $syncSession = New-Object WinSCP.Session | ||
| + | $syncSession.Open($using:sessionOptions) | ||
| + | |||
| + | while ($True) | ||
| + | { | ||
| + | [System.Threading.Monitor]::Enter($using:differenceEnumerator) | ||
| + | try | ||
| + | { | ||
| + | if (!($using:differenceEnumerator).MoveNext()) | ||
| + | { | ||
| + | break | ||
| + | } | ||
| + | |||
| + | $difference = ($using:differenceEnumerator).Current | ||
| + | ($using:stats).count++ | ||
| + | } | ||
| + | finally | ||
| + | { | ||
| + | [System.Threading.Monitor]::Exit($using:differenceEnumerator) | ||
| + | } | ||
| + | |||
| + | Write-Host "$difference in $no..." | ||
| + | $difference.Resolve($syncSession) | Out-Null | ||
| + | } | ||
| + | |||
| + | Write-Host "Connection $no done" | ||
| + | } | ||
| + | finally | ||
| + | { | ||
| + | $syncSession.Dispose() | ||
| + | } | ||
| + | } | Out-Null | ||
| + | } | ||
| + | |||
| + | Write-Host "Waiting for connections to complete..." | ||
| + | Get-Job | Receive-Job -Wait -ErrorAction Stop | ||
| + | |||
| + | Write-Host "Done" | ||
| + | } | ||
| + | |||
| + | $ended = Get-Date | ||
| + | Write-Host "Took $(New-TimeSpan -Start $started -End $ended)" | ||
| + | Write-Host "Synchronized $($stats.count) differences" | ||
| + | } | ||
| + | finally | ||
| + | { | ||
| + | # Disconnect, clean up | ||
| + | $session.Dispose() | ||
| + | } | ||
| + | |||
| + | exit 0 | ||
| + | } | ||
| + | catch | ||
| + | { | ||
| + | Write-Host "Error: $($_.Exception.Message)" | ||
| + | exit 1 | ||
| + | } | ||
| + | </code> | ||
| + | |||