Differences
This shows you the differences between the selected revisions of the page.
| 2021-03-25 | 2021-03-25 | ||
| modernizing c# download code (martin) | full featured powershell version of the download code (martin) | ||
| Line 119: | Line 119: | ||
| ==== [[powershell]] PowerShell ==== | ==== [[powershell]] PowerShell ==== | ||
| - | The code is not equivalent to the C# example above. The PowerShell code does not download subdirectories. It also split the files to batches by their count only, instead of using a queue like the C# code. | + | The following code uses [[ps>threadjob/start-threadjob|''Start-ThreadJob'' cmdlet]] from ''ThreadJob'' module. It is a part of PowerShell 6 and newer. In PowerShell 5, it can be installed using ''Install-Module ThreadJob''. |
| <code powershell> | <code powershell> | ||
| param ( | param ( | ||
| $sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...=@example.com/", | $sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...=@example.com/", | ||
| - | $remotePath = "/home/user/", | + | $remotePath = "/remote/path/", |
| - | $localPath = "c:\downloaded\", | + | $localPath = "c:\local\path\", |
| $batches = 3 | $batches = 3 | ||
| ) | ) | ||
| + | · | ||
| try | try | ||
| { | { | ||
| - | $dllPath = (Join-Path $PSScriptRoot "WinSCPnet.dll") | + | $assemblyFilePath = "WinSCPnet.dll" |
| # Load WinSCP .NET assembly | # Load WinSCP .NET assembly | ||
| - | Add-Type -Path $dllPath | + | Add-Type -Path $assemblyFilePath |
| + | · | ||
| # Setup session options | # Setup session options | ||
| $sessionOptions = New-Object WinSCP.SessionOptions | $sessionOptions = New-Object WinSCP.SessionOptions | ||
| $sessionOptions.ParseUrl($sessionUrl) | $sessionOptions.ParseUrl($sessionUrl) | ||
| + | · | ||
| $started = Get-Date | $started = Get-Date | ||
| + | # Plain variables cannot be modified in job threads | ||
| + | $stats = @{ | ||
| + | count = 0 | ||
| + | bytes = [long]0 | ||
| + | } | ||
| + | · | ||
| try | try | ||
| { | { | ||
| Line 148: | Line 153: | ||
| $session.Open($sessionOptions) | $session.Open($sessionOptions) | ||
| - | # Retrieve list of files and sort them from larges to smallest | + | Write-Host "Starting files enumeration..." |
| - | [array]$files = | + | $files = |
| - | $session.ListDirectory($remotePath).Files | | + | $session.EnumerateRemoteFiles( |
| - | ···········Where-Object { -Not $_.IsDirectory } | | + | $remotePath, $Null, [WinSCP.EnumerationOptions]::AllDirectories) |
| - | Sort-Object Length -Descending | + | # An explicit implementation of IEnumerable cannot be accessed directly in PowerShell |
| + | ········$filesEnumerator = | ||
| + | [System.Collections.IEnumerable].GetMethod("GetEnumerator").Invoke($files, $Null) | ||
| + | |||
| + | ········for ($i = 1; $i -le $batches; $i++) | ||
| + | { | ||
| + | Start-ThreadJob -Name "Batch $i" -ArgumentList $i { | ||
| + | param ($no) | ||
| - | ········# Calculate total size of all files | + | ················try |
| - | ·······$total = ($files | Measure-Object -Property Length -Sum).Sum | + | ···············{ |
| - | + | ···················Write-Host "Starting download $no..." | |
| - | # And batch size | + | |
| - | ········$batch = [int]($total / $batches) | + | |
| - | ········Write-Host ( | + | ····················$downloadSession = New-Object WinSCP.Session |
| - | ···········"Will download $($files.Count) files totaling $total bytes in " + | + | ···················$downloadSession.Open($using:sessionOptions) |
| - | ···········"$batches parallel batches, $batch bytes on average in each") | + | |
| - | + | ||
| - | $start = 0 | + | |
| - | $sum = 0 | + | |
| - | $no = 0 | + | |
| - | ········for ($i = 0; $i -lt $files.Count; $i++) | + | ····················while ($True) |
| - | { | + | |
| - | $sum += $files[$i].Length | + | |
| - | + | ||
| - | # Found enough files for the next batch | + | |
| - | if (($sum -ge $batch) -or ($i -eq $files.Count - 1)) | + | |
| - | { | + | |
| - | Write-Host "Starting batch $no to download $($i - $start + 1) files totaling $sum" | + | |
| - | + | ||
| - | $fileList = $files[$start..$i] -join ";" | + | |
| - | + | ||
| - | # Start the background job for the batch | + | |
| - | Start-Job -Name "Batch $no" ` | + | |
| - | -ArgumentList $dllPath, $sessionUrl, $remotePath, $localPath, $no, $fileList { | + | |
| - | param ( | + | |
| - | [Parameter(Position = 0)] | + | |
| - | $dllPath, | + | |
| - | [Parameter(Position = 1)] | + | |
| - | $sessionUrl, | + | |
| - | [Parameter(Position = 2)] | + | |
| - | $remotePath, | + | |
| - | [Parameter(Position = 3)] | + | |
| - | $localPath, | + | |
| - | [Parameter(Position = 4)] | + | |
| - | $no, | + | |
| - | [Parameter(Position = 5)] | + | |
| - | $fileList | + | |
| - | ) | + | |
| - | + | ||
| - | try | + | |
| { | { | ||
| - | Write-Host "Starting batch $no" | + | [System.Threading.Monitor]::Enter($using:filesEnumerator) |
| - | + | ||
| - | # Load WinSCP .NET assembly. | + | |
| - | ························# Need to use an absolute path as the Job is started | + | |
| - | ························# from user's documents folder. | + | |
| - | Add-Type -Path $dllPath | + | |
| - | + | ||
| - | # Setup session options | + | |
| - | $sessionOptions = New-Object WinSCP.SessionOptions | + | |
| - | $sessionOptions.ParseUrl($sessionUrl) | + | |
| - | ························ | + | |
| try | try | ||
| { | { | ||
| - | Write-Host "Connecting batch $no..." | + | if (!($using:filesEnumerator).MoveNext()) |
| - | $session = New-Object WinSCP.Session | + | |
| - | + | ||
| - | $session.Open($sessionOptions) | + | |
| - | + | ||
| - | $files = $fileList -split ";" | + | |
| - | + | ||
| - | # Download the files selected for this batch | + | |
| - | foreach ($file in $files) | + | |
| { | { | ||
| - | $remoteFilePath = "$remotePath/$file" | + | break |
| - | $localFilePath = "$localPath\$file" | + | ···························} |
| - | Write-Host "Downloading $remoteFilePath to $localFilePath in $no"; | + | |
| - | ································$session.GetFiles( | + | ····························$file = ($using:filesEnumerator).Current |
| - | ····································[WinSCP.RemotePath]::EscapeFileMask($remoteFilePath), | + | ····························($using:stats).bytes += $file.Length |
| - | ···································$localFilePath).Check() | + | ($using:stats).count++ |
| - | } | + | $remoteFilePath = $file.FullName |
| } | } | ||
| finally | finally | ||
| { | { | ||
| - | # Disconnect, clean up | + | [System.Threading.Monitor]::Exit($using:filesEnumerator) |
| - | ····························$session.Dispose() | + | |
| } | } | ||
| - | ························ | + | |
| - | Write-Host "Batch $no done" | + | $localFilePath = |
| + | [WinSCP.RemotePath]::TranslateRemotePathToLocal( | ||
| + | $remoteFilePath, $using:remotePath, $using:localPath) | ||
| + | Write-Host "Downloading $remoteFilePath to $localFilePath in $no..." | ||
| + | $localFileDir = (Split-Path -Parent $localFilePath) | ||
| + | New-Item -ItemType directory -Path $localFileDir -Force | Out-Null | ||
| + | $downloadSession.GetFileToDirectory($remoteFilePath, $localFileDir) | | ||
| + | Out-Null | ||
| } | } | ||
| - | catch | + | |
| - | { | + | Write-Host "Download $no done" |
| - | ························Write-Host "Error: $($_.Exception.Message)" | + | ···············} |
| - | exit 1 | + | finally |
| - | ····················} | + | { |
| - | } | Out-Null | + | ···················$downloadSession.Dispose() |
| - | · | + | } |
| - | # Reset for the next batch | + | ···········} | Out-Null |
| - | ···············$no++ | + | |
| - | ················$sum = 0 | + | |
| - | $start = $i + 1 | + | |
| - | ············} | + | |
| } | } | ||
| - | Write-Host "Waiting for batches to complete" | + | Write-Host "Waiting for downloads to complete..." |
| Get-Job | Receive-Job -Wait | Get-Job | Receive-Job -Wait | ||
| + | · | ||
| Write-Host "Done" | Write-Host "Done" | ||
| + | · | ||
| $ended = Get-Date | $ended = Get-Date | ||
| Write-Host "Took $(New-TimeSpan -Start $started -End $ended)" | Write-Host "Took $(New-TimeSpan -Start $started -End $ended)" | ||
| + | Write-Host ("Downloaded $($stats.count) files, " + | ||
| + | "totaling $($stats.bytes.ToString("N0")) bytes") | ||
| } | } | ||
| finally | finally | ||
| Line 266: | Line 227: | ||
| $session.Dispose() | $session.Dispose() | ||
| } | } | ||
| + | · | ||
| exit 0 | exit 0 | ||
| } | } | ||