library_example_parallel_transfers » Revisions »

Differences

This shows you the differences between the selected revisions of the page.

library_example_parallel_transfers 2017-10-03		library_example_parallel_transfers 2023-11-27 (current)
	Line 1:		Line 1:
-	====== Automating download in parallel connections over SFTP/FTP protocol ======	+	====== Automating transfers or synchronization in parallel connections over SFTP/FTP protocol ======

-	===== [[csharp]] C# ====	+	===== Download =====
		+
		+	==== [[download_csharp]] C# ====

	The example opens by default three parallel connections and uses them to download remote file tree to local folder in parallel.		The example opens by default three parallel connections and uses them to download remote file tree to local folder in parallel.
	Line 19:		Line 21:
	{		{
	// Setup session options		// Setup session options
-	SessionOptions sessionOptions = new SessionOptions	+	var sessionOptions = new SessionOptions
	{		{
	Protocol = Protocol.Sftp,		Protocol = Protocol.Sftp,
	Line 25:		Line 27:
	UserName = "user",		UserName = "user",
	Password = "mypassword",		Password = "mypassword",
-	SshHostKeyFingerprint = "ssh-rsa 2048 xx:xx:xx:xx:xx:xx:xx:xx..."	+	SshHostKeyFingerprint = "ssh-rsa 2048 xxxxxxxxxxx..."
	};		};

-	const string localPath = "C:\\local\\path";	+	const string localPath = @"C:\local\path";
	const string remotePath = "/remote/path";		const string remotePath = "/remote/path";
	const int batches = 3;		const int batches = 3;

-	DateTime started = DateTime.Now;	+	var started = DateTime.Now;
	int count = 0;		int count = 0;
-	Int64 bytes = 0;	+	long bytes = 0;

-	using (Session session = new Session())	+	using (var session = new Session())
	{		{
	Console.WriteLine("Connecting...");		Console.WriteLine("Connecting...");
	Line 42:		Line 44:

	Console.WriteLine("Starting files enumeration...");		Console.WriteLine("Starting files enumeration...");
		+	var opts = WinSCP.EnumerationOptions.AllDirectories;
	IEnumerable<RemoteFileInfo> files =		IEnumerable<RemoteFileInfo> files =
-	session.EnumerateRemoteFiles(	+	session.EnumerateRemoteFiles(remotePath, null, opts);
-	························remotePath, null, EnumerationOptions.AllDirectories);	+
	IEnumerator<RemoteFileInfo> filesEnumerator = files.GetEnumerator();		IEnumerator<RemoteFileInfo> filesEnumerator = files.GetEnumerator();

-	List<Task> tasks = new List<Task>();	+	var tasks = new List<Task>();

	for (int i = 1; i <= batches; i++)		for (int i = 1; i <= batches; i++)
	Line 53:		Line 55:
	int no = i;		int no = i;

-	Task task = new Task(() =>	+	var task = new Task(() =>
		+	{
		+	using (var downloadSession = new Session())
	{		{
-	using (Session downloadSession = new Session())	+	Console.WriteLine($"Starting download {no}...");
		+	downloadSession.Open(sessionOptions);
		+
		+	while (true)
	{		{
-	Console.WriteLine("Starting download {0}...";, no);	+	string remoteFilePath;
-	downloadSession.Open(sessionOptions);	+	lock (filesEnumerator)
-		+
-	while (true)	+
	{		{
-	string remoteFilePath;	+	if (!filesEnumerator.MoveNext())
-	····································lock (filesEnumerator)	+
	{		{
-	if (!filesEnumerator.MoveNext())	+	break;
-	{	+
-	············································break;	+
-	}	+
-		+
-	RemoteFileInfo file = filesEnumerator.Current;	+
-	bytes += file.Length;	+
-	count++;	+
-	remoteFilePath = file.FullName;	+
	}		}

-	string localFilePath =	+	RemoteFileInfo file = filesEnumerator.Current;
-	session.TranslateRemotePathToLocal(	+	bytes += file.Length;
-	remoteFilePath, remotePath, localPath);	+	count++;
-	Console.WriteLine(	+	remoteFilePath = file.FullName;
-	·······································"Downloading {0} to {1} in {2}...",	+
-	remoteFilePath, localFilePath, no);	+
-	Directory.CreateDirectory(	+
-	Path.GetDirectoryName(localFilePath));	+
-	downloadSession.GetFiles(	+
-	session.EscapeFileMask(remoteFilePath), localFilePath).	+
-	Check();	+
	}		}

-	Console.WriteLine("Download {0} done", no);	+	string localFilePath =
		+	RemotePath.TranslateRemotePathToLocal(
		+	remoteFilePath, remotePath, localPath);
		+	································Console.WriteLine(
		+	$"Downloading {remoteFilePath} to {localFilePath} in {no}...");
		+	string localFileDir = Path.GetDirectoryName(localFilePath);
		+	Directory.CreateDirectory(localFileDir);
		+	downloadSession.GetFileToDirectory(remoteFilePath, localFileDir);
	}		}

-	························});	+	Console.WriteLine($"Download {no} done");
		+	}
		+	····················});

	tasks.Add(task);		tasks.Add(task);
	Line 104:		Line 102:
	Console.WriteLine("Done");		Console.WriteLine("Done");

-	DateTime ended = DateTime.Now;	+	var ended = DateTime.Now;
-	Console.WriteLine("Took {0}", (ended - started));	+	Console.WriteLine($"Took {ended - started}");
-	Console.WriteLine("Downloaded {0} files, totaling {1:N0} bytes", count, bytes);	+	Console.WriteLine($"Downloaded {count} files, totaling {bytes:N0} bytes");

	return 0;		return 0;
	Line 112:		Line 110:
	catch (Exception e)		catch (Exception e)
	{		{
-	Console.WriteLine("Error: {0}", e);	+	Console.WriteLine($"Error: {e}");
	return 1;		return 1;
	}		}
	Line 119:		Line 117:
	</code>		</code>

-	===== PowerShell ====	+	==== [[powershell]] PowerShell ====

-	The code is not equivalent to the C# example above. The PowerShell code does not download subdirectories. It also split the files to batches by their count only, instead of using a queue like the C# code.	+	The following code uses [[ps>threadjob/start-threadjob\|''Start-ThreadJob'' cmdlet]] from ''ThreadJob'' module. It is a part of PowerShell 6 and newer. In PowerShell 5, it can be installed using ''Install-Module ThreadJob''.

	<code powershell>		<code powershell>
	param (		param (
-	$sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xx-xx-xx@example.com/",	+	$sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...@example.com/",
-	$remotePath = "/home/user/",	+	$remotePath = "/remote/path/",
-	$localPath = "c:\downloaded\",	+	$localPath = "c:\local\path\",
	$batches = 3		$batches = 3
	)		)
		+	·
	try		try
	{		{
-	$dllPath = (Join-Path $PSScriptRoot "WinSCPnet.dll")	+	$assemblyFilePath = "WinSCPnet.dll"
	# Load WinSCP .NET assembly		# Load WinSCP .NET assembly
-	Add-Type -Path $dllPath	+	Add-Type -Path $assemblyFilePath
		+	·
	# Setup session options		# Setup session options
	$sessionOptions = New-Object WinSCP.SessionOptions		$sessionOptions = New-Object WinSCP.SessionOptions
	$sessionOptions.ParseUrl($sessionUrl)		$sessionOptions.ParseUrl($sessionUrl)
		+	·
	$started = Get-Date		$started = Get-Date
		+	# Plain variables cannot be modified in job threads
		+	$stats = @{
		+	count = 0
		+	bytes = [long]0
		+	}
		+	·
	try		try
	{		{
	Line 150:		Line 153:
	$session.Open($sessionOptions)		$session.Open($sessionOptions)

-	# Retrieve list of files and sort them from larges to smallest	+	Write-Host "Starting files enumeration..."
-	········[array]$files =	+	········$files =
-	$session.ListDirectory($remotePath).Files \|	+	$session.EnumerateRemoteFiles(
-	···········Where-Object { -Not $_.IsDirectory } \|	+	················$remotePath, $Null, [WinSCP.EnumerationOptions]::AllDirectories)
-	Sort-Object Length -Descending	+	$filesEnumerator = $files.GetEnumerator()
		+
		+	········for ($i = 1; $i -le $batches; $i++)
		+	{
		+	Start-ThreadJob -Name "Batch $i" -ArgumentList $i {
		+	param ($no)

-	········# Calculate total size of all files	+	················try
-	·······$total = ($files \| Measure-Object -Property Length -Sum).Sum	+	···············{
-		+	···················Write-Host "Starting download $no..."
-	# And batch size	+
-	········$batch = [int]($total / $batches)	+

-	········Write-Host (	+	····················$downloadSession = New-Object WinSCP.Session
-	···········"Will download $($files.Count) files totaling $total bytes in " +	+	···················$downloadSession.Open($using:sessionOptions)
-	···········"$batches parallel batches, $batch bytes on average in each")	+
-		+
-	$start = 0	+
-	$sum = 0	+
-	$no = 0	+

-	········for ($i = 0; $i -lt $files.Count; $i++)	+	····················while ($True)
		+	{
		+	[System.Threading.Monitor]::Enter($using:filesEnumerator)
		+	try
		+	{
		+	if (!($using:filesEnumerator).MoveNext())
		+	{
		+	break
		+	}
		+
		+	$file = ($using:filesEnumerator).Current
		+	($using:stats).bytes += $file.Length
		+	($using:stats).count++
		+	$remoteFilePath = $file.FullName
		+	}
		+	finally
		+	{
		+	[System.Threading.Monitor]::<;nohilite>Exit</nohilite>($using:filesEnumerator)
		+	}
		+
		+	························$localFilePath =
		+	[WinSCP.RemotePath]::TranslateRemotePathToLocal(
		+	$remoteFilePath, $using:remotePath, $using:localPath)
		+	Write-Host "Downloading $remoteFilePath to $localFilePath in $no...";
		+	·······················$localFileDir = (Split-Path -Parent $localFilePath)
		+	New-Item -ItemType directory -Path $localFileDir -Force \| Out-Null
		+	$downloadSession.GetFileToDirectory($remoteFilePath, $localFileDir) \|
		+	Out-Null
		+	}
		+
		+	Write-Host "Download $no done"
		+	}
		+	finally
		+	{
		+	$downloadSession.Dispose()
		+	}
		+	} \| Out-Null
		+	}
		+
		+	Write-Host "Waiting for downloads to complete..."
		+	Get-Job \| Receive-Job -Wait -ErrorAction Stop
		+
		+	Write-Host "Done"
		+
		+	$ended = Get-Date
		+	Write-Host "Took $(New-TimeSpan -Start $started -End $ended)"
		+	Write-Host ("Downloaded $($stats.count) files, " +
		+	"totaling $($stats.bytes.ToString("N0")) bytes")
		+	}
		+	finally
		+	{
		+	# Disconnect, clean up
		+	$session.Dispose()
		+	}
		+
		+	exit 0
		+	}
		+	catch
		+	{
		+	Write-Host "Error: $($_.Exception.Message)"
		+	exit 1
		+	}
		+	</code>
		+
		+	===== Upload =====
		+
		+	==== [[upload_csharp]] C# ====
		+
		+	The example opens by default three parallel connections and uses them to upload locale file tree to remote folder in parallel.
		+
		+	<code csharp>
		+	using System;
		+	using System.Collections.Generic;
		+	using System.IO;
		+	using System.Threading.Tasks;
		+	using WinSCP;
		+
		+	class Example
		+	{
		+	static int Main()
		+	{
		+	try
	{		{
-	$sum += $files[$i].Length	+	// Setup session options
		+	SessionOptions sessionOptions = new SessionOptions
		+	{
		+	Protocol = Protocol.Sftp,
		+	HostName = "example.com",
		+	UserName = "user",
		+	Password = "password",
		+	SshHostKeyFingerprint = "ssh-rsa 2048 xxxxxxxxxxx..."
		+	};
		+
		+	const string localPath = @"C:\local\path";
		+	const string remotePath = "/remote/path";
		+	const int batches = 3;
		+
		+	DateTime started = DateTime.Now;
		+	int count = 0;
		+	Int64 bytes = 0;
		+
		+	Console.WriteLine("Starting files enumeration...");
		+	IEnumerable<string> files =
		+	Directory.EnumerateFiles(localPath, ".", SearchOption.AllDirectories);
		+	IEnumerator<string> filesEnumerator = files.GetEnumerator();
		+
		+	List<Task> tasks = new List<Task>();
		+
		+	HashSet<string> existingRemotePaths = new HashSet<string>();

-	# Found enough files for the next batch	+	for (int i = 1; i <= batches; i++)
-	···········if (($sum -ge $batch) -or ($i -eq $files.Count - 1))	+
	{		{
-	Write-Host "Starting batch $no to download $($i - $start + 1) files totaling $sum"	+	int no = i;
-		+
-	$fileList = $files[$start..$i] -join ";;"	+
-		+
-	# Start the background job for the batch	+
-	Start-Job -Name "Batch $no" `	+
-	-ArgumentList $dllPath, $sessionUrl, $remotePath, $localPath, $no, $fileList {	+
-	param (	+
-	[Parameter(Position = 0)]	+
-	$dllPath,	+
-	[Parameter(Position = 1)]	+
-	$sessionUrl,	+
-	[Parameter(Position = 2)]	+
-	$remotePath,	+
-	[Parameter(Position = 3)]	+
-	$localPath,	+
-	[Parameter(Position = 4)]	+
-	$no,	+
-	[Parameter(Position = 5)]	+
-	$fileList	+
-	)	+

-	····················try	+	················Task task = new Task(() =>
		+	{
		+	using (Session uploadSession = new Session())
	{		{
-	Write-Host "Starting batch $no"	+	while (true)
		+	·······················{
		+	string localFilePath;
		+	lock (filesEnumerator)
		+	{
		+	if (!filesEnumerator.MoveNext())
		+	{
		+	break;
		+	}

-	························# Load WinSCP .NET assembly.	+	································localFilePath = filesEnumerator.Current;
-	·······················# Need to use an absolute path as the Job is started	+	·······························bytes += new FileInfo(localFilePath).Length;
-	························# from user's documents folder.	+	·······························count++;
-	·······················Add-Type -Path $dllPath	+	···························}

-	························# Setup session options	+	····························if (!uploadSession.Opened)
-	························$sessionOptions = New-Object WinSCP.SessionOptions	+	···························{
-	·······················$sessionOptions.ParseUrl($sessionUrl)	+	·······························Console.WriteLine("Starting upload {0}...", no);
-		+	·······························uploadSession.Open(sessionOptions);
-	·······················try	+	}
-	························{	+
-	····························Write-Host "Connecting batch $no..."	+
-	$session = New-Object WinSCP.Session	+

-	$session.Open($sessionOptions)	+	string remoteFilePath =
-		+	································RemotePath.TranslateLocalPathToRemote(
-	···························$files = $fileList -split ";"	+	····································localFilePath, localPath, remotePath);
		+	Console.WriteLine(
		+	·······························"Uploading {0} to {1} in {2}...",
		+	································localFilePath, remoteFilePath, no);

-	# Download the files selected for this batch	+	string path =
-	foreach ($file in $files)	+	remoteFilePath.Substring(0, remoteFilePath.LastIndexOf('/'));
		+	string current = "";
		+
		+	if (path.Substring(0, 1) == "/";)
	{		{
-	$remoteFilePath = "$remotePath/$file";	+	path = path.Substring(1);
-	·······························$localFilePath = "$localPath\$file"	+	···························}
-	Write-Host "Downloading $remoteFilePath to $localFilePath in $no"	+

-	································$session.GetFiles(	+	····························while (!string.IsNullOrEmpty(path))
-	···································$session.EscapeFileMask($remoteFilePath), $localFilePath).	+	···························{
-	Check()	+	int p = path.IndexOf('/');
		+	current += '/';
		+	if (p >= 0)
		+	{
		+	current += path.Substring(0, p);
		+	path = path.Substring(p + 1);
		+	}
		+	else
		+	{
		+	current += path;
		+	path = "";
		+	}
		+
		+	lock (existingRemotePaths)
		+	{
		+	if (!existingRemotePaths.Contains(current)) // optimization
		+	{
		+	if (!uploadSession.FileExists(current))
		+	{
		+	Console.WriteLine("Creating {0}...", current);
		+	uploadSession.CreateDirectory(current);
		+	}
		+	existingRemotePaths.Add(current);
		+	}
		+	}
	}		}
		+
		+	uploadSession.PutFiles(
		+	localFilePath, RemotePath.EscapeFileMask(remoteFilePath)).
		+	Check();
	}		}
-	························finally	+
		+	························if (uploadSession.Opened)
	{		{
-	# Disconnect, clean up	+	Console.WriteLine("Upload {0} done", no);
-	···························$session.Dispose()	+	}
		+	·······················else
		+	{
		+	Console.WriteLine("Upload {0} had nothing to do", no);
	}		}
-
-	Write-Host "Batch $no done"
	}		}
-	····················catch [Exception]	+
		+	});
		+
		+	tasks.Add(task);
		+	task.Start();
		+	}
		+
		+	Console.WriteLine("Waiting for uploads to complete...");
		+	Task.WaitAll(tasks.ToArray());
		+
		+	Console.WriteLine("Done");
		+
		+	DateTime ended = DateTime.Now;
		+	Console.WriteLine("Took {0}", (ended - started));
		+	Console.WriteLine("Uploaded {0} files, totaling {1:N0} bytes", count, bytes);
		+
		+	return 0;
		+	}
		+	········catch (Exception e)
		+	{
		+	Console.WriteLine("Error: {0}", e);
		+	return 1;
		+	}
		+	}
		+	}
		+	</code>
		+
		+	===== [[synchronization]] Synchronization =====
		+
		+	==== [[synchronization_powershell]] PowerShell ====
		+
		+	//Regarding ''Start-ThreadJob'' cmdlet, see the comment in [[#powershell\|Download section]].//
		+
		+	<code powershell>
		+	param (
		+	$sessionUrl = "sftp://user:password;fingerprint=ssh-rsa-xxxxxxxxxxx...@example.com/",
		+	$remotePath = "/remote/path/",
		+	$localPath = "c:\local\path\",
		+	$removeFiles = $False,
		+	$connections = 3
		+	)
		+
		+	try
		+	{
		+	$assemblyFilePath = "WinSCPnet.dll"
		+	# Load WinSCP .NET assembly
		+	Add-Type -Path $assemblyFilePath
		+
		+	# Setup session options
		+	$sessionOptions = New-Object WinSCP.SessionOptions
		+	$sessionOptions.ParseUrl($sessionUrl)
		+
		+	$started = Get-Date
		+	# Plain variables cannot be modified in job threads
		+	$stats = @{
		+	count = 0
		+	}
		+
		+	try
		+	{
		+	# Connect
		+	Write-Host "Connecting..."
		+	$session = New-Object WinSCP.Session
		+	$session.Open($sessionOptions)
		+
		+	Write-Host "Comparing directories..."
		+	$differences =
		+	$session.CompareDirectories(
		+	[WinSCP.SynchronizationMode]::Both, $localPath, $remotePath, $removeFiles)
		+	if ($differences.Count -eq 0)
		+	{
		+	Write-Host "No changes found."
		+	}
		+	else
		+	{
		+	if ($differences.Count -lt $connections)
		+	{
		+	$connections = $differences.Count;
		+	}
		+	$differenceEnumerator = $differences.GetEnumerator()
		+
		+	for ($i = 1; $i -le $connections; $i++)
		+	{
		+	Start-ThreadJob -Name "Connection $i" -ArgumentList $i {
		+	param ($no)
		+
		+	try
		+	{
		+	Write-Host "Starting connection $no..."
		+
		+	$syncSession = New-Object WinSCP.Session
		+	$syncSession.Open($using:sessionOptions)
		+
		+	while ($True)
		+	{
		+	[System.Threading.Monitor]::Enter($using:differenceEnumerator)
		+	try
		+	{
		+	if (!($using:differenceEnumerator).MoveNext())
		+	{
		+	break
		+	}
		+
		+	$difference = ($using:differenceEnumerator).Current
		+	($using:stats).count++
		+	}
		+	finally
		+	{
		+	[System.Threading.Monitor]::Exit($using:differenceEnumerator)
		+	}
		+
		+	Write-Host "$difference in $no..."
		+	$difference.Resolve($syncSession) \| Out-Null
		+	}
		+
		+	Write-Host "Connection $no done"
		+	}
		+	finally
	{		{
-	Write-Host "Error: $($_.Exception.Message)"	+	$syncSession.Dispose()
-	exit 1	+
	}		}
	} \| Out-Null		} \| Out-Null
-
-	# Reset for the next batch
-	$no++
-	$sum = 0
-	$start = $i + 1
	}		}
		+
		+	Write-Host "Waiting for connections to complete..."
		+	Get-Job \| Receive-Job -Wait -ErrorAction Stop
		+
		+	Write-Host "Done"
	}		}
-
-	Write-Host "Waiting for batches to complete"
-	Get-Job \| Receive-Job -Wait
-
-	Write-Host "Done"

	$ended = Get-Date		$ended = Get-Date
	Write-Host "Took $(New-TimeSpan -Start $started -End $ended)"		Write-Host "Took $(New-TimeSpan -Start $started -End $ended)"
		+	Write-Host "Synchronized $($stats.count) differences"
	}		}
	finally		finally
	Line 268:		Line 508:
	$session.Dispose()		$session.Dispose()
	}		}
		+	·
	exit 0		exit 0
	}		}
-	catch [Exception]	+	catch
	{		{
	Write-Host "Error: $($_.Exception.Message)"		Write-Host "Error: $($_.Exception.Message)"
	Line 277:		Line 517:
	}		}
	</code>		</code>
-

Differences

Documentation

Support

Associations

Follow Us