Post a reply

Username

Subject

Message body

Font colour:

Font size:

[quote="CinnTech"]I currently have the below script which works great. The problem is I have to manually upload a 'completed.txt' file to mark the folder as completed.

I was wondering if there is a way to automate this once the current folder has been marked as synced? I'm aware that winscp only checks files and not directories so just checking if there is another solution.

[code]# This script will login to a SFTP and download/sync new files via WINSCP. 
# - Once a folder is synced it will upload a completed file so you can move the local folder.
# - If the remote folder is emptied it will remove the completed file and the folder.
# - This script is intended to be put in a scheduled task. 
# - v1.0

# SET THE VARIABLES
	$downloadPath = "s:\Downloads" 
	$remotePath = "/home/downloads"
	$drivePath = "s:" #this can be a network drive and should be the same as download path drive.
	$checkFile = "completed.txt" #change this to what file you want to tag the folder and skip it. 
	$logFile = "$drivePath\torSync.log" #where you want your log file saved.
	$hostName = "ftp.server.com"
	$userName = "ftpuser" #ftp
	$passWord = "ftppass" #ftp
	$mapUser = "uncuser" #username to map network drive
	$mapPass = "uncpass" #password to map network drive
	$mapPath = "\\networkpath"
	$sshKey = "ssh-rsa 2048 yoursshkey (it's like xx:xx:xx: etc)"
	$winscpPath = "C:\Program Files (x86)\WinSCP"
    $localIp = "0.0.0.0" #if you want to force a VPN put your public static IP here. If you don't have a Static IP change the script to -> if (!($myIp -eq $localIp)) <- and put your VPNs static IP here (otherwise put 0.0.0.0 or some bogus IP, and it will not check VPN status).
	$countFiles = 0 #Don't change this.
	$((Get-Date).DateTime) #this will show the date on screen.
	
	#This is to map network drive if not already mapped.
	if (!([System.IO.Directory]::Exists("$drivePath\")))
	{
		net use $drivePath $mapPath /user:$mapUser $mapPass /persistent:yes
	}
	
	#Disable these to disable WinSCP logging.
	# $WinSCPDebugLogFile = "s:\WinSCPDebugLogFile.txt"
	# $WinSCPSessionLogFile = "s:\WinSCPSessionLogFile.txt"
	# $WinSCPXMLLogFile = "s:\WinSCPXMLLogFile.xml"
	
	# START the Email Log File
	Write-Output "" >> $logFile
	Write-Output "Synchronization Started"  >> $logFile

#Let's verify we're on our VPN
 function Get-ExternalIP {
(Invoke-WebRequest ifconfig.me/ip).Content
}
 
$myIp1 = Get-ExternalIP
$myIp = $myIp1.replace("`n","")
 if ($myIp -eq $localIp)
	{
	Write-Output "--> Your not connected to the VPN $myIp <--" >> $logFile
	}
else
{

#WINSCP 
		try {
			[Reflection.Assembly]::LoadFile("$winscpPath\WinSCPnet.dll") | Out-Null
		} catch {
			Write-Error "$winscpPathwinSCPnet.dll failed to load, you must have WinSCP 5.20+ installed with the .NET assembly."
		}

# Session.FileTransferred event handler
		 
		function FileTransferred
		{
			Param($e)
		 
			if ($e.Error -eq $Null)
			{
				Write-Output ("$currentTime || Download of {0} succeeded " -f $e.FileName) >> $logFile
			}
			else
			{
				Write-Output ("$currentTime || Download of {0} failed: {1}" -f $e.FileName, $e.Error) >> $logFile
			}
		}
		
		function FileTransferProgress # This won't show in the window if Scheduled Task running without a logged in user.
		{
			Param($e)
			# New line for every new file
			if (($script:lastFileName -ne $Null) -and
				($script:lastFileName -ne $e.FileName))
			{
				 Write-Host  
			}
		 
			# Print transfer progress (shows in window only - not in log file :( )
			Write-Progress  ("`r{0} ({1:P0})" -f $e.FileName, $e.FileProgress)
			
			# Remember a name of the last file reported
			$script:lastFileName = $e.FileName
		}
	 
		 
		# Main script
		
		$script:lastFileName = $Null
		 
		try
		{
			$sessionOptions = New-Object WinSCP.SessionOptions
			$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
			$sessionOptions.HostName = $hostName
			$sessionOptions.UserName = $userName
			$sessionOptions.Password = $passWord
			$sessionOptions.SshHostKeyFingerprint = $sshKey
			
		 $session = New-Object WinSCP.Session

try
			{
				# Will continuously report progress of synchronization
				$session.add_FileTransferred( { FileTransferred($_) } )
				$session.add_FileTransferProgress( { FileTransferProgress($_) } )
				
				# Setup session log files locations 
				
				$session.DebugLogPath = $WinSCPDebugLogFile 
				$session.SessionLogPath = $WinSCPSessionLogFile 
				$session.XMLLogPath = $WinSCPXMLLogFile 
				
				# Connect
				$session.Open($sessionOptions)
				
				 #Get a list of remote folders & sort them.
				$directory = $session.ListDirectory("$remotePath")
		 
				foreach ($fileInfo in $directory.Files |  Sort-Object { [regex]::Replace($_.Name, '\d+', { $args[0].Value.PadLeft(20) }) })
				{
				$currentTime = Get-Date
					# Write-Host ("{0}" -f $fileInfo.Name)  >> $logFile
					
					$currentFolder = 	("{0}" -f $fileInfo.Name)
					
					# I don't want to sync . or .. as it will sync everything. Also, ignoring .htaccess file
					if (!($currentFolder -eq "." -or $currentFolder -eq ".." -or $currentFolder -eq ".htaccess"))
					{
						#check to see if current folder is completed first.
					
						$completedCheck = "$remotePath/$currentFolder/$checkFile"
								 
						if ($session.FileExists($completedCheck))
						{
							# Now let's see if it's the only file in the folder remove the file and the folder if it is.
							# Gel list of files in the directory
							$wildcard = "*.*" #we want to list all files
							$directory = $session.ListDirectory("$remotePath/$currentFolder")
							#let's count the files and omit [.] and [..] (there should be no .htaccess files) if there is then we have to add that to the script.
							foreach ($fileInfo in $directory.Files |
									Where-Object { $_.Name -Like $wildcard -AND $_.Name -NotLike "." -AND $_.Name -NotLike ".."})
								{
									#this is for testing.
									# Write-Host ("{0}" -f $fileInfo.Name)
									
									$countFiles++
										
								}
							if ($countFiles -gt 1)
							{
								 Write-Output ("$currentTime || SKIPPING - FOLDER NOT EMPTY || $currentFolder || File {0} exists " -f $completedCheck)  >> $logFile
							}
								elseif ($countFiles -eq 1)
								{
									Write-Output ("$currentTime || DELETING - FOLDER WAS EMPTY || $currentFolder || File {0} was the only thing to exist" -f $completedCheck)  >> $logFile
									$removalFileResult = $session.RemoveFiles("$completedCheck")
									$removalDirResult = $session.RemoveFiles("$remotePath/$currentFolder/")
 									if ($removalFileResult.IsSuccess)
									{
										Write-Output ("$currentTime || DELETE file completed.txt SUCCESS")  >> $logFile
									}
									else
									{
										Write-Output ("$currentTime || !! DELETE FAILED !!")  >> $logFile
									}
									if ($removalDirResult.IsSuccess)
									{
										Write-Output ("$currentTime || DELETE FOLDER SUCCESS")  >> $logFile
									}
									else
									{
										Write-Output ("$currentTime || !! DELETE FOLDER FAILED !!")  >> $logFile
									}
								}
							else 
							{
								Write-Output ("$currentTime || ERROR in count files")  >> $logFile
							}
						}
						
						# If we don't find the 'skip' then sync this folder 
						else 
						{	
							# Create folder if it doesn't exist
							if (!(Test-Path -Path "$downloadPath\$currentFolder" -ErrorAction SilentlyContinue )) 
							{ 
								New-Item "$downloadPath\$currentFolder" -Type Directory -ErrorAction SilentlyContinue 
								Write-Output "$currentTime || FOLDER CREATED || $downloadPath\$currentFolder" >> $logFile
							}
							# Now let's synchronize the files in the folder.
							Write-Output "$currentTime || Currently Syncing || $remotePath\$currentFolder to || $downloadPath\$currentFolder"  >> $logFile
							$synchronizationResult = $session.SynchronizeDirectories(
								[WinSCP.SynchronizationMode]::Local, "$downloadPath\$currentFolder", "$remotePath/$currentFolder", $False, [WinSCP.SynchronizationCriteria]::Either)
							# If the folder has finished synchronization then upload the 'skip' file so we can move the folder locally.
							if ($synchronizationResult)
							{ 
								$session.PutFiles("$drivePath\$checkFile", "$remotePath/$currentFolder/$checkFile",$False) 
								Write-Output "$currentTime || FOLDER COMPLETED and uploaded $checkFile to $remotePath/$currentFolder" >> $logFile
							}
							# Throw on any error
							 $synchronizationResult.Check()			
						} 
					}
				}

}
			finally
			{
				 # Terminate line after the last file (if any)
				if ($script:lastFileName -ne $Null)
				{
					 Write-Host  
				}
				# Disconnect, clean up
				$session.Dispose()
			}
			exit 0
		}

catch [Exception]
	{
	   Write-Output $_.Exception.Message >> $logFile
		exit 1
	}
}
[/code]

** I've updated the code as it is now working as intended - so putting here for other's.[/quote]

Options

Disable BBCode in this post

Notify me when a reply is posted (registered users only)

Add an Attachment

If you do not want to add an Attachment to your Post, please leave the Fields blank.

Filename

(maximum 10 MB; please compress large files; only common media, archive, text and programming file formats are allowed)

File Comment

Options

Private file (available to author and site moderators only)

Topic review

Guest

2014-11-01 02:47

That worked, thanks!

martin

2014-10-31

The synchronization is finished when Session.SynchronizeDirectories finishes. Just follow it by Session.PutFiles.

Guest

2014-10-29 15:13

I'm just wondering if there is a way to check if/when all contents of the folder are synced - if they are - then upload the file automatically. Currently I have to manually monitor (via log file or otherwise); when a folder is synced I manually upload the 'completed.txt' file. Then I can move the local copy and not have it sync again.

Does WINSCP give a true/false return if a file is synced? Maybe I could do a count files on a folder and a count for true and compare, if they equal upload a file?

martin

Re: Folder Sync - tagging completed folders (upload file)

2014-10-29

I'm not sure if I understand a problem. Are you asking how to create a file in remote directory (i.e. upload an empty file) "completed.txt" to the server?

CinnTech

Folder Sync - tagging completed folders (upload file)

2014-10-28 18:34

I currently have the below script which works great. The problem is I have to manually upload a 'completed.txt' file to mark the folder as completed.

I was wondering if there is a way to automate this once the current folder has been marked as synced? I'm aware that winscp only checks files and not directories so just checking if there is another solution.

# This script will login to a SFTP and download/sync new files via WINSCP. 

# - Once a folder is synced it will upload a completed file so you can move the local folder.

# - If the remote folder is emptied it will remove the completed file and the folder.

# - This script is intended to be put in a scheduled task. 

# - v1.0



   # SET THE VARIABLES

   $downloadPath = "s:\Downloads" 

   $remotePath = "/home/downloads"

   $drivePath = "s:" #this can be a network drive and should be the same as download path drive.

   $checkFile = "completed.txt" #change this to what file you want to tag the folder and skip it. 

   $logFile = "$drivePath\torSync.log" #where you want your log file saved.

   $hostName = "ftp.server.com"

   $userName = "ftpuser" #ftp

   $passWord = "ftppass" #ftp

   $mapUser = "uncuser" #username to map network drive

   $mapPass = "uncpass" #password to map network drive

   $mapPath = "\\networkpath"

   $sshKey = "ssh-rsa 2048 yoursshkey (it's like xx:xx:xx: etc)"

   $winscpPath = "C:\Program Files (x86)\WinSCP"

    $localIp = "0.0.0.0" #if you want to force a VPN put your public static IP here. If you don't have a Static IP change the script to -> if (!($myIp -eq $localIp)) <- and put your VPNs static IP here (otherwise put 0.0.0.0 or some bogus IP, and it will not check VPN status).

   $countFiles = 0 #Don't change this.

   $((Get-Date).DateTime) #this will show the date on screen.

   

   #This is to map network drive if not already mapped.

   if (!([System.IO.Directory]::Exists("$drivePath\")))

   {

      net use $drivePath $mapPath /user:$mapUser $mapPass /persistent:yes

   }

   

   #Disable these to disable WinSCP logging.

   # $WinSCPDebugLogFile = "s:\WinSCPDebugLogFile.txt"

   # $WinSCPSessionLogFile = "s:\WinSCPSessionLogFile.txt"

   # $WinSCPXMLLogFile = "s:\WinSCPXMLLogFile.xml"

   

   # START the Email Log File

   Write-Output "" >> $logFile

   Write-Output "Synchronization Started"  >> $logFile

   





#Let's verify we're on our VPN

 function Get-ExternalIP {

(Invoke-WebRequest ifconfig.me/ip).Content

}

 

$myIp1 = Get-ExternalIP

$myIp = $myIp1.replace("`n","")

 if ($myIp -eq $localIp)

   {

   Write-Output "--> Your not connected to the VPN $myIp <--" >> $logFile

   }

else

{ 



   #WINSCP 

      try {

         [Reflection.Assembly]::LoadFile("$winscpPath\WinSCPnet.dll") | Out-Null

      } catch {

         Write-Error "$winscpPathwinSCPnet.dll failed to load, you must have WinSCP 5.20+ installed with the .NET assembly."

      }



       # Session.FileTransferred event handler

       

      function FileTransferred

      {

         Param($e)

       

         if ($e.Error -eq $Null)

         {

            Write-Output ("$currentTime || Download of {0} succeeded " -f $e.FileName) >> $logFile

         }

         else

         {

            Write-Output ("$currentTime || Download of {0} failed: {1}" -f $e.FileName, $e.Error) >> $logFile

         }

      }

      

      function FileTransferProgress # This won't show in the window if Scheduled Task running without a logged in user.

      {

         Param($e)

         # New line for every new file

         if (($script:lastFileName -ne $Null) -and

            ($script:lastFileName -ne $e.FileName))

         {

             Write-Host  

         }

       

         # Print transfer progress (shows in window only - not in log file :( )

         Write-Progress  ("`r{0} ({1:P0})" -f $e.FileName, $e.FileProgress)

         

         # Remember a name of the last file reported

         $script:lastFileName = $e.FileName

      }

    

       

      # Main script

      

      $script:lastFileName = $Null

       

      try

      {

         $sessionOptions = New-Object WinSCP.SessionOptions

         $sessionOptions.Protocol = [WinSCP.Protocol]::Sftp

         $sessionOptions.HostName = $hostName

         $sessionOptions.UserName = $userName

         $sessionOptions.Password = $passWord

         $sessionOptions.SshHostKeyFingerprint = $sshKey

         

       $session = New-Object WinSCP.Session   



         try

         {

            # Will continuously report progress of synchronization

            $session.add_FileTransferred( { FileTransferred($_) } )

            $session.add_FileTransferProgress( { FileTransferProgress($_) } )

            

            # Setup session log files locations 

            

            $session.DebugLogPath = $WinSCPDebugLogFile 

            $session.SessionLogPath = $WinSCPSessionLogFile 

            $session.XMLLogPath = $WinSCPXMLLogFile 

            

            # Connect

            $session.Open($sessionOptions)

            

             #Get a list of remote folders & sort them.

            $directory = $session.ListDirectory("$remotePath")

       

            foreach ($fileInfo in $directory.Files |  Sort-Object { [regex]::Replace($_.Name, '\d+', { $args[0].Value.PadLeft(20) }) })

            {

            $currentTime = Get-Date

               # Write-Host ("{0}" -f $fileInfo.Name)  >> $logFile

               

               $currentFolder =    ("{0}" -f $fileInfo.Name)

               

               # I don't want to sync . or .. as it will sync everything. Also, ignoring .htaccess file

               if (!($currentFolder -eq "." -or $currentFolder -eq ".." -or $currentFolder -eq ".htaccess"))

               {

                  #check to see if current folder is completed first.

               

                  $completedCheck = "$remotePath/$currentFolder/$checkFile"

                         

                  if ($session.FileExists($completedCheck))

                  {

                     # Now let's see if it's the only file in the folder remove the file and the folder if it is.

                     # Gel list of files in the directory

                     $wildcard = "*.*" #we want to list all files

                     $directory = $session.ListDirectory("$remotePath/$currentFolder")

                     #let's count the files and omit [.] and [..] (there should be no .htaccess files) if there is then we have to add that to the script.

                     foreach ($fileInfo in $directory.Files |

                           Where-Object { $_.Name -Like $wildcard -AND $_.Name -NotLike "." -AND $_.Name -NotLike ".."})

                        {

                           #this is for testing.

                           # Write-Host ("{0}" -f $fileInfo.Name)

                           

                           $countFiles++

                              

                        }

                     if ($countFiles -gt 1)

                     {

                         Write-Output ("$currentTime || SKIPPING - FOLDER NOT EMPTY || $currentFolder || File {0} exists " -f $completedCheck)  >> $logFile

                     }

                        elseif ($countFiles -eq 1)

                        {

                           Write-Output ("$currentTime || DELETING - FOLDER WAS EMPTY || $currentFolder || File {0} was the only thing to exist" -f $completedCheck)  >> $logFile

                           $removalFileResult = $session.RemoveFiles("$completedCheck")

                           $removalDirResult = $session.RemoveFiles("$remotePath/$currentFolder/")

                            if ($removalFileResult.IsSuccess)

                           {

                              Write-Output ("$currentTime || DELETE file completed.txt SUCCESS")  >> $logFile

                           }

                           else

                           {

                              Write-Output ("$currentTime || !! DELETE FAILED !!")  >> $logFile

                           }

                           if ($removalDirResult.IsSuccess)

                           {

                              Write-Output ("$currentTime || DELETE FOLDER SUCCESS")  >> $logFile

                           }

                           else

                           {

                              Write-Output ("$currentTime || !! DELETE FOLDER FAILED !!")  >> $logFile

                           }

                        }

                     else 

                     {

                        Write-Output ("$currentTime || ERROR in count files")  >> $logFile

                     }

                  }

                  

                  # If we don't find the 'skip' then sync this folder 

                  else 

                  {   

                     # Create folder if it doesn't exist

                     if (!(Test-Path -Path "$downloadPath\$currentFolder" -ErrorAction SilentlyContinue )) 

                     { 

                        New-Item "$downloadPath\$currentFolder" -Type Directory -ErrorAction SilentlyContinue 

                        Write-Output "$currentTime || FOLDER CREATED || $downloadPath\$currentFolder" >> $logFile

                     }

                     # Now let's synchronize the files in the folder.

                     Write-Output "$currentTime || Currently Syncing || $remotePath\$currentFolder to || $downloadPath\$currentFolder"  >> $logFile

                     $synchronizationResult = $session.SynchronizeDirectories(

                        [WinSCP.SynchronizationMode]::Local, "$downloadPath\$currentFolder", "$remotePath/$currentFolder", $False, [WinSCP.SynchronizationCriteria]::Either)

                     # If the folder has finished synchronization then upload the 'skip' file so we can move the folder locally.

                     if ($synchronizationResult)

                     { 

                        $session.PutFiles("$drivePath\$checkFile", "$remotePath/$currentFolder/$checkFile",$False) 

                        Write-Output "$currentTime || FOLDER COMPLETED and uploaded $checkFile to $remotePath/$currentFolder" >> $logFile

                     }

                     # Throw on any error

                      $synchronizationResult.Check()         

                  } 

               }

            }



         }

         finally

         {

             # Terminate line after the last file (if any)

            if ($script:lastFileName -ne $Null)

            {

                Write-Host  

            }

            # Disconnect, clean up

            $session.Dispose()

         }

         exit 0

      }



   catch [Exception]

   {

      Write-Output $_.Exception.Message >> $logFile

      exit 1

   }

}

** I've updated the code as it is now working as intended - so putting here for other's.

Post a reply

Topic review

Re: Folder Sync - tagging completed folders (upload file)

Folder Sync - tagging completed folders (upload file)

Documentation

Support

Associations

Follow Us