Retrieve Your Data

<#
.SYNOPSIS
    Downloads SysTrack Data Egress export files from Azure Blob Storage.

.DESCRIPTION
    This script downloads exported SysTrack data files for a specific export invocation.
    It retrieves the file list and access credentials from the SysTrack API, then downloads
    all files while organizing them into subdirectories based on the table name.
    
    The script includes robust error handling with automatic retries and exponential backoff
    for transient network issues. It will continue downloading remaining files even if 
    individual files fail after all retry attempts.

.PARAMETER SysTrackUrl
    The base URL of your SysTrack server. Do not include a trailing slash.

.PARAMETER ApiKey
    Your SysTrack API key for Data Egress. This key is used to authenticate with the 
    SysTrack API to retrieve download URLs and file lists.

.PARAMETER InvocationId
    The GUID identifying the specific export invocation you want to download.
    This is obtained via the SysTrack Data Egress UI, or your configured Webhook invocation.

.PARAMETER OutputFolder
    Optional. The base directory where files will be downloaded. If not specified,
    you will be prompted to enter one. If left blank at the prompt, the current
    working directory will be used (with a warning).

.PARAMETER NoSubdirectory
    Optional. If specified, files will be downloaded directly into the OutputFolder.
    By default, a subdirectory named after the InvocationId will be created to
    contain the downloaded files.

.EXAMPLE
    .\Download-SysTrackExport.ps1 -SysTrackUrl "https://myserver.com" -ApiKey "your-data-egress-api-key" -InvocationId "12345678-1234-1234-1234-123456789abc" -OutputFolder "C:\SysTrack Exports"
    
    Downloads all files for the specified invocation to "C:\SysTrack Exports\12345678-1234-1234-1234-123456789abc\"

.EXAMPLE
    .\Download-SysTrackExport.ps1 -SysTrackUrl "https://myserver.com" -ApiKey "your-data-egress-api-key" -InvocationId "12345678-1234-1234-1234-123456789abc" -OutputFolder "C:\Downloads" -NoSubdirectory
    
    Downloads all files directly to "C:\Downloads\" without creating an invocation subdirectory.

.NOTES
    Author: Lakeside Software
    Version: 1.0
    
    This script is intended as a reference implementation. You may customize it
    to fit your specific requirements and infrastructure.

    This script was generated with AI assistance (Github Copilot, Claude Sonnet 4.5).
    It was reviewed and tested by a human author.
#>

[CmdletBinding()]
param(
    [Parameter(Mandatory=$true, HelpMessage="Base URL of your SysTrack server (e.g., https://myserver.com)")]
    [string]$SysTrackUrl,
    
    [Parameter(Mandatory=$true, HelpMessage="Your SysTrack API key for Data Egress")]
    [string]$ApiKey,
    
    [Parameter(Mandatory=$true, HelpMessage="The GUID of the export invocation")]
    [string]$InvocationId,
    
    [Parameter(Mandatory=$false, HelpMessage="Base directory for downloaded files")]
    [string]$OutputFolder,
    
    [Parameter(Mandatory=$false, HelpMessage="Skip creating invocation subdirectory")]
    [switch]$NoSubdirectory
)

# ==============================================================================
# CONFIGURATION SECTION
# You can customize these values to match your requirements
# ==============================================================================

# Retry configuration for failed downloads
$MaxRetries = 3
$InitialRetryDelaySeconds = 2
$RetryBackoffMultiplier = 2

# Timeout for large file downloads (1 hour = 3600 seconds)
# Adjust this based on your expected file sizes and network speed
$DownloadTimeoutSeconds = 3600

# ==============================================================================
# INITIALIZATION
# ==============================================================================

# Track start time for performance reporting
$ScriptStartTime = Get-Date

# Initialize counters for tracking success/failure
$TotalFilesProcessed = 0
$SuccessfulDownloads = 0
$FailedDownloads = 0
$FailedFilesList = @()

# ==============================================================================
# HELPER FUNCTIONS
# ==============================================================================

<#
.SYNOPSIS
    Invokes a web request with retry logic and exponential backoff.
#>
function Invoke-WebRequestWithRetry {
    param(
        [string]$Uri,
        [hashtable]$Headers,
        [string]$Method = "GET",
        [string]$OutFile = $null,
        [int]$TimeoutSec = 300
    )
    
    for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) {
        try {
            $params = @{
                Uri = $Uri
                Method = $Method
                Headers = $Headers
                TimeoutSec = $TimeoutSec
                UseBasicParsing = $true
            }
            
            if ($OutFile) {
                $params['OutFile'] = $OutFile
                Invoke-WebRequest @params
            } else {
                return Invoke-RestMethod @params
            }
            
            # If we get here, the request succeeded
            return $true
            
        } catch {
            $errorMessage = $_.Exception.Message
            
            if ($attempt -lt $MaxRetries) {
                # Calculate exponential backoff delay: 2s, 4s, 8s
                $delaySeconds = $InitialRetryDelaySeconds * [Math]::Pow($RetryBackoffMultiplier, $attempt - 1)
                Write-Host "  ⚠️ Attempt $attempt failed: $errorMessage" -ForegroundColor Yellow
                Write-Host "  ⏳ Retrying in $delaySeconds seconds..." -ForegroundColor Cyan
                Start-Sleep -Seconds $delaySeconds
            } else {
                # All retries exhausted
                Write-Host "  ❌ All $MaxRetries attempts failed: $errorMessage" -ForegroundColor Red
                throw $_
            }
        }
    }
}

# ==============================================================================
# OUTPUT FOLDER SETUP
# ==============================================================================

Write-Host "`n═══════════════════════════════════════════════════════════" -ForegroundColor Cyan
Write-Host "  SysTrack Data Egress Download Script" -ForegroundColor Cyan
Write-Host "═══════════════════════════════════════════════════════════`n" -ForegroundColor Cyan

# Handle output folder parameter
if ([string]::IsNullOrWhiteSpace($OutputFolder)) {
    $OutputFolder = Read-Host "Enter output folder path (leave blank to use current directory)"
    
    if ([string]::IsNullOrWhiteSpace($OutputFolder)) {
        $OutputFolder = $PWD.Path
        Write-Warning "No output folder specified. Using current directory: $OutputFolder"
    }
}

# Ensure output folder exists
if (-not (Test-Path -Path $OutputFolder)) {
    Write-Host "📁 Creating output folder: $OutputFolder" -ForegroundColor Cyan
    try {
        New-Item -ItemType Directory -Path $OutputFolder -Force | Out-Null
    } catch {
        Write-Host "❌ ERROR: Unable to create output folder: $_" -ForegroundColor Red
        exit 1
    }
}

# Determine final target folder (with or without invocation subdirectory)
if ($NoSubdirectory) {
    $TargetFolder = $OutputFolder
    Write-Host "📂 Output location: $TargetFolder" -ForegroundColor Green
} else {
    $TargetFolder = Join-Path -Path $OutputFolder -ChildPath $InvocationId
    Write-Host "📂 Output location: $TargetFolder" -ForegroundColor Green
    
    if (-not (Test-Path -Path $TargetFolder)) {
        try {
            New-Item -ItemType Directory -Path $TargetFolder -Force | Out-Null
        } catch {
            Write-Host "❌ ERROR: Unable to create invocation subdirectory: $_" -ForegroundColor Red
            exit 1
        }
    }
}

# ==============================================================================
# API CALL TO RETRIEVE FILE LIST
# ==============================================================================

Write-Host "`n🔗 Connecting to SysTrack API..." -ForegroundColor Cyan

# Construct API endpoint URL
# DO NOT MODIFY THIS ENDPOINT FORMAT - it must match the SysTrack API specification
$ApiUrl = "$SysTrackUrl/api/dataegress/geturl/$InvocationId"

# Create headers with API key
# DO NOT MODIFY THE HEADER NAME - it must match what the SysTrack API expects
$Headers = @{
    "systrack-api-key" = $ApiKey
}

# Call API with retry logic
try {
    Write-Host "📡 Retrieving file list for invocation: $InvocationId" -ForegroundColor Cyan
    $ApiResponse = Invoke-WebRequestWithRetry -Uri $ApiUrl -Headers $Headers -Method "GET" -TimeoutSec 60
    
    # Validate response structure
    if (-not $ApiResponse.baseUrl -or -not $ApiResponse.sasToken -or -not $ApiResponse.files) {
        Write-Host "❌ ERROR: API response is missing required fields (baseUrl, sasToken, or files)" -ForegroundColor Red
        exit 1
    }
    
    $BaseUrl = $ApiResponse.baseUrl
    $SasToken = $ApiResponse.sasToken
    $FilesToDownload = $ApiResponse.files
    
    Write-Host "✅ Retrieved file list successfully" -ForegroundColor Green
    Write-Host "📊 Total files to download: $($FilesToDownload.Count)" -ForegroundColor Green
    
} catch {
    $statusCode = $_.Exception.Response.StatusCode.value__
    
    if ($statusCode -eq 401) {
        Write-Host "❌ ERROR: Invalid API key. Please check your credentials." -ForegroundColor Red
    } elseif ($statusCode -eq 404) {
        Write-Host "❌ ERROR: Invocation ID not found. Please verify the invocation ID." -ForegroundColor Red
    } else {
        Write-Host "❌ ERROR: Failed to retrieve file list from API: $_" -ForegroundColor Red
    }
    
    exit 1
}

# ==============================================================================
# FILE DOWNLOAD LOOP
# ==============================================================================

Write-Host "`n📥 Starting file downloads...`n" -ForegroundColor Cyan

# Iterate through each file in the list
for ($i = 0; $i -lt $FilesToDownload.Count; $i++) {
    $filePathInBlob = $FilesToDownload[$i]
    $TotalFilesProcessed++
    
    # Calculate progress percentage
    $percentComplete = [int](($i / $FilesToDownload.Count) * 100)
    
    # Parse table name and filename
    # Table name is everything before the LAST slash
    # Filename is everything after the LAST slash
    # DO NOT MODIFY THIS PARSING LOGIC - it matches the file structure in Azure Blob Storage
    $lastSlashIndex = $filePathInBlob.LastIndexOf('/')
    
    if ($lastSlashIndex -eq -1) {
        Write-Host "⚠️ WARNING: File path '$filePathInBlob' does not contain a slash. Skipping." -ForegroundColor Yellow
        $FailedDownloads++
        $FailedFilesList += $filePathInBlob
        continue
    }
    
    $tableName = $filePathInBlob.Substring(0, $lastSlashIndex)
    $fileName = $filePathInBlob.Substring($lastSlashIndex + 1)
    
    # Create table subdirectory if it doesn't exist
    $tableDirectory = Join-Path -Path $TargetFolder -ChildPath $tableName
    if (-not (Test-Path -Path $tableDirectory)) {
        try {
            New-Item -ItemType Directory -Path $tableDirectory -Force | Out-Null
        } catch {
            Write-Host "❌ ERROR: Unable to create table directory '$tableDirectory': $_" -ForegroundColor Red
            $FailedDownloads++
            $FailedFilesList += $filePathInBlob
            continue
        }
    }
    
    # Construct download URL
    # DO NOT MODIFY THIS URL CONSTRUCTION - the SAS token must be appended as-is
    $downloadUrl = "$BaseUrl/$filePathInBlob$SasToken"
    
    # Construct local file path
    $localFilePath = Join-Path -Path $tableDirectory -ChildPath $fileName
    
    # Display progress
    Write-Progress -Activity "Downloading SysTrack Export Files" `
                   -Status "File $($i + 1) of $($FilesToDownload.Count): $tableName\$fileName" `
                   -PercentComplete $percentComplete
    
    Write-Host "[$($i + 1)/$($FilesToDownload.Count)] Downloading: $tableName\$fileName" -ForegroundColor Cyan
    
    # Attempt download with retry logic
    try {
        # Note: We don't pass headers here because the SAS token in the URL provides authentication
        Invoke-WebRequestWithRetry -Uri $downloadUrl `
                                    -Headers @{} `
                                    -Method "GET" `
                                    -OutFile $localFilePath `
                                    -TimeoutSec $DownloadTimeoutSeconds | Out-Null
        
        Write-Host "  ✅ Downloaded successfully" -ForegroundColor Green
        $SuccessfulDownloads++
        
    } catch {
        Write-Host "  ❌ Failed to download after $MaxRetries attempts" -ForegroundColor Red
        $FailedDownloads++
        $FailedFilesList += $filePathInBlob
        
        # Continue to next file even after failure
        continue
    }
}

# Clear the progress bar
Write-Progress -Activity "Downloading SysTrack Export Files" -Completed

# ==============================================================================
# SUMMARY REPORT
# ==============================================================================

$ScriptEndTime = Get-Date
$TotalDuration = $ScriptEndTime - $ScriptStartTime

Write-Host "`n═══════════════════════════════════════════════════════════" -ForegroundColor Cyan
Write-Host "  Download Summary" -ForegroundColor Cyan
Write-Host "═══════════════════════════════════════════════════════════`n" -ForegroundColor Cyan

Write-Host "📊 Total files processed:    $TotalFilesProcessed" -ForegroundColor White
Write-Host "✅ Successful downloads:     $SuccessfulDownloads" -ForegroundColor Green
Write-Host "❌ Failed downloads:         $FailedDownloads" -ForegroundColor $(if ($FailedDownloads -eq 0) { "Green" } else { "Red" })
Write-Host "⏱️ Total time elapsed:       $($TotalDuration.ToString('hh\:mm\:ss'))" -ForegroundColor White
Write-Host "📂 Output location:          $TargetFolder" -ForegroundColor White

if ($FailedDownloads -gt 0) {
    Write-Host "`n⚠️ The following files failed to download:" -ForegroundColor Yellow
    foreach ($failedFile in $FailedFilesList) {
        Write-Host "   • $failedFile" -ForegroundColor Yellow
    }
    Write-Host "`nYou may re-run this script to retry failed downloads." -ForegroundColor Yellow
}

Write-Host "`n═══════════════════════════════════════════════════════════`n" -ForegroundColor Cyan

if ($FailedDownloads -eq 0) {
    Write-Host "🎉 All files downloaded successfully!" -ForegroundColor Green
    exit 0
} else {
    Write-Host "⚠️ Script completed with some failures. Please review the failed files list above." -ForegroundColor Yellow
    exit 1
}

"""
Simple script to download SysTrack export files.

Usage:
    python Download-SysTrackExport.py <SysTrackUrl> <ApiKey> <InvocationId> <OutputFolder>

Example:
    python Download-SysTrackExport.py "https://myserver.com" "your-api-key" "12345678-1234-1234-1234-123456789abc" "C:\\Downloads"

Author:
    Lakeside Software

Version:
    1.0

Notes:
    This script is intended as a reference implementation. You may customize it
    to fit your specific requirements and infrastructure.

    This script was generated with AI assistance (Github Copilot, Claude Sonnet 4.5).
    It was reviewed and tested by a human author.
"""

import sys
import os
import requests

# Check for required parameters
if len(sys.argv) != 5:
    print("ERROR: Missing required parameters")
    print(f"Usage: python {sys.argv[0]} <SysTrackUrl> <ApiKey> <InvocationId> <OutputFolder>")
    sys.exit(1)

systrack_url = sys.argv[1]
api_key = sys.argv[2]
invocation_id = sys.argv[3]
output_folder = sys.argv[4]

print(f"Starting download for invocation: {invocation_id}")

# Step 1: Get the file list from SysTrack API
api_url = f"{systrack_url}/api/dataegress/geturl/{invocation_id}"
headers = {"systrack-api-key": api_key}

print("Retrieving file list from API...")
response = requests.get(api_url, headers=headers)
response.raise_for_status()

# Step 2: Extract the download information
data = response.json()
base_url = data["baseUrl"]
sas_token = data["sasToken"]
files = data["files"]

print(f"Found {len(files)} files to download\n")

# Step 3: Download each file
success_count = 0
for file_path in files:
    # Parse the table name and filename
    last_slash = file_path.rfind('/')
    table_name = file_path[:last_slash]
    file_name = file_path[last_slash + 1:]
    
    # Create the table directory if it doesn't exist
    table_dir = os.path.join(output_folder, table_name)
    os.makedirs(table_dir, exist_ok=True)
    
    # Download the file
    download_url = f"{base_url}/{file_path}{sas_token}"
    local_path = os.path.join(table_dir, file_name)
    
    print(f"Downloading: {table_name}\\{file_name}")
    file_response = requests.get(download_url)
    file_response.raise_for_status()
    
    with open(local_path, 'wb') as f:
        f.write(file_response.content)
    
    success_count += 1

print(f"\nDownload complete! Downloaded {success_count} of {len(files)} files to: {output_folder}")