linux-scripts/windows-sql-smoke-tests.ps1

###############################################################################
#### windows-sql-smoke-tests.ps1 — Verify SQL Server health               ####
#### Checks services, TCP/IP, auth, database states, AG sync,             ####
#### backup age, TempDB, error log, memory, and disk space.               ####
####                                                                       ####
#### Author:  Phil Connor                                                  ####
#### Contact: contact@mylinux.work                                         ####
#### License: MIT                                                          ####
#### Version: 1.0                                                          ####
####                                                                       ####
#### Usage:   .\windows-sql-smoke-tests.ps1                                ####
####          .\windows-sql-smoke-tests.ps1 -SqlInstance SERVER01\PROD     ####
####          .\windows-sql-smoke-tests.ps1 -OutputFormat tap              ####
####                                                                       ####
#### See -Help for all options.                                            ####
###############################################################################

[CmdletBinding()]
param(
    [string]$SqlInstance = "localhost",
    [int]$SqlPort = 1433,
    [int]$MaxFullBackupAgeHours = 25,
    [int]$MaxLogBackupAgeHours = 1,
    [ValidateSet("text","tap")]
    [string]$OutputFormat = "text",
    [switch]$NoColor,
    [switch]$Help
)

$ErrorActionPreference = "Continue"

# ============================================================================
# HELP
# ============================================================================

if ($Help) {
    @"
Usage: .\windows-sql-smoke-tests.ps1 [OPTIONS]

Smoke-test SQL Server health. PowerShell 5.1+.
Uses Windows authentication. Prefers Invoke-Sqlcmd, falls back to SqlClient.

Parameters:
  -SqlInstance INSTANCE       SQL instance (default: localhost). Use SERVER\INSTANCE for named.
  -SqlPort PORT               TCP port (default: 1433)
  -MaxFullBackupAgeHours N    Max hours since last full backup (default: 25)
  -MaxLogBackupAgeHours N     Max hours since last log backup (default: 1)
  -OutputFormat FORMAT        Output: text (default), tap
  -NoColor                    Disable coloured output
  -Verbose                    Show debug output
  -Help                       Show this help

Examples:
  .\windows-sql-smoke-tests.ps1
  .\windows-sql-smoke-tests.ps1 -SqlInstance "SERVER01\SQLPROD"
  .\windows-sql-smoke-tests.ps1 -MaxFullBackupAgeHours 48 -OutputFormat tap
  .\windows-sql-smoke-tests.ps1 -NoColor
"@
    exit 0
}

# ============================================================================
# STATE
# ============================================================================

$script:Pass = 0
$script:Fail = 0
$script:Skip = 0
$script:Total = 0
$script:Results = @()
$script:StartTime = Get-Date
$script:SqlConn = $null
$script:UseSqlcmd = $false

# ============================================================================
# COLORS
# ============================================================================

function Write-Color {
    param([string]$Text, [string]$Color = "White")
    if ($NoColor) {
        Write-Host $Text
    } else {
        Write-Host $Text -ForegroundColor $Color
    }
}

function Write-Log   { param([string]$Msg) Write-Color "[INFO] $Msg" "Cyan" }
function Write-Warn  { param([string]$Msg) Write-Color "[WARN] $Msg" "Yellow" }
function Write-Err   { param([string]$Msg) Write-Color "[ERROR] $Msg" "Red" }

# ============================================================================
# TEST RESULT RECORDING
# ============================================================================

function Record-Pass {
    param([string]$Name, [string]$Detail = "")
    $script:Pass++
    $script:Total++
    $script:Results += [PSCustomObject]@{ Status="PASS"; Name=$Name; Detail=$Detail }
    if ($OutputFormat -eq "tap") {
        Write-Host "ok $($script:Total) - $Name$(if($Detail){" ($Detail)"})"
    } else {
        $mark = if ($NoColor) { "[PASS]" } else { [char]0x2713 }
        $msg = "  $mark $Name"
        if ($Detail) { $msg += " - $Detail" }
        Write-Color $msg "Green"
    }
}

function Record-Fail {
    param([string]$Name, [string]$Detail = "")
    $script:Fail++
    $script:Total++
    $script:Results += [PSCustomObject]@{ Status="FAIL"; Name=$Name; Detail=$Detail }
    if ($OutputFormat -eq "tap") {
        Write-Host "not ok $($script:Total) - $Name"
        if ($Detail) { Write-Host "  # $Detail" }
    } else {
        $mark = if ($NoColor) { "[FAIL]" } else { [char]0x2717 }
        $msg = "  $mark $Name"
        if ($Detail) { $msg += " - $Detail" }
        Write-Color $msg "Red"
    }
}

function Record-Skip {
    param([string]$Name, [string]$Reason = "")
    $script:Skip++
    $script:Total++
    $script:Results += [PSCustomObject]@{ Status="SKIP"; Name=$Name; Detail=$Reason }
    if ($OutputFormat -eq "tap") {
        Write-Host "ok $($script:Total) - $Name # SKIP $Reason"
    } else {
        $mark = if ($NoColor) { "[SKIP]" } else { [char]0x2298 }
        $msg = "  $mark $Name"
        if ($Reason) { $msg += " - $Reason" }
        Write-Color $msg "Yellow"
    }
}

# ============================================================================
# HELPERS
# ============================================================================

function Test-CommandExists {
    param([string]$Command)
    $null -ne (Get-Command $Command -ErrorAction SilentlyContinue)
}

function Write-Section {
    param([string]$Name)
    if ($OutputFormat -eq "text") {
        Write-Host ""
        Write-Color $Name "White"
    }
}

function Get-ServiceName {
    if ($SqlInstance -eq "localhost" -or $SqlInstance -notmatch "\\") {
        return @{ Engine = "MSSQLSERVER"; Agent = "SQLSERVERAGENT" }
    }
    $instanceName = ($SqlInstance -split "\\")[1]
    return @{ Engine = "MSSQL`$$instanceName"; Agent = "SQLAgent`$$instanceName" }
}

function Get-ConnectionString {
    $server = $SqlInstance
    if ($SqlPort -ne 1433 -and $SqlInstance -notmatch ",") {
        $host_ = if ($SqlInstance -match "\\") { ($SqlInstance -split "\\")[0] } else { $SqlInstance }
        $server = "$host_,$SqlPort"
        if ($SqlInstance -match "\\") {
            $server = "$host_\$(($SqlInstance -split '\\')[1]),$SqlPort"
        }
    }
    return "Server=$server;Integrated Security=True;Connection Timeout=10"
}

function Invoke-SqlQuery {
    param([string]$Query)

    if ($script:UseSqlcmd) {
        try {
            $params = @{
                ServerInstance = $SqlInstance
                Query          = $Query
                ErrorAction    = "Stop"
                QueryTimeout   = 30
            }
            return Invoke-Sqlcmd @params
        } catch {
            Write-Verbose "Invoke-Sqlcmd failed: $($_.Exception.Message)"
            return $null
        }
    }

    try {
        $connStr = Get-ConnectionString
        $conn = New-Object System.Data.SqlClient.SqlConnection($connStr)
        $conn.Open()
        $cmd = $conn.CreateCommand()
        $cmd.CommandText = $Query
        $cmd.CommandTimeout = 30
        $adapter = New-Object System.Data.SqlClient.SqlDataAdapter($cmd)
        $dataset = New-Object System.Data.DataSet
        $adapter.Fill($dataset) | Out-Null
        $conn.Close()
        return $dataset.Tables[0]
    } catch {
        Write-Verbose "SqlClient failed: $($_.Exception.Message)"
        return $null
    }
}

# ============================================================================
# TESTS
# ============================================================================

# -- 1. SQL Server Service ---------------------------------------------------

function Test-SqlService {
    Write-Section "Services"

    $svcNames = Get-ServiceName
    try {
        $svc = Get-Service -Name $svcNames.Engine -ErrorAction Stop
        if ($svc.Status -eq "Running") {
            Record-Pass "SQL Server service running" $svcNames.Engine
        } else {
            Record-Fail "SQL Server service running" "$($svcNames.Engine) status: $($svc.Status)"
        }
    } catch {
        Record-Fail "SQL Server service running" "$($svcNames.Engine) - service not found"
    }
}

# -- 2. SQL Server Agent Service ---------------------------------------------

function Test-AgentService {
    $svcNames = Get-ServiceName
    try {
        $svc = Get-Service -Name $svcNames.Agent -ErrorAction Stop
        if ($svc.Status -eq "Running") {
            Record-Pass "SQL Server Agent running" $svcNames.Agent
        } else {
            Record-Fail "SQL Server Agent running" "$($svcNames.Agent) status: $($svc.Status)"
        }
    } catch {
        Record-Skip "SQL Server Agent running" "Agent service not found"
    }
}

# -- 3. TCP/IP Connectivity -------------------------------------------------

function Test-TcpConnectivity {
    Write-Section "Connectivity"

    $host_ = if ($SqlInstance -match "\\") { ($SqlInstance -split "\\")[0] } else { $SqlInstance }
    if ($host_ -eq "localhost" -or $host_ -eq "." -or $host_ -eq "(local)") { $host_ = "127.0.0.1" }

    try {
        $tcp = Test-NetConnection -ComputerName $host_ -Port $SqlPort -WarningAction SilentlyContinue -ErrorAction Stop
        if ($tcp.TcpTestSucceeded) {
            Record-Pass "TCP/IP connectivity" "${host_}:${SqlPort}"
        } else {
            Record-Fail "TCP/IP connectivity" "${host_}:${SqlPort} - port closed"
        }
    } catch {
        Record-Fail "TCP/IP connectivity" "${host_}:${SqlPort} - $($_.Exception.Message)"
    }
}

# -- 4. SQL Authentication --------------------------------------------------

function Test-SqlAuth {
    $result = Invoke-SqlQuery "SELECT 1 AS TestResult"
    if ($result) {
        Record-Pass "SQL authentication" "SELECT 1 succeeded"
        return $true
    } else {
        Record-Fail "SQL authentication" "could not connect or execute query"
        return $false
    }
}

# -- 5. Database States ------------------------------------------------------

function Test-DatabaseStates {
    Write-Section "Databases"

    $result = Invoke-SqlQuery @"
SELECT name, state_desc
FROM sys.databases
WHERE database_id > 4
ORDER BY name
"@

    if (-not $result) {
        Record-Skip "database states" "could not query sys.databases"
        return
    }

    $rows = @($result)
    $offline = @($rows | Where-Object { $_.state_desc -ne "ONLINE" })

    if ($offline.Count -eq 0) {
        Record-Pass "database states" "$($rows.Count) databases ONLINE"
    } else {
        $offList = ($offline | ForEach-Object { "$($_.name)=$($_.state_desc)" }) -join ", "
        Record-Fail "database states" "$($offline.Count) not ONLINE: $offList"
    }
}

# -- 6. Availability Group Sync Health --------------------------------------

function Test-AGSyncHealth {
    Write-Section "Availability Groups"

    $result = Invoke-SqlQuery @"
SELECT ag.name AS ag_name,
       ags.synchronization_health_desc
FROM sys.availability_groups ag
JOIN sys.dm_hadr_availability_group_states ags ON ag.group_id = ags.group_id
"@

    if (-not $result -or @($result).Count -eq 0) {
        Record-Skip "AG sync health" "no Availability Groups configured"
        return
    }

    foreach ($row in @($result)) {
        $health = $row.synchronization_health_desc
        if ($health -eq "HEALTHY") {
            Record-Pass "AG sync health" "$($row.ag_name) $health"
        } else {
            Record-Fail "AG sync health" "$($row.ag_name) $health"
        }
    }
}

# -- 7. AG Replica Health ----------------------------------------------------

function Test-AGReplicaHealth {
    $result = Invoke-SqlQuery @"
SELECT ar.replica_server_name,
       ars.synchronization_state_desc,
       ars.connected_state_desc
FROM sys.availability_replicas ar
JOIN sys.dm_hadr_availability_replica_states ars ON ar.replica_id = ars.replica_id
"@

    if (-not $result -or @($result).Count -eq 0) {
        Record-Skip "AG replica health" "no Availability Groups configured"
        return
    }

    $rows = @($result)
    $unhealthy = @($rows | Where-Object {
        $_.synchronization_state_desc -notin @("SYNCHRONIZED","SYNCHRONIZING") -or
        $_.connected_state_desc -ne "CONNECTED"
    })

    if ($unhealthy.Count -eq 0) {
        Record-Pass "AG replica health" "$($rows.Count) replicas healthy"
    } else {
        $list = ($unhealthy | ForEach-Object {
            "$($_.replica_server_name)=$($_.synchronization_state_desc)/$($_.connected_state_desc)"
        }) -join ", "
        Record-Fail "AG replica health" "$($unhealthy.Count) unhealthy: $list"
    }
}

# -- 8. AG Listener Connectivity --------------------------------------------

function Test-AGListener {
    $result = Invoke-SqlQuery @"
SELECT dns_name, port
FROM sys.availability_group_listeners
"@

    if (-not $result -or @($result).Count -eq 0) {
        Record-Skip "AG listener" "no AG listener configured"
        return
    }

    foreach ($row in @($result)) {
        try {
            $tcp = Test-NetConnection -ComputerName $row.dns_name -Port $row.port -WarningAction SilentlyContinue -ErrorAction Stop
            if ($tcp.TcpTestSucceeded) {
                Record-Pass "AG listener" "$($row.dns_name):$($row.port)"
            } else {
                Record-Fail "AG listener" "$($row.dns_name):$($row.port) - unreachable"
            }
        } catch {
            Record-Fail "AG listener" "$($row.dns_name):$($row.port) - $($_.Exception.Message)"
        }
    }
}

# -- 9. Full Backup Age -----------------------------------------------------

function Test-FullBackupAge {
    Write-Section "Backups"

    $result = Invoke-SqlQuery @"
SELECT d.name,
       MAX(b.backup_finish_date) AS last_full
FROM sys.databases d
LEFT JOIN msdb.dbo.backupset b
    ON d.name = b.database_name AND b.type = 'D'
WHERE d.database_id > 4 AND d.state = 0
GROUP BY d.name
ORDER BY d.name
"@

    if (-not $result) {
        Record-Skip "full backup age" "could not query backup history"
        return
    }

    $now = Get-Date
    $failures = @()

    foreach ($row in @($result)) {
        if (-not $row.last_full -or $row.last_full -eq [DBNull]::Value) {
            $failures += "$($row.name)=NEVER"
            continue
        }
        $age = ($now - [datetime]$row.last_full).TotalHours
        if ($age -gt $MaxFullBackupAgeHours) {
            $failures += "$($row.name)=$([math]::Floor($age))h ago"
        }
    }

    if ($failures.Count -eq 0) {
        Record-Pass "full backup age" "all databases within ${MaxFullBackupAgeHours}h threshold"
    } else {
        Record-Fail "full backup age" ($failures -join ", ")
    }
}

# -- 10. Log Backup Age ----------------------------------------------------

function Test-LogBackupAge {
    $result = Invoke-SqlQuery @"
SELECT d.name, d.recovery_model_desc,
       MAX(b.backup_finish_date) AS last_log
FROM sys.databases d
LEFT JOIN msdb.dbo.backupset b
    ON d.name = b.database_name AND b.type = 'L'
WHERE d.database_id > 4 AND d.state = 0
  AND d.recovery_model_desc IN ('FULL','BULK_LOGGED')
GROUP BY d.name, d.recovery_model_desc
ORDER BY d.name
"@

    if (-not $result -or @($result).Count -eq 0) {
        Record-Skip "log backup age" "no FULL/BULK_LOGGED recovery model databases"
        return
    }

    $now = Get-Date
    $failures = @()

    foreach ($row in @($result)) {
        if (-not $row.last_log -or $row.last_log -eq [DBNull]::Value) {
            $failures += "$($row.name)=NEVER"
            continue
        }
        $age = ($now - [datetime]$row.last_log).TotalHours
        if ($age -gt $MaxLogBackupAgeHours) {
            $ageMin = [math]::Floor(($now - [datetime]$row.last_log).TotalMinutes)
            $failures += "$($row.name)=${ageMin}m ago"
        }
    }

    if ($failures.Count -eq 0) {
        Record-Pass "log backup age" "all databases within ${MaxLogBackupAgeHours}h threshold"
    } else {
        Record-Fail "log backup age" ($failures -join ", ")
    }
}

# -- 11. TempDB Configuration -----------------------------------------------

function Test-TempDBConfig {
    Write-Section "TempDB"

    $result = Invoke-SqlQuery @"
SELECT
    (SELECT COUNT(*) FROM tempdb.sys.database_files WHERE type = 0) AS file_count,
    (SELECT cpu_count FROM sys.dm_os_sys_info) AS cpu_count
"@

    if (-not $result) {
        Record-Skip "TempDB configuration" "could not query TempDB info"
        return
    }

    $row = @($result)[0]
    $files = [int]$row.file_count
    $cpus = [int]$row.cpu_count
    $recommended = [math]::Min($cpus, 8)

    if ($files -ge $recommended) {
        Record-Pass "TempDB file count" "$files files, $cpus cores"
    } else {
        Record-Fail "TempDB file count" "$files files but $cpus cores (recommend $recommended)"
    }
}

# -- 12. TempDB Free Space -------------------------------------------------

function Test-TempDBSpace {
    $result = Invoke-SqlQuery @"
SELECT
    SUM(unallocated_extent_page_count) * 8.0 / 1024 AS free_mb
FROM tempdb.sys.dm_db_file_space_usage
"@

    if (-not $result) {
        Record-Skip "TempDB free space" "could not query TempDB space"
        return
    }

    $freeMB = [math]::Round([double]@($result)[0].free_mb, 1)
    $freeGB = [math]::Round($freeMB / 1024, 2)

    if ($freeMB -gt 100) {
        Record-Pass "TempDB free space" "${freeGB} GB free"
    } else {
        Record-Fail "TempDB free space" "${freeMB} MB free — low"
    }
}

# -- 13. Error Log -----------------------------------------------------------

function Test-ErrorLog {
    Write-Section "Error Log"

    $cutoff = (Get-Date).AddHours(-24).ToString("yyyy-MM-ddTHH:mm:ss")

    $result = Invoke-SqlQuery @"
EXEC xp_readerrorlog 0, 1, NULL, NULL, '$cutoff', NULL
"@

    if (-not $result) {
        Record-Pass "error log" "no entries or xp_readerrorlog not accessible"
        return
    }

    $severe = @($result | Where-Object {
        $text = if ($_.Text) { $_.Text } elseif ($_.LogText) { $_.LogText } else { "" }
        $text -match "Severity:\s*(1[7-9]|2[0-5])"
    })

    if ($severe.Count -eq 0) {
        Record-Pass "error log clean" "0 severity 17+ in last 24h"
    } else {
        $sample = $severe[0]
        $text = if ($sample.Text) { $sample.Text } elseif ($sample.LogText) { $sample.LogText } else { "unknown" }
        $short = $text.Substring(0, [math]::Min(80, $text.Length))
        Record-Fail "error log" "$($severe.Count) severity 17+ in 24h — latest: $short"
    }
}

# -- 14. Max Memory Configuration -------------------------------------------

function Test-MaxMemory {
    Write-Section "Configuration"

    $result = Invoke-SqlQuery @"
SELECT value_in_use
FROM sys.configurations
WHERE name = 'max server memory (MB)'
"@

    if (-not $result) {
        Record-Skip "max memory" "could not query sys.configurations"
        return
    }

    $maxMB = [int64]@($result)[0].value_in_use

    if ($maxMB -ge 2147483647) {
        Record-Fail "max memory configured" "still default (2 TB) — set an explicit limit"
    } else {
        Record-Pass "max memory configured" "$maxMB MB"
    }
}

# -- 15. Disk Space ----------------------------------------------------------

function Test-DiskSpace {
    Write-Section "Disk Space"

    $result = Invoke-SqlQuery @"
SELECT DISTINCT
    LEFT(physical_name, 3) AS drive
FROM sys.master_files
WHERE database_id > 0
"@

    if (-not $result) {
        Record-Skip "disk space" "could not query sys.master_files"
        return
    }

    $drives = @($result) | ForEach-Object { $_.drive.TrimEnd("\") }
    $driveInfo = @()
    $allOk = $true

    foreach ($d in ($drives | Sort-Object -Unique)) {
        try {
            $vol = Get-PSDrive -Name $d.TrimEnd(":") -ErrorAction Stop
            $freeGB = [math]::Round($vol.Free / 1GB, 1)
            $usedGB = [math]::Round($vol.Used / 1GB, 1)
            $totalGB = $freeGB + $usedGB
            $pctFree = if ($totalGB -gt 0) { [math]::Round(($freeGB / $totalGB) * 100, 0) } else { 0 }
            $driveInfo += "${d}: ${pctFree}% free (${freeGB} GB)"
            if ($pctFree -lt 10) { $allOk = $false }
        } catch {
            $driveInfo += "${d}: unknown"
        }
    }

    $detail = $driveInfo -join ", "
    if ($allOk) {
        Record-Pass "disk space" $detail
    } else {
        Record-Fail "disk space" "$detail — drive below 10% free"
    }
}

# ============================================================================
# OUTPUT
# ============================================================================

function Write-Header {
    if ($OutputFormat -eq "tap") {
        Write-Host "TAP version 13"
    } else {
        Write-Host ""
        Write-Color "Windows SQL Smoke Tests" "White"
        Write-Host "Instance: $SqlInstance"
        Write-Host "Port:     $SqlPort"
        Write-Host "Time:     $(Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ')"
    }
}

function Write-Summary {
    $duration = [math]::Floor(((Get-Date) - $script:StartTime).TotalSeconds)

    if ($OutputFormat -eq "tap") {
        Write-Host "1..$($script:Total)"
        Write-Host "# pass $($script:Pass)"
        Write-Host "# fail $($script:Fail)"
        Write-Host "# skip $($script:Skip)"
    } else {
        Write-Host ""
        $separator = [string]::new([char]0x2500, 40)
        Write-Color $separator "White"
        Write-Color "Summary  $SqlInstance" "White"

        $summaryLine = "  $($script:Pass) passed  $($script:Fail) failed  $($script:Skip) skipped  (${duration}s)"
        Write-Host $summaryLine
        Write-Color $separator "White"

        if ($script:Fail -eq 0) {
            Write-Color "All tests passed." "Green"
        } else {
            Write-Color "$($script:Fail) test(s) failed." "Red"
        }
    }
}

# ============================================================================
# MAIN
# ============================================================================

Write-Header

# Detect SQL connectivity method
try {
    Import-Module SqlServer -ErrorAction Stop
    $script:UseSqlcmd = $true
    Write-Verbose "Using SqlServer module (Invoke-Sqlcmd)"
} catch {
    try {
        Import-Module SQLPS -DisableNameChecking -ErrorAction Stop
        $script:UseSqlcmd = $true
        Write-Verbose "Using SQLPS module (Invoke-Sqlcmd)"
    } catch {
        $script:UseSqlcmd = $false
        Write-Verbose "Using System.Data.SqlClient (no SQL module)"
    }
}

# Run all tests
Test-SqlService
Test-AgentService
Test-TcpConnectivity

$authOk = Test-SqlAuth
if ($authOk) {
    Test-DatabaseStates
    Test-AGSyncHealth
    Test-AGReplicaHealth
    Test-AGListener
    Test-FullBackupAge
    Test-LogBackupAge
    Test-TempDBConfig
    Test-TempDBSpace
    Test-ErrorLog
    Test-MaxMemory
    Test-DiskSpace
} else {
    Write-Warn "Skipping SQL query tests — authentication failed"
    Record-Skip "database states" "auth failed"
    Record-Skip "AG sync health" "auth failed"
    Record-Skip "AG replica health" "auth failed"
    Record-Skip "AG listener" "auth failed"
    Record-Skip "full backup age" "auth failed"
    Record-Skip "log backup age" "auth failed"
    Record-Skip "TempDB configuration" "auth failed"
    Record-Skip "TempDB free space" "auth failed"
    Record-Skip "error log" "auth failed"
    Record-Skip "max memory" "auth failed"
    Record-Skip "disk space" "auth failed"
}

Write-Summary

if ($script:Fail -eq 0) { exit 0 } else { exit 1 }