Kiln » KilnSupportScripts Powershell Scripts to help monitor a Kiln environment. Contact Fog Creek support before using.
Clone URL:  
Pushed to one repository · View In Graph Contained in tip

merge

Changeset 3e61a62ffd2c

Parents bb959c445c8a

Parents 57dcd44b2fb2

by Profile picture of Developer Wiki Admin AccountDeveloper Wiki Admin Account

Changes to 3 files · Browse files at 3e61a62ffd2c Showing diff from parent bb959c445c8a 57dcd44b2fb2 Diff from another changeset...

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
 
 
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
 
 
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
 
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
 <#  .SYNOPSIS   Checks the vital signs of a Kiln Server    .DESCRIPTION   This script will check a variety of vital signs to see if a Kiln Server is in good health.   When errors are found, the script can write them to std-out, email a message, and/or attempt   to automatically restart the necessary Kiln Services.    .EXAMPLE   ./Monitor-Kiln.ps1 -kilnURL "http://localhost/fogbugz/kiln"   This is the most basic check, it will simply perform checks on all Kiln services at their default locations, write   information to std-out, and attempt to automatically restart the Kiln Storage Service and Kiln Queuing   Service when errors are found.    .EXAMPLE   ./Monitor-Kiln.ps1 -kilnURL "http://localhost/fogbugz/kiln" -smtpServer smtp.myserver.com -from kiln-error-report@company.com -to kiln-admin@company.com -username bob -password pa55w0rd -smtpSsl   This will perform checks on all Kiln services at their default locations, and attempt to automatically   restart the Kiln Storage Service and Kiln Queuing Service when errors are found. Additionally, any errors   found will generate an email to "kiln-admin@company.com" from "kiln-error-report@company.com" which will be   sent via the smtp server "smtp.myserver.com" using SSL and the username "bob" with password "pa55w0rd" to   authenticate.    .EXAMPLE   ./Monitor-Kiln.ps1 -kilnURL "http://localhost/fogbugz/kiln" -iisSiteName Default   This will perform checks on all Kiln services at their default location, write information to std-out,   and attempt to automatically restart the Kiln Storage Service and Kiln Queuing Service when errors are   found. It will also attempt to restart the IIS website named "Default" if the Kiln front page won't load.    .EXAMPLE   ./Monitor-Kiln.ps1 -kilnURL "http://localhost/fogbugz/kiln" -noRestart   This will perform checks on all Kiln services at their default location, write information to std-out,   but will not attempt to restart any services.    .PARAMETER kilnURL   The URL to reach the Kiln web page.    .PARAMETER smtpServer   An SMPT server that this script can use to send error messages.    .PARAMETER from   The from address to appear in emailed error messages.    .PARAMETER to   The destination address for emailing error messages.    .PARAMETER username   The username to log into the SMTP server.    .PARAMETER password   The password to log into the SMTP server.    .PARAMETER smtpSsl   A switch to use SSL when connecting to the SMTP server (by default SSL is off).    .PARAMETER smtpPort   The port to connect to the SMTP server on (uses 587 by defualt).    .PARAMETER backendVersionUrl   The URL of the Kiln backend. The default value is almost always correct.    .PARAMETER esUrl   The URL of the ElasticSearch web front end. The default value is almost always correct.    .PARAMETER queueStatsUrl   The URL of the Queue Stats page. The default value is almost always correct.    .PARAMETER iisSiteName   The name of the IIS Website that FogBugz and Kiln are running under. If you provide this,   the script will attempt to restart IIS if it cannot load the Kiln web page.   -.PARAMETER noRestart - A flag which when set will force the script to only send alert messages and not - attempt to restart any services automatically. +.PARAMETER tryAutoRestart + A flag which when set will modify the behavior of the script to attempt to restart services + which it detects are not running or to reenqueue tasks when too many are found.    .PARAMETER reenqueuePath   The path to the Reenqueue script, this is used to try to resuscitate the Queue when problems are detected.   The file is located:   https://developers.kilnhg.com/Code/Kiln/Group/KilnSupportScripts/File/misc/reenqueue_2.9_tasks.ps1?rev=tip    .NOTES   Author: Quentin Schroeder   Date: Dec 13, 2012  #>        param([string]$kilnURL = "http://localhost/fogbugz/kiln",   [string]$smtpServer = "",   [string]$from = "",   [string]$to = "",   [string]$username = "",   [string]$password = "",   [switch]$smtpSsl,   [string]$smtpPort = 587,   [string]$backendVersionUrl = "http://localhost:56783/version",   [string]$esUrl = "http://localhost:9200/",   [string]$queueStatsUrl = "http://localhost:56785/stats.json",   [string]$iisSiteName = "", - [switch]$noRestart, + [switch]$tryAutoRestart,   [string]$reenqueuePath = ".\reenqueue_2.9_tasks.ps1"   )            ############################# FUNCTION DEFINITIONS #############################    function Write-WithTime($msg, $error=$false){   if ($error)   {   write-host "$(Get-Date -format o) $msg" -foregroundcolor "red"   return   }   write-host "$(Get-Date -format o) $msg"  }        function Handle-Error($msg)  {   Write-WithTime " ERROR!" $true   if ($smtpServer -and $from -and $to -and $username -and $password)   {   try   {   $SmtpClient = New-Object Net.Mail.SmtpClient($smtpServer, $smtpPor)   $SmtpClient.EnableSsl = $smtpSsl   $SmtpClient.Credentials = New-Object System.Net.NetworkCredential($username, $password);   $SmtpClient.Send($from, $to, "Kiln Server Error Report!", $msg)   }   catch [Exception]   {   $exceptionMessage = $_.Exception.Message   Write-WithTime " Unable to send email, an error occurred.`n$exceptionMessage" $true   }   }   elseif ($smtpServer -or $from -or $to -or $username -or $password -or $smtpSsl -or ($smtpPort -ne 587))   {   Write-WithTime " Unable to send email, some required parameters are missing! Writing to std-out." $true   Write-WithTime " $msg" $true   }   else   {   Write-WithTime " $msg" $true   }  }        function Test-ProcessRunning($processName, $count = 1, [ref]$serviceNeedsRestart)  {   Write-WithTime "Checking that $count instance(s) of $processName are running..."   $result = (Get-Process $processName -ea SilentlyContinue | select id | measure).count -ge $count   if (!($result))   {   Handle-Error("Critical Kiln Server error. Process `'$processName`' could not be found!")   $serviceNeedsRestart.value = $true   }  }        function Test-ElasticSearchRunning([ref]$serviceNeedsRestart)  {   Write-WithTime "Checking that 1 instances(s) of ElasticSearch are running..."   $result = $false   (Get-WmiObject win32_process -Filter "name like 'java.exe'") | ForEach-Object {   if (($_.CommandLine | select-string "ElasticSearch" | measure).count -eq 1) {   $result = $true   }   }   if (!($result))   {   Handle-Error("Critical Kiln Server error. Elastic Search is not running!")   $serviceNeedsRestart.value = $true   }  }        function Test-HttpResponse($url, $searchString, [ref]$serviceNeedsRestart)  {   Write-WithTime "Checking for expected response from URL ($url)..."     try   {   $response = (New-Object net.webclient).DownloadString($url)   $response = (New-Object net.webclient).DownloadString($url)   $result = $response | select-string $searchString   if (!($result))   {   Handle-Error("Critical Kiln Server error. URL [$url] did not contain the expected string `'$searchString`'")   $serviceNeedsRestart.value = $true   }   }   catch [Net.WebException]   {   $exceptionMessage = $_.Exception.Message   Handle-Error("Critical Kiln Server error. Could not reach URL [$url]`n$exceptionMessage")   $serviceNeedsRestart.value = $true   }  }      function Test-QueueStats([ref]$serviceNeedsRestart)  {   try   {   $response = (New-Object System.Net.WebClient).DownloadString($queueStatsUrl)     $stats = Convert-JsonToXml($response)     $statsTime = [long]$stats.root.currentTime."#text"   $timeDiff = (((Get-Date).ToUniversalTime().Ticks / 10e6) - $statsTime)   $queueLength = [int]$stats.root.queueLength."#text"   $runningTaskCount = $stats.root.runningTasks.ChildNodes.count   $oldestTaskAge = [int]$stats.root.oldestRunningTaskTotalSeconds."#text"       Write-WithTime ""   Write-WithTime "--Queue Stats Information--"   Write-WithTime " Time Accuracy: within $timeDiff seconds"   Write-WithTime " Queue Length: $queueLength"   Write-WithTime " Running Task Count: $runningTaskCount"   Write-WithTime " Age of oldest task: $oldestTaskAge seconds"       if (($timeDiff -gt 5) -or ($timeDiff -lt -5))   {   $serviceNeedsRestart.value = $true   }       # This is a warning sign, so just alert, but don't attempt to restart the service yet.   if ($queueLength -gt 50)   {   Handle-Error("The Kiln Queue is exceptionally long ($queueLength tasks), this might by a symptom of a recent large change or a problem.")   }       # This is a warning sign, so just alert, but don't attempt to restart the service yet.   if ($runningTaskCount -gt 16)   {   Handle-Error("Too many tasks ($runningTaskCount) are currently running in the Kiln Queue.")   }       # Longer than 15 minutes on a task? This is a bad sign, time to give it some help!   if ( $oldestTaskAge -gt (15*60) )   {   Handle-Error("Oldest task has been running for $oldestTaskAge seconds! Attempting to retry all running tasks.")     if (test-path $reenqueuePath)   {   & $reenqueuePath -retryRunning   }   else   {   Write-WithTime "Reenqueue script not found at ($reenqueuePath), unable to retry running tasks in Kiln Queue."   }   }     }   catch [Exception]   {   $exceptionMessage = $_.Exception.Message   Handle-Error("Error obtaining or processing Kiln Queue Stats`n$exceptionMessage")   $serviceNeedsRestart.value = $true   }  }          # This is provided as an alternative to ConvertFrom-JSON, which requires Powershell V3.  # Powershell V3 is not on most servers by default and requires a reboot to install, so here we are instead...  # Source: (https://www.cogmotive.com/blog/powershell/parsing-json-in-powershell-xml-the-member-item-is-already-present)  Add-Type -Assembly System.ServiceModel.Web,System.Runtime.Serialization  function Convert-JsonToXml([string]$json)  {   $bytes = [byte[]][char[]]$json   $quotas = [System.Xml.XmlDictionaryReaderQuotas]::Max   $jsonReader = [System.Runtime.Serialization.Json.JsonReaderWriterFactory]::CreateJsonReader($bytes,$quotas)   try   {   $xml = new-object System.Xml.XmlDocument   $xml.Load($jsonReader)   $xml   }   finally   {   $jsonReader.Close()   }  }        ########################### END FUNCTION DEFINITIONS ###########################           -$tryAutoRestart = !($noRestart) # Needed a negative name for the parameter since the default is true. Sanity rename here.  $kssNeedsRestart = $false  $kqsNeedsRestart = $false  $iisNeedsRestart = $false    Test-ProcessRunning "backend" 1 -serviceNeedsRestart ([ref]$kssNeedsRestart)  Test-ProcessRunning "redis-server" 2 -serviceNeedsRestart ([ref]$kssNeedsRestart)  Test-ElasticSearchRunning -serviceNeedsRestart ([ref]$kssNeedsRestart)  Test-ProcessRunning "QueueService" -serviceNeedsRestart ([ref]$kqsNeedsRestart)    Test-HttpResponse $kilnURL "Log on to Kiln" -serviceNeedsRestart ([ref]$iisNeedsRestart)  Test-HttpResponse $backendVersionUrl '"hg_version"' -serviceNeedsRestart ([ref]$kssNeedsRestart)  Test-HttpResponse $esUrl '"status" : 200' -serviceNeedsRestart ([ref]$kssNeedsRestart)  Test-HttpResponse $queueStatsUrl '"queueName":"Kiln"' -serviceNeedsRestart ([ref]$kqsNeedsRestart)    Test-QueueStats -serviceNeedsRestart ([ref]$kqsNeedsRestart)                Write-Host "`n"        if ($iisNeedsRestart -and $iisSiteName -and $tryAutoRestart)  {   Write-WithTime "Attemping to restart site $iisSiteName in IIS...`n"     $appcmd = $env:SystemRoot + "\system32\inetsrv\appcmd.exe"   $stopArgs = "stop site /site.name:$iisSiteName"   $startArgs = "start site /site.name:$iisSiteName"     Start-Process $appcmd $stopArgs   Start-Process $appcmd $startArgs  }      if ($kssNeedsRestart -and $tryAutoRestart)  {   Write-WithTime "Attempting to restart Kiln Storage Service..."     # Find and stop the java.exe process that is running Elastic Search   (Get-WmiObject win32_process -Filter "name like 'java.exe'") | ForEach-Object {   if (($_.CommandLine | select-string "ElasticSearch" | measure).count -eq 1) {   Stop-Process -id $_.ProcessId -Force -EV Err -EA "SilentlyContinue"   }   }     # Try to stop the processes, but don't show any errors (it might already be stopped)     # This will stop both of the redis-server.exe processes because the have the same name   Stop-Process -name redis-server -Force -EV Err -EA "SilentlyContinue"     Stop-Process -name backend -Force -EV Err -EA "SilentlyContinue"     Start-Service KilnStorageService  }        if ($kqsNeedsRestart -and $tryAutoRestart)  {   Write-WithTime "Attempting to restart Kiln Queuing Service..."     # Try to stop the process, but don't show any errors (it might already be stopped)   Stop-Process -name QueueService -Force -EV Err -EA "SilentlyContinue"     Start-Sleep -s 1   Start-Service "Kiln Queuing Service"  }            write-host "`n"
 
6
7
8
9
 
 
10
11
12
 
143
144
145
 
 
 
 
 
 
146
147
148
 
6
7
8
 
9
10
11
12
13
 
144
145
146
147
148
149
150
151
152
153
154
155
@@ -6,7 +6,8 @@
 #  # The script should be run on a machine which has sufficient disk space to clone copies of ALL of the  # repositories from the old kiln server. This means it also requires that Mercurial be installed in order to run. - +# +# Beware: Branches will not migrate properly if the parent repo has been changed since the original point of the branch!       @@ -143,6 +144,12 @@
  # If we don't find an existing repo, then create a new repo with the information we grabbed from the old server   if (not dest_repo):   if (src_repo['ixParent']): + # By setting the ixParent field, when Repo/Create is called, it will create a new branch from the ixParent Repo. + # This means, that the point at which the src repo branched off from the parent can and most likely will be different from the + # point at which the dest repo gets branched. (One option I know of to fix this, after the fact, is to strip changesets in + # the dest repo back to the point where the branch was created in the src repo. Then push the src repo to the dest repo and + # the src and dest branches should be back in sync. + debug_print(" !!! Warning: Branches may not migrate properly !!!")   data = urllib.urlencode({'token': destinationToken,   'sName': src_repo['sName'],   'sDescription': src_repo['sDescription'],
Change 1 of 1 Show Entire File misc/​repair_all_repos.ps1 Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
@@ -0,0 +1,117 @@
+<# +.SYNOPSIS + Repairs all repositories on a Kiln server + +.DESCRIPTION + Uses the Kiln API to gather information about all repositories hosted on the Kiln server and then + issues API commands to repair each of the repositories. + +.EXAMPLE + .\repair_all_repos.ps1 -kilnUrl "http://my.fogbugz/kiln" -token "gfd4567f2g7d8f2g4d6f8sdsdg88" + +.PARAMETER kilnUrl + Where Kiln is hosted, this URL must be reachable from the local machine + +.PARAMETER token + A valid API token for the Kiln server + +.NOTES + Author: Quentin Schroeder + Version: 1.0 + Date: Jan 22, 2013 +#> + + +param([string]$kilnUrl = "http://localhost/kiln", + [string]$token = "" + ) + + +function main() { + if ($token -eq "") {write-host "Must provide a token to authorize access to Kiln."; exit;} + + $url = "{0}/Api/1.0/Project?token={1}" -f ($kilnUrl, $token) + write-host ("Getting repo information from {0}" -f $url) + $response = (New-Object System.Net.WebClient).DownloadString($url) + if (!($?)) {write-host "Failed to get repo information. Confirm the kilnUrl parameter is correct."; exit;} + + $responseXml = Convert-JsonToXml($response) + $projects = $responseXml.root.childnodes + + foreach ($project in $projects) { + foreach ($group in $project.repoGroups.childnodes) { + foreach ($repo in $group.repos.childnodes) { + $ixRepo = $repo.ixRepo."#text" + $repoName = $repo.sName."#text" + $URI = "{0}/api/1.0/repo/{1}/repair?token={2}" -f ($kilnUrl, $ixRepo, $token) + write-host ("Attempting to repair repo '{0}' (ixRepo: {1})" -f ($repoName, $ixRepo) ) + Execute-HTTPPostCommand $URI "a=1" + if (!($?)) {write-host "Failed!!"} + } + } + } +} + + + +# This is provided as an alternative to ConvertFrom-JSON, which requires Powershell V3 +# which is not on most servers by default and requires a reboot to install. +# Source: (https://www.cogmotive.com/blog/powershell/parsing-json-in-powershell-xml-the-member-item-is-already-present) +Add-Type -Assembly System.ServiceModel.Web,System.Runtime.Serialization +function Convert-JsonToXml([string]$json) +{ + $bytes = [byte[]][char[]]$json + $quotas = [System.Xml.XmlDictionaryReaderQuotas]::Max + $jsonReader = [System.Runtime.Serialization.Json.JsonReaderWriterFactory]::CreateJsonReader($bytes,$quotas) + try + { + $xml = new-object System.Xml.XmlDocument + $xml.Load($jsonReader) + $xml + } + finally + { + $jsonReader.Close() + } +} + + + +# Source: (http://stackoverflow.com/questions/5246836/user-powershell-script-to-post-to-url) +function Execute-HTTPPostCommand() { + param( + [string] $target = $null, + [string] $post = $null + ) + + $webRequest = [System.Net.WebRequest]::Create($target) + $webRequest.ContentType = "text/html" + $PostStr = [System.Text.Encoding]::UTF8.GetBytes($post) + $webrequest.ContentLength = $PostStr.Length + $webRequest.ServicePoint.Expect100Continue = $false + $webRequest.Method = "POST" + + $requestStream = $webRequest.GetRequestStream() + $requestStream.Write($PostStr, 0,$PostStr.length) + $requestStream.Close() + + try + { + [System.Net.WebResponse] $resp = $webRequest.GetResponse() + } + catch [Net.WebException] + { + write-host $_.Exception.ToString() + } + + $rs = $resp.GetResponseStream() + [System.IO.StreamReader] $sr = New-Object System.IO.StreamReader -argumentList $rs + [string] $results = $sr.ReadToEnd() + + return $results +} + + + +main +