File:
1.19.1b/server/bin/tester.php (
View as Code)
1: 2: /* -------------------------------------------------------------
3: This file is part of FreeNATS
4:
5: FreeNATS is (C) Copyright 2008-2017 PurplePixie Systems
6:
7: FreeNATS is free software: you can redistribute it and/or modify
8: it under the terms of the GNU General Public License as published by
9: the Free Software Foundation, either version 3 of the License, or
10: (at your option) any later version.
11:
12: FreeNATS is distributed in the hope that it will be useful,
13: but WITHOUT ANY WARRANTY; without even the implied warranty of
14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15: GNU General Public License for more details.
16:
17: You should have received a copy of the GNU General Public License
18: along with FreeNATS. If not, see www.gnu.org/licenses
19:
20: For more information see www.purplepixie.org/freenats
21: -------------------------------------------------------------- */
22:
23: if ((isset($argc))&&(isset($argv))) // specific node or all nodes
24: {
25: if ($argc>1)
26: {
27: $nfilter=$argv[1];
28: }
29: else $nfilter="";
30: }
31:
32: require("include.php");
33:
34: $dbt="";
35:
36: function db($txt,$nl=true) // debug text
37: {
38: global $dbt;
39: echo $txt;
40: $dbt.=$txt;
41: if ($nl)
42: {
43: echo "\n";
44: $dbt.="
\n";
45: }
46: }
47:
48: $NATS->Start();
49: if ($nfilter!="") $st=": Node ".$nfilter;
50: else $st="";
51: db("NATS Tester Script Starting".$st);
52:
53: $suspend = $NATS->Cfg->Get("site.tester.suspended",0);
54:
55: if ($suspend != 1)
56: {
57:
58: $highalertlevel=-1;
59: $talertc=0;
60:
61: // check if already running
62: $still_running=false;
63: $cq="SELECT trid,startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1";
64: $cr=$NATS->DB->Query($cq);
65:
66: if ($runrow=$NATS->DB->Fetch_Array($cr))
67: { // yes there is a testrun session for this node(s)
68: $timelimit=$NATS->Cfg->Get("test.session.limit",60*60);
69: if ( (!is_numeric($timelimit)) || ($timelimit<0) ) $timelimit=60*60; // bogus config value
70:
71: // n.b. a timelimit of 0 means the session will never expire so...
72: if ( ($timelimit>0) && ((time()-$runrow['startx'])>$timelimit) )
73: {
74: // valid time limit and the difference is more than the limit so close it
75: $uq="UPDATE fntestrun SET finishx=1 WHERE trid=".$runrow['trid'];
76: $NATS->DB->Query($uq);
77: if ($NATS->DB->Affected_Rows()>0)
78: {
79: $NATS->Event("Tester Already Running: Cleared",3,"Tester","Stale");
80: db("Tester Already Running: Cleared");
81: }
82: else $NATS->Event("Tester Already Running: Failed to Clear",1,"Tester","Stale"); // weirdness... run anyway
83: }
84: else $still_running=true; // either never timesout or newer than timelimit ago
85: }
86: $NATS->DB->Free($cr);
87:
88: // and if it is then don't continue
89: if ($still_running)
90: {
91: $NATS->Event("Tester Already Running: Aborted",1,"Tester","Error");
92: db("Tester Already Running: Aborted");
93: $NATS->Stop();
94: exit();
95: }
96:
97:
98: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")";
99: $NATS->DB->Query($gq);
100: $trid=$NATS->DB->Insert_Id();
101: db("Test ID: ".$trid." (Started at ".nicedt(time()).")");
102: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start");
103:
104: db(" ");
105:
106: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx)
107:
108: $q="SELECT * FROM fnnode WHERE nodeenabled=1";
109: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\"";
110: $q.=" AND nextrunx<=".time();
111:
112: $r=$NATS->DB->Query($q);
113:
114:
115: while ($row=$NATS->DB->Fetch_Array($r))
116: {
117: $dotests=true;
118: $alertlevel=0;
119: $alerts=array();
120: $alertc=0;
121: db("NodeID: ".$row['nodeid']);
122: $NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node");
123:
124: // Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested
125: if ($row['scheduleid']!=0) // has a schedule
126: {
127: db(" Has Schedule: Yes - Checking");
128: $run=run_x_in_schedule(time(),$row['scheduleid']);
129: if (!$run)
130: {
131: db(" In Schedule: No - Skipping Tests");
132: $NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node");
133: $dotests=false;
134: $alertlevel=-1;
135: }
136: else db(" In Schedule: Yes");
137: }
138:
139: $eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests );
140: $NATS->EventHandler("node_test_start",$eventdata);
141:
142:
143: $ptr=0;
144: $pal=0;
145:
146:
147: // Update lastrun and nextrun regardless of dotests
148: $q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\"";
149: $NATS->DB->Query($q);
150:
151: $pingpassed=false; // this will only be set to true if a test is done and passes - for the "child" nodes
152: if ($row['pingtest']&&$dotests)
153: {
154: db(" Ping Test: Yes");
155: $NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping");
156: $ptr=PingTest($row['hostname']);
157: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
158: db(" Ping Returned: ".$ptr);
159: if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) )
160: {
161: $att=$NATS->Cfg->Get("test.icmp.attempts","2");
162: for ($a=2; $a<=$att; $a++) // starting on second attempt
163: {
164: // try again...
165: test_sleep();
166: db(" Trying Ping Again - X".$a);
167: $NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping");
168: $ptr=PingTest($row['hostname']);
169: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
170: db(" Ping Returned: ".$ptr);
171: if ($ptr>0) $a=$att+1; // break out of the loop
172: }
173: }
174:
175: if ($ptr<=0)
176: {
177: $alertlevel=2;
178: db(" Ping Test: Failed");
179: $alerts[$alertc++]="ping failed";
180: $pal=2;
181: }
182: else
183: {
184: db(" Ping Test: Passed");
185: $pingpassed=true;
186: }
187:
188: // pingtest output bodge
189: // is there a test entry for ICMP
190: $fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
191: $fr=$NATS->DB->Query($fq);
192: $ltid_icmp="";
193: if ($irow=$NATS->DB->Fetch_Array($fr))
194: { // exists
195: $uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid'];
196: $ltid_icmp=$irow['localtestid'];
197: //echo $uq;
198: $NATS->DB->Query($uq);
199: }
200: else
201: { // doesn't exist
202: $uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")";
203: //echo $uq;
204: $NATS->DB->Query($uq);
205: $ltid_icmp=$NATS->DB->Insert_Id();
206: }
207: $NATS->DB->Free($fr);
208:
209: // record the ICMP bodge-test here
210: $rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")";
211: $NATS->DB->Query($rq);
212: //echo $rq." ".$NATS->DB->Affected_Rows()."\n";
213:
214: }
215: else
216: { // further ICMP bodge - update to -1 or do nothing if the test doesn't exist
217: $uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
218: $NATS->DB->Query($uq);
219: }
220:
221: if ($dotests&&($row['pingfatal'])&&($ptr<=0))
222: {
223: db(" Ping Fatal: Yes - Not Continuing");
224: $NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping");
225: $dotests=false;
226: }
227:
228: // do the tests - only actually exec if dotests true
229:
230: $first_test=true;
231:
232: db("Doing Local Tests");
233: $NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test");
234: $q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC";
235: $res=$NATS->DB->Query($q);
236: while ($lrow=$NATS->DB->Fetch_Array($res))
237: {
238: if ($lrow['nextrunx']<=time()) $testdue=true;
239: else $testdue=false;
240:
241: if ($first_test)
242: {
243: $first_test=false;
244: if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping
245: }
246: else test_sleep();
247:
248: if ($testdue)
249: {
250:
251: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']);
252: $NATS->EventHandler("localtest_start",$eventdata);
253:
254: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")");
255:
256: // Build parameter array
257: $params=array();
258: $params[0]=$lrow['testparam']; // pass standard param in as 0
259: for ($a=1; $a<10; $a++)
260: {
261: $parstr="testparam".$a;
262: $params[$a]=$lrow[$parstr];
263: }
264:
265: if ($dotests)
266: {
267: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
268: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
269: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
270: db(" Result: ".$result);
271: }
272: else $result=0;
273:
274: if ($dotests)
275: {
276: // evaluation
277: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
278: else $lvl=nats_eval("L".$lrow['localtestid'],$result);
279: db(" Eval: ".$lvl);
280:
281: // put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status
282: $att=$lrow['attempts'];
283: if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) )
284: {
285: for ($a=2; $a<=$att; $a++)
286: {
287: test_sleep();
288: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a);
289: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
290: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
291: db(" Result: ".$result);
292: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
293: else $lvl=nats_eval("L".$lrow['localtestid'],$result);
294: db(" Eval: ".$lvl);
295: if ($lvl==0) $a=$att+1; // test passed
296: }
297: }
298:
299: // $lvl is now the last lvl regardless of where it came from
300:
301: if ($lvl>$alertlevel) $alertlevel=$lvl;
302: if ($lvl>0)
303: {
304: if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." ";
305: else $s=$lrow['testname']." ";
306: /*
307: if ($lvl>1) $s.=$NATS->Cfg->Get("site.text.failed","failed");
308: else $s.=$NATS->Cfg->Get("site.text.warning","warning");
309: */
310: $s.=oText($lvl);
311: // site.alert.showvalue -- includes value in alert messages if numeric
312: // site.alert.showtext -- includes value in alert messages if textual
313: if (is_numeric($result))
314: {
315: if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")";
316: }
317: else // non-numeric
318: {
319: if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")";
320: }
321: $alerts[$alertc++]=$s;
322: }
323: } else $lvl=-1;
324:
325: // record it
326: if ($lrow['testrecord']==1)
327: {
328: $tid="L".$lrow['localtestid'];
329: $iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES(";
330: $iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")";
331: $NATS->DB->Query($iq);
332: db(" Recording Test");
333: }
334: if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net
335:
336: // update localtest record
337: $uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid'];
338: $NATS->DB->Query($uq);
339:
340: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl);
341: $NATS->EventHandler("localtest_finish",$eventdata);
342: }
343:
344: else // test not due so take pre-existing level for it
345: {
346: $lvl=$lrow['alertlevel'];
347: if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl;
348: }
349:
350:
351: }
352:
353: // Node-side testy magic
354: db("Nodeside Testing");
355: $freshdata=false;
356: if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull
357: {
358: $pullalert=$row['nspullalert']; // what happened the last time we tried
359:
360: if ($row['nsnextx']<=time()) // the time is right
361: {
362: db(" Pulling Data");
363: $pull_result=$NATS->Nodeside_Pull($row['nodeid']);
364:
365: if ($pull_result===false) // Pull Failed
366: {
367: db(" Pull Failed");
368: $pullalert=1; // alert
369: $alerts[$alertc++]="pull failed";
370: $alertlevel=2;
371: }
372: else // Pull Worked
373: {
374: $freshdata=true;
375: $pullalert=0; // ok
376: db(" Pull Succeeded");
377: }
378:
379:
380: db(" Updating Pull nslast/nextx and nspullalert");
381: $uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\"";
382: $NATS->DB->Query($uq);
383: if ($NATS->DB->Affected_Rows()<=0) db(" - Failed");
384: }
385: /*
386: // Process for alerts in here - whether pulled or not!
387: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
388: $tr=$NATS->DB->Query($tq);
389: while ($trow=$NATS->DB->Fetch_Array($tr))
390: {
391: if ($trow['testname']=="") $tname=$trow['testtype'];
392: else $tname=$trow['testname'];
393: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
394: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
395: }
396: */
397:
398: // and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one
399: // if pull not scheduled yet
400: if ($pullalert>0) $alertlevel=2; // so mark a failure
401:
402: }
403:
404: if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) || // pull and tests are on
405: (($row['nsenabled']==1)&&($row['nspushenabled']==1)) ) // or pushed
406: {
407: if ($row['nsfreshpush']==1)
408: {
409: $freshdata=true;
410: $uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\"";
411: $NATS->DB->Query($uq);
412: }
413: // Process for alerts in here - whether pulled or not!
414: $tq="SELECT testtype,testname,alertlevel,lastvalue FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
415: $tr=$NATS->DB->Query($tq);
416: while ($trow=$NATS->DB->Fetch_Array($tr))
417: {
418: if ($trow['testname']=="") $tname=$trow['testtype'];
419: else $tname=$trow['testname'];
420: if ($freshdata) // only record text to log if fresh
421: {
422: $s=$tname." ".oText($trow['alertlevel']);
423: $result=$trow['lastvalue'];
424: if (is_numeric($result))
425: {
426: if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")";
427: }
428: else // non-numeric
429: {
430: if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")";
431: }
432: $alerts[$alertc++]=$s;
433: }
434: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
435: }
436: }
437:
438: $NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node");
439:
440: $eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel );
441: $NATS->EventHandler("node_test_finish",$eventdata);
442:
443: db("Highest Alert Level: ".$alertlevel);
444: db("Alert Count : ".$alertc);
445: $als="";
446: foreach($alerts as $al) $als.=$al.", ";
447: db("Alerts: ".$als);
448:
449: $NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts);
450:
451: // This is where child/slave nodes spawn
452:
453: // $pingpassed bool holds if pingtest has passed
454: // $alertlevel holds the overall status
455: $chq="SELECT nodeid,masterjustping FROM fnnode WHERE masterid=\"".$row['nodeid']."\"";
456: //echo $chq;
457:
458: $chr=$NATS->DB->Query($chq);
459: $spawnlist=array();
460:
461: while ($child=$NATS->DB->Fetch_Array($chr))
462: {
463: if (($child['masterjustping']==1)&&($pingpassed)) $spawnlist[]=$child['nodeid'];
464: else if ($alertlevel==0) $spawnlist[]=$child['nodeid'];
465: // logic: if the child requires a ping and ping has passed ok then spawn it
466: // otherwise (pass on any alert) if everything has passed spawn it
467: }
468: $NATS->DB->Free($chr);
469:
470: if (count($spawnlist)>0)
471: {
472: $cmd="php ./test-threaded.php";
473: foreach($spawnlist as $child)
474: $cmd.=" ".$child;
475: $cmd.=" > /tmp/ns.master.".$row['nodeid']." &";
476: db("Children Spawning: ".$cmd);
477: exec($cmd);
478: }
479:
480:
481: // End of the node... carry forward highest level
482:
483: db(" ");
484:
485: if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel;
486: $talertc+=$alertc;
487:
488: }
489:
490:
491:
492: db("Finished Tests... Finishing Off");
493: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc);
494: if ($highalertlevel>-1)
495: {
496: $uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid;
497: $NATS->DB->Query($uq);
498: }
499: else
500: {
501: $uq="DELETE FROM fntestrun WHERE trid=".$trid;
502: $NATS->DB->Query($uq);
503: }
504:
505:
506: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat");
507: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop");
508:
509: // in here for now...
510: $NATS->ActionFlush();
511:
512: } // end of suspend check
513: else
514: {
515: // Suspended
516: db("NATS Testing Suspended");
517: }
518:
519: $NATS->Stop();
520: db("NATS Stopped... Finished");
521: ?>
522:
523: