File:
1.04.5a/server/bin/tester.php (
View as Code)
1: 2: /* -------------------------------------------------------------
3: This file is part of FreeNATS
4:
5: FreeNATS is (C) Copyright 2008 PurplePixie Systems
6:
7: FreeNATS is free software: you can redistribute it and/or modify
8: it under the terms of the GNU General Public License as published by
9: the Free Software Foundation, either version 3 of the License, or
10: (at your option) any later version.
11:
12: FreeNATS is distributed in the hope that it will be useful,
13: but WITHOUT ANY WARRANTY; without even the implied warranty of
14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15: GNU General Public License for more details.
16:
17: You should have received a copy of the GNU General Public License
18: along with FreeNATS. If not, see www.gnu.org/licenses
19:
20: For more information see www.purplepixie.org/freenats
21: -------------------------------------------------------------- */
22:
23: if ((isset($argc))&&(isset($argv))) // specific node or all nodes
24: {
25: if ($argc>1)
26: {
27: $nfilter=$argv[1];
28: }
29: else $nfilter="";
30: }
31:
32: require("include.php");
33:
34: $dbt="";
35:
36: function db($txt,$nl=true) // debug text
37: {
38: global $dbt;
39: echo $txt;
40: $dbt.=$txt;
41: if ($nl)
42: {
43: echo "\n";
44: $dbt.="
\n";
45: }
46: }
47:
48: $NATS->Start();
49: if ($nfilter!="") $st=": Node ".$nfilter;
50: else $st="";
51: db("NATS Tester Script Starting".$st);
52:
53: $highalertlevel=-1;
54: $talertc=0;
55:
56: // check if already running
57: $still_running=false;
58: $cq="SELECT trid,startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1";
59: $cr=$NATS->DB->Query($cq);
60:
61: if ($runrow=$NATS->DB->Fetch_Array($cr))
62: { // yes there is a testrun session for this node(s)
63: $timelimit=$NATS->Cfg->Get("test.session.limit",60*60);
64: if ( (!is_numeric($timelimit)) || ($timelimit<0) ) $timelimit=60*60; // bogus config value
65:
66: // n.b. a timelimit of 0 means the session will never expire so...
67: if ( ($timelimit>0) && ((time()-$runrow['startx'])>$timelimit) )
68: {
69: // valid time limit and the difference is more than the limit so close it
70: $uq="UPDATE fntestrun SET finishx=1 WHERE trid=".$runrow['trid'];
71: $NATS->DB->Query($uq);
72: if ($NATS->DB->Affected_Rows()>0)
73: {
74: $NATS->Event("Tester Already Running: Cleared",3,"Tester","Stale");
75: db("Tester Already Running: Cleared");
76: }
77: else $NATS->Event("Tester Already Running: Failed to Clear",1,"Tester","Stale"); // weirdness... run anyway
78: }
79: else $still_running=true; // either never timesout or newer than timelimit ago
80: }
81: $NATS->DB->Free($cr);
82:
83: // and if it is then don't continue
84: if ($still_running)
85: {
86: $NATS->Event("Tester Already Running: Aborted",1,"Tester","Error");
87: db("Tester Already Running: Aborted");
88: $NATS->Stop();
89: exit();
90: }
91:
92:
93: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")";
94: $NATS->DB->Query($gq);
95: $trid=$NATS->DB->Insert_Id();
96: db("Test ID: ".$trid." (Started at ".nicedt(time()).")");
97: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start");
98:
99: db(" ");
100:
101: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx)
102:
103: $q="SELECT * FROM fnnode WHERE nodeenabled=1";
104: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\"";
105: $q.=" AND nextrunx<=".time();
106:
107: $r=$NATS->DB->Query($q);
108:
109:
110: while ($row=$NATS->DB->Fetch_Array($r))
111: {
112: $dotests=true;
113: $alertlevel=0;
114: $alerts=array();
115: $alertc=0;
116: db("NodeID: ".$row['nodeid']);
117: $NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node");
118:
119: // Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested
120: if ($row['scheduleid']!=0) // has a schedule
121: {
122: db(" Has Schedule: Yes - Checking");
123: $run=run_x_in_schedule(time(),$row['scheduleid']);
124: if (!$run)
125: {
126: db(" In Schedule: No - Skipping Tests");
127: $NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node");
128: $dotests=false;
129: $alertlevel=-1;
130: }
131: else db(" In Schedule: Yes");
132: }
133:
134: $eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests );
135: $NATS->EventHandler("node_test_start",$eventdata);
136:
137:
138: $ptr=0;
139: $pal=0;
140:
141:
142: // Update lastrun and nextrun regardless of dotests
143: $q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\"";
144: $NATS->DB->Query($q);
145:
146: $pingpassed=false; // this will only be set to true if a test is done and passes - for the "child" nodes
147: if ($row['pingtest']&&$dotests)
148: {
149: db(" Ping Test: Yes");
150: $NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping");
151: $ptr=PingTest($row['hostname']);
152: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
153: db(" Ping Returned: ".$ptr);
154: if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) )
155: {
156: $att=$NATS->Cfg->Get("test.icmp.attempts","2");
157: for ($a=2; $a<=$att; $a++) // starting on second attempt
158: {
159: // try again...
160: test_sleep();
161: db(" Trying Ping Again - X".$a);
162: $NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping");
163: $ptr=PingTest($row['hostname']);
164: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
165: db(" Ping Returned: ".$ptr);
166: if ($ptr>0) $a=$att+1; // break out of the loop
167: }
168: }
169:
170: if ($ptr<=0)
171: {
172: $alertlevel=2;
173: db(" Ping Test: Failed");
174: $alerts[$alertc++]="ping failed";
175: $pal=2;
176: }
177: else
178: {
179: db(" Ping Test: Passed");
180: $pingpassed=true;
181: }
182:
183: // pingtest output bodge
184: // is there a test entry for ICMP
185: $fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
186: $fr=$NATS->DB->Query($fq);
187: $ltid_icmp="";
188: if ($irow=$NATS->DB->Fetch_Array($fr))
189: { // exists
190: $uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid'];
191: $ltid_icmp=$irow['localtestid'];
192: //echo $uq;
193: $NATS->DB->Query($uq);
194: }
195: else
196: { // doesn't exist
197: $uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")";
198: //echo $uq;
199: $NATS->DB->Query($uq);
200: $ltid_icmp=$NATS->DB->Insert_Id();
201: }
202: $NATS->DB->Free($fr);
203:
204: // record the ICMP bodge-test here
205: $rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")";
206: $NATS->DB->Query($rq);
207: //echo $rq." ".$NATS->DB->Affected_Rows()."\n";
208:
209: }
210: else
211: { // further ICMP bodge - update to -1 or do nothing if the test doesn't exist
212: $uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
213: $NATS->DB->Query($uq);
214: }
215:
216: if ($dotests&&($row['pingfatal'])&&($ptr<=0))
217: {
218: db(" Ping Fatal: Yes - Not Continuing");
219: $NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping");
220: $dotests=false;
221: }
222:
223: // do the tests - only actually exec if dotests true
224:
225: $first_test=true;
226:
227: db("Doing Local Tests");
228: $NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test");
229: $q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC";
230: $res=$NATS->DB->Query($q);
231: while ($lrow=$NATS->DB->Fetch_Array($res))
232: {
233: if ($lrow['nextrunx']<=time()) $testdue=true;
234: else $testdue=false;
235:
236: if ($first_test)
237: {
238: $first_test=false;
239: if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping
240: }
241: else test_sleep();
242:
243: if ($testdue)
244: {
245:
246: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']);
247: $NATS->EventHandler("localtest_start",$eventdata);
248:
249: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")");
250:
251: // Build parameter array
252: $params=array();
253: $params[0]=$lrow['testparam']; // pass standard param in as 0
254: for ($a=1; $a<10; $a++)
255: {
256: $parstr="testparam".$a;
257: $params[$a]=$lrow[$parstr];
258: }
259:
260: if ($dotests)
261: {
262: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
263: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
264: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
265: db(" Result: ".$result);
266: }
267: else $result=0;
268:
269: if ($dotests)
270: {
271: // evaluation
272: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
273: else $lvl=nats_eval("L".$lrow['localtestid'],$result);
274: db(" Eval: ".$lvl);
275:
276: // put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status
277: $att=$lrow['attempts'];
278: if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) )
279: {
280: for ($a=2; $a<=$att; $a++)
281: {
282: test_sleep();
283: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a);
284: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
285: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
286: db(" Result: ".$result);
287: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
288: else $lvl=nats_eval("L".$lrow['localtestid'],$result);
289: db(" Eval: ".$lvl);
290: if ($lvl==0) $a=$att+1; // test passed
291: }
292: }
293:
294: // $lvl is now the last lvl regardless of where it came from
295:
296: if ($lvl>$alertlevel) $alertlevel=$lvl;
297: if ($lvl>0)
298: {
299: if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." ";
300: else $s=$lrow['testname']." ";
301: if ($lvl>1) $s.="failed";
302: else $s.="warning";
303: $alerts[$alertc++]=$s;
304: }
305: } else $lvl=-1;
306:
307: // record it
308: if ($lrow['testrecord']==1)
309: {
310: $tid="L".$lrow['localtestid'];
311: $iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES(";
312: $iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")";
313: $NATS->DB->Query($iq);
314: db(" Recording Test");
315: }
316: if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net
317:
318: // update localtest record
319: $uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid'];
320: $NATS->DB->Query($uq);
321:
322: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl);
323: $NATS->EventHandler("localtest_finish",$eventdata);
324: }
325:
326: else // test not due so take pre-existing level for it
327: {
328: $lvl=$lrow['alertlevel'];
329: if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl;
330: }
331:
332:
333: }
334:
335: // Node-side testy magic
336: db("Nodeside Testing");
337: $freshdata=false;
338: if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull
339: {
340: $pullalert=$row['nspullalert']; // what happened the last time we tried
341:
342: if ($row['nsnextx']<=time()) // the time is right
343: {
344: db(" Pulling Data");
345: $pull_result=$NATS->Nodeside_Pull($row['nodeid']);
346:
347: if ($pull_result===false) // Pull Failed
348: {
349: db(" Pull Failed");
350: $pullalert=1; // alert
351: $alerts[$alertc++]="pull failed";
352: $alertlevel=2;
353: }
354: else // Pull Worked
355: {
356: $freshdata=true;
357: $pullalert=0; // ok
358: db(" Pull Succeeded");
359: }
360:
361:
362: db(" Updating Pull nslast/nextx and nspullalert");
363: $uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\"";
364: $NATS->DB->Query($uq);
365: if ($NATS->DB->Affected_Rows()<=0) db(" - Failed");
366: }
367: /*
368: // Process for alerts in here - whether pulled or not!
369: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
370: $tr=$NATS->DB->Query($tq);
371: while ($trow=$NATS->DB->Fetch_Array($tr))
372: {
373: if ($trow['testname']=="") $tname=$trow['testtype'];
374: else $tname=$trow['testname'];
375: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
376: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
377: }
378: */
379:
380: // and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one
381: // if pull not scheduled yet
382: if ($pullalert>0) $alertlevel=2; // so mark a failure
383:
384: }
385:
386: if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) || // pull and tests are on
387: (($row['nsenabled']==1)&&($row['nspushenabled']==1)) ) // or pushed
388: {
389: if ($row['nsfreshpush']==1)
390: {
391: $freshdata=true;
392: $uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\"";
393: $NATS->DB->Query($uq);
394: }
395: // Process for alerts in here - whether pulled or not!
396: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
397: $tr=$NATS->DB->Query($tq);
398: while ($trow=$NATS->DB->Fetch_Array($tr))
399: {
400: if ($trow['testname']=="") $tname=$trow['testtype'];
401: else $tname=$trow['testname'];
402: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
403: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
404: }
405: }
406:
407: $NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node");
408:
409: $eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel );
410: $NATS->EventHandler("node_test_finish",$eventdata);
411:
412: db("Highest Alert Level: ".$alertlevel);
413: db("Alert Count : ".$alertc);
414: $als="";
415: foreach($alerts as $al) $als.=$al.", ";
416: db("Alerts: ".$als);
417:
418: $NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts);
419:
420: // This is where child/slave nodes spawn
421:
422: // $pingpassed bool holds if pingtest has passed
423: // $alertlevel holds the overall status
424: $chq="SELECT nodeid,masterjustping FROM fnnode WHERE masterid=\"".$row['nodeid']."\"";
425: //echo $chq;
426:
427: $chr=$NATS->DB->Query($chq);
428: $spawnlist=array();
429:
430: while ($child=$NATS->DB->Fetch_Array($chr))
431: {
432: if (($child['masterjustping']==1)&&($pingpassed)) $spawnlist[]=$child['nodeid'];
433: else if ($alertlevel==0) $spawnlist[]=$child['nodeid'];
434: // logic: if the child requires a ping and ping has passed ok then spawn it
435: // otherwise (pass on any alert) if everything has passed spawn it
436: }
437: $NATS->DB->Free($chr);
438:
439: if (count($spawnlist)>0)
440: {
441: $cmd="php ./test-threaded.php";
442: foreach($spawnlist as $child)
443: $cmd.=" ".$child;
444: $cmd.=" > /tmp/ns.master.".$row['nodeid']." &";
445: db("Children Spawning: ".$cmd);
446: exec($cmd);
447: }
448:
449:
450: // End of the node... carry forward highest level
451:
452: db(" ");
453:
454: if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel;
455: $talertc+=$alertc;
456:
457: }
458:
459:
460:
461: db("Finished Tests... Finishing Off");
462: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc);
463: if ($highalertlevel>-1)
464: {
465: $uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid;
466: $NATS->DB->Query($uq);
467: }
468: else
469: {
470: $uq="DELETE FROM fntestrun WHERE trid=".$trid;
471: $NATS->DB->Query($uq);
472: }
473:
474:
475: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat");
476: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop");
477:
478: // in here for now...
479: $NATS->ActionFlush();
480:
481: $NATS->Stop();
482: db("NATS Stopped... Finished");
483: ?>
484:
485: