File:
1.02.1a/server/bin/tester.php (
View as Code)
1: 2: /* -------------------------------------------------------------
3: This file is part of FreeNATS
4:
5: FreeNATS is (C) Copyright 2008 PurplePixie Systems
6:
7: FreeNATS is free software: you can redistribute it and/or modify
8: it under the terms of the GNU General Public License as published by
9: the Free Software Foundation, either version 3 of the License, or
10: (at your option) any later version.
11:
12: FreeNATS is distributed in the hope that it will be useful,
13: but WITHOUT ANY WARRANTY; without even the implied warranty of
14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15: GNU General Public License for more details.
16:
17: You should have received a copy of the GNU General Public License
18: along with FreeNATS. If not, see www.gnu.org/licenses
19:
20: For more information see www.purplepixie.org/freenats
21: -------------------------------------------------------------- */
22:
23: if ((isset($argc))&&(isset($argv)))
24: {
25: if ($argc>1)
26: {
27: $nfilter=$argv[1];
28: }
29: else $nfilter="";
30: }
31:
32: require("include.php");
33:
34: $dbt="";
35:
36: function db($txt,$nl=true)
37: {
38: global $dbt;
39: echo $txt;
40: $dbt.=$txt;
41: if ($nl)
42: {
43: echo "\n";
44: $dbt.="
\n";
45: }
46: }
47:
48: $NATS->Start();
49: if ($nfilter!="") $st=": Node ".$nfilter;
50: else $st="";
51: db("NATS Tester Script Starting".$st);
52:
53: $highalertlevel=-1;
54: $talertc=0;
55:
56: // check if already running
57: $cq="SELECT startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1";
58: $cr=$NATS->DB->Query($cq);
59: if ($NATS->DB->Num_Rows($cr)>0)
60: {
61: $NATS->Event("Tester Already Running: Aborted",1,"Tester","Error");
62: db("Tester Already Running: Aborted");
63: $NATS->Stop();
64: exit();
65: }
66: $NATS->DB->Free($cr);
67:
68: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")";
69: $NATS->DB->Query($gq);
70: $trid=$NATS->DB->Insert_Id();
71: db("Test ID: ".$trid." (Started at ".nicedt(time()).")");
72: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start");
73:
74: db(" ");
75:
76: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx)
77:
78: $q="SELECT * FROM fnnode WHERE nodeenabled=1";
79: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\"";
80: $q.=" AND nextrunx<=".time();
81:
82: $r=$NATS->DB->Query($q);
83:
84:
85: while ($row=$NATS->DB->Fetch_Array($r))
86: {
87: $dotests=true;
88: $alertlevel=0;
89: $alerts=array();
90: $alertc=0;
91: db("NodeID: ".$row['nodeid']);
92: $NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node");
93:
94: // Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested
95: if ($row['scheduleid']!=0) // has a schedule
96: {
97: db(" Has Schedule: Yes - Checking");
98: $run=run_x_in_schedule(time(),$row['scheduleid']);
99: if (!$run)
100: {
101: db(" In Schedule: No - Skipping Tests");
102: $NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node");
103: $dotests=false;
104: $alertlevel=-1;
105: }
106: else db(" In Schedule: Yes");
107: }
108:
109: $eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests );
110: $NATS->EventHandler("node_test_start",$eventdata);
111:
112:
113: $ptr=0;
114: $pal=0;
115:
116:
117: // Update lastrun and nextrun regardless of dotests
118: $q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\"";
119: $NATS->DB->Query($q);
120:
121:
122: if ($row['pingtest']&&$dotests)
123: {
124: db(" Ping Test: Yes");
125: $NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping");
126: $ptr=PingTest($row['hostname']);
127: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
128: db(" Ping Returned: ".$ptr);
129: if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) )
130: {
131: $att=$NATS->Cfg->Get("test.icmp.attempts","2");
132: for ($a=2; $a<=$att; $a++) // starting on second attempt
133: {
134: // try again...
135: test_sleep();
136: db(" Trying Ping Again - X".$a);
137: $NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping");
138: $ptr=PingTest($row['hostname']);
139: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
140: db(" Ping Returned: ".$ptr);
141: if ($ptr>0) $a=$att+1; // break out of the loop
142: }
143: }
144:
145: if ($ptr<=0)
146: {
147: $alertlevel=2;
148: db(" Ping Test: Failed");
149: $alerts[$alertc++]="ping failed";
150: $pal=2;
151: }
152: else db(" Ping Test: Passed");
153:
154: // pingtest output bodge
155: // is there a test entry for ICMP
156: $fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
157: $fr=$NATS->DB->Query($fq);
158: $ltid_icmp="";
159: if ($irow=$NATS->DB->Fetch_Array($fr))
160: { // exists
161: $uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid'];
162: $ltid_icmp=$irow['localtestid'];
163: //echo $uq;
164: $NATS->DB->Query($uq);
165: }
166: else
167: { // doesn't exist
168: $uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")";
169: //echo $uq;
170: $NATS->DB->Query($uq);
171: $ltid_icmp=$NATS->DB->Insert_Id();
172: }
173: $NATS->DB->Free($fr);
174:
175: // record the ICMP bodge-test here
176: $rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")";
177: $NATS->DB->Query($rq);
178: //echo $rq." ".$NATS->DB->Affected_Rows()."\n";
179:
180: }
181: else
182: { // further ICMP bodge - update to -1 or do nothing if the test doesn't exist
183: $uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
184: $NATS->DB->Query($uq);
185: }
186:
187: if ($dotests&&($row['pingfatal'])&&($ptr<=0))
188: {
189: db(" Ping Fatal: Yes - Not Continuing");
190: $NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping");
191: $dotests=false;
192: }
193:
194: // do the tests - only actually exec if dotests true
195:
196: $first_test=true;
197:
198: db("Doing Local Tests");
199: $NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test");
200: $q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC";
201: $res=$NATS->DB->Query($q);
202: while ($lrow=$NATS->DB->Fetch_Array($res))
203: {
204: if ($lrow['nextrunx']<=time()) $testdue=true;
205: else $testdue=false;
206:
207: if ($first_test)
208: {
209: $first_test=false;
210: if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping
211: }
212: else test_sleep();
213:
214: if ($testdue)
215: {
216:
217: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']);
218: $NATS->EventHandler("localtest_start",$eventdata);
219:
220: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")");
221:
222: // Build parameter array
223: $params=array();
224: $params[0]=$lrow['testparam']; // pass standard param in as 0
225: for ($a=1; $a<10; $a++)
226: {
227: $parstr="testparam".$a;
228: $params[$a]=$lrow[$parstr];
229: }
230:
231: if ($dotests)
232: {
233: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
234: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
235: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
236: db(" Result: ".$result);
237: }
238: else $result=0;
239:
240: if ($dotests)
241: {
242: // evaluation
243: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
244: else $lvl=nats_eval("L".$lrow['localtestid'],$result);
245: db(" Eval: ".$lvl);
246:
247: // put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status
248: $att=$lrow['attempts'];
249: if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) )
250: {
251: for ($a=2; $a<=$att; $a++)
252: {
253: test_sleep();
254: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a);
255: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
256: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
257: db(" Result: ".$result);
258: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
259: else $lvl=nats_eval("L".$lrow['localtestid'],$result);
260: db(" Eval: ".$lvl);
261: if ($lvl==0) $a=$att+1; // test passed
262: }
263: }
264:
265: // $lvl is now the last lvl regardless of where it came from
266:
267: if ($lvl>$alertlevel) $alertlevel=$lvl;
268: if ($lvl>0)
269: {
270: if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." ";
271: else $s=$lrow['testname']." ";
272: if ($lvl>1) $s.="failed";
273: else $s.="warning";
274: $alerts[$alertc++]=$s;
275: }
276: } else $lvl=-1;
277:
278: // record it
279: if ($lrow['testrecord']==1)
280: {
281: $tid="L".$lrow['localtestid'];
282: $iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES(";
283: $iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")";
284: $NATS->DB->Query($iq);
285: db(" Recording Test");
286: }
287: if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net
288:
289: // update localtest record
290: $uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid'];
291: $NATS->DB->Query($uq);
292:
293: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl);
294: $NATS->EventHandler("localtest_finish",$eventdata);
295: }
296:
297: else // test not due so take pre-existing level for it
298: {
299: $lvl=$lrow['alertlevel'];
300: if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl;
301: }
302:
303:
304: }
305:
306: // Node-side testy magic
307: db("Nodeside Testing");
308: $freshdata=false;
309: if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull
310: {
311: $pullalert=$row['nspullalert']; // what happened the last time we tried
312:
313: if ($row['nsnextx']<=time()) // the time is right
314: {
315: db(" Pulling Data");
316: $pull_result=$NATS->Nodeside_Pull($row['nodeid']);
317:
318: if ($pull_result===false) // Pull Failed
319: {
320: db(" Pull Failed");
321: $pullalert=1; // alert
322: $alerts[$alertc++]="pull failed";
323: $alertlevel=2;
324: }
325: else // Pull Worked
326: {
327: $freshdata=true;
328: $pullalert=0; // ok
329: db(" Pull Succeeded");
330: }
331:
332:
333: db(" Updating Pull nslast/nextx and nspullalert");
334: $uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\"";
335: $NATS->DB->Query($uq);
336: if ($NATS->DB->Affected_Rows()<=0) db(" - Failed");
337: }
338: /*
339: // Process for alerts in here - whether pulled or not!
340: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
341: $tr=$NATS->DB->Query($tq);
342: while ($trow=$NATS->DB->Fetch_Array($tr))
343: {
344: if ($trow['testname']=="") $tname=$trow['testtype'];
345: else $tname=$trow['testname'];
346: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
347: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
348: }
349: */
350:
351: // and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one
352: // if pull not scheduled yet
353: if ($pullalert>0) $alertlevel=2; // so mark a failure
354:
355: }
356:
357: if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) || // pull and tests are on
358: (($row['nsenabled']==1)&&($row['nspushenabled']==1)) ) // or pushed
359: {
360: if ($row['nsfreshpush']==1)
361: {
362: $freshdata=true;
363: $uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\"";
364: $NATS->DB->Query($uq);
365: }
366: // Process for alerts in here - whether pulled or not!
367: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
368: $tr=$NATS->DB->Query($tq);
369: while ($trow=$NATS->DB->Fetch_Array($tr))
370: {
371: if ($trow['testname']=="") $tname=$trow['testtype'];
372: else $tname=$trow['testname'];
373: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
374: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
375: }
376: }
377:
378: $NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node");
379:
380: $eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel );
381: $NATS->EventHandler("node_test_finish",$eventdata);
382:
383: db("Highest Alert Level: ".$alertlevel);
384: db("Alert Count : ".$alertc);
385: $als="";
386: foreach($alerts as $al) $als.=$al.", ";
387: db("Alerts: ".$als);
388:
389: $NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts);
390:
391: // This is where child/slave nodes would spawn
392:
393:
394: // End of the node... carry forward highest level
395:
396: db(" ");
397:
398: if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel;
399: $talertc+=$alertc;
400:
401: }
402:
403:
404:
405: db("Finished Tests... Finishing Off");
406: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc);
407: if ($highalertlevel>-1)
408: {
409: $uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid;
410: $NATS->DB->Query($uq);
411: }
412: else
413: {
414: $uq="DELETE FROM fntestrun WHERE trid=".$trid;
415: $NATS->DB->Query($uq);
416: }
417:
418:
419: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat");
420: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop");
421:
422: // in here for now...
423: $NATS->ActionFlush();
424:
425: $NATS->Stop();
426: db("NATS Stopped... Finished");
427: ?>
428:
429: