File: 1.05.1a/server/bin/tester.php (View as HTML)

  1: <?php
  2: /* -------------------------------------------------------------
  3: This file is part of FreeNATS
  4: 
  5: FreeNATS is (C) Copyright 2008 PurplePixie Systems
  6: 
  7: FreeNATS is free software: you can redistribute it and/or modify
  8: it under the terms of the GNU General Public License as published by
  9: the Free Software Foundation, either version 3 of the License, or
 10: (at your option) any later version.
 11: 
 12: FreeNATS is distributed in the hope that it will be useful,
 13: but WITHOUT ANY WARRANTY; without even the implied warranty of
 14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 15: GNU General Public License for more details.
 16: 
 17: You should have received a copy of the GNU General Public License
 18: along with FreeNATS.  If not, see www.gnu.org/licenses
 19: 
 20: For more information see www.purplepixie.org/freenats
 21: -------------------------------------------------------------- */
 22: 
 23: if ((isset($argc))&&(isset($argv))) // specific node or all nodes
 24: 	{
 25: 	if ($argc>1)
 26: 		{
 27: 		$nfilter=$argv[1];
 28: 		}
 29: 	else $nfilter="";
 30: 	}
 31: 
 32: require("include.php");
 33: 
 34: $dbt="";
 35: 
 36: function db($txt,$nl=true) // debug text
 37: {
 38: global $dbt;
 39: echo $txt;
 40: $dbt.=$txt;
 41: if ($nl) 
 42: 	{
 43: 	echo "\n";
 44: 	$dbt.=" <br>\n";
 45: 	}
 46: }
 47: 
 48: $NATS->Start();
 49: if ($nfilter!="") $st=": Node ".$nfilter;
 50: else $st="";
 51: db("NATS Tester Script Starting".$st);
 52: 
 53: $highalertlevel=-1;
 54: $talertc=0;
 55: 
 56: // check if already running
 57: $still_running=false;
 58: $cq="SELECT trid,startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1";
 59: $cr=$NATS->DB->Query($cq);
 60: 
 61: if ($runrow=$NATS->DB->Fetch_Array($cr))
 62: 	{ // yes there is a testrun session for this node(s)
 63: 	$timelimit=$NATS->Cfg->Get("test.session.limit",60*60);
 64: 	if ( (!is_numeric($timelimit)) || ($timelimit<0) ) $timelimit=60*60; // bogus config value
 65: 	
 66: 	// n.b. a timelimit of 0 means the session will never expire so...
 67: 	if ( ($timelimit>0) && ((time()-$runrow['startx'])>$timelimit) )
 68: 		{
 69: 		// valid time limit and the difference is more than the limit so close it
 70: 		$uq="UPDATE fntestrun SET finishx=1 WHERE trid=".$runrow['trid'];
 71: 		$NATS->DB->Query($uq);
 72: 		if ($NATS->DB->Affected_Rows()>0) 
 73: 			{
 74: 			$NATS->Event("Tester Already Running: Cleared",3,"Tester","Stale");
 75: 			db("Tester Already Running: Cleared");
 76: 			}
 77: 		else $NATS->Event("Tester Already Running: Failed to Clear",1,"Tester","Stale"); // weirdness... run anyway
 78: 		}
 79: 	else $still_running=true; // either never timesout or newer than timelimit ago
 80: 	}
 81: $NATS->DB->Free($cr);
 82: 
 83: // and if it is then don't continue
 84: if ($still_running)
 85: 	{
 86: 	$NATS->Event("Tester Already Running: Aborted",1,"Tester","Error");
 87: 	db("Tester Already Running: Aborted");
 88: 	$NATS->Stop();
 89: 	exit();
 90: 	}
 91: 
 92: 
 93: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")";
 94: $NATS->DB->Query($gq);
 95: $trid=$NATS->DB->Insert_Id();
 96: db("Test ID: ".$trid." (Started at ".nicedt(time()).")");
 97: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start");
 98: 
 99: db(" ");
100: 
101: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx)
102: 
103: $q="SELECT * FROM fnnode WHERE nodeenabled=1";
104: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\"";
105: $q.=" AND nextrunx<=".time();
106: 
107: $r=$NATS->DB->Query($q);
108: 
109: 
110: while ($row=$NATS->DB->Fetch_Array($r))
111: 	{
112: 	$dotests=true;
113: 	$alertlevel=0;
114: 	$alerts=array();
115: 	$alertc=0;
116: 	db("NodeID: ".$row['nodeid']);
117: 	$NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node");
118: 
119: 	// Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested
120: 	if ($row['scheduleid']!=0) // has a schedule
121: 		{
122: 		db(" Has Schedule: Yes - Checking");
123: 		$run=run_x_in_schedule(time(),$row['scheduleid']);
124: 		if (!$run)
125: 			{
126: 			db(" In Schedule: No - Skipping Tests");
127: 			$NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node");
128: 			$dotests=false;
129: 			$alertlevel=-1;
130: 			}
131: 		else db(" In Schedule: Yes");
132: 		}
133: 	
134: 	$eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests );
135: 	$NATS->EventHandler("node_test_start",$eventdata);
136: 	
137: 	
138: 	$ptr=0;
139: 	$pal=0;
140: 	
141: 
142: 	// Update lastrun and nextrun regardless of dotests
143: 	$q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\"";
144: 	$NATS->DB->Query($q);
145: 
146: 	$pingpassed=false; // this will only be set to true if a test is done and passes - for the "child" nodes
147: 	if ($row['pingtest']&&$dotests)
148: 		{
149: 		db(" Ping Test: Yes");
150: 		$NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping");
151: 		$ptr=PingTest($row['hostname']);
152: 		$NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
153: 		db(" Ping Returned: ".$ptr);
154: 		if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) )
155: 			{
156: 			$att=$NATS->Cfg->Get("test.icmp.attempts","2");
157: 			for ($a=2; $a<=$att; $a++) // starting on second attempt
158: 				{
159: 				// try again...
160: 				test_sleep();
161: 				db(" Trying Ping Again - X".$a);
162: 				$NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping");
163: 				$ptr=PingTest($row['hostname']);
164: 				$NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
165: 				db(" Ping Returned: ".$ptr);
166: 				if ($ptr>0) $a=$att+1; // break out of the loop
167: 				}
168: 			}
169: 			
170: 		if ($ptr<=0) 
171: 			{
172: 			$alertlevel=2;
173: 			db(" Ping Test: Failed");
174: 			$alerts[$alertc++]="ping failed";
175: 			$pal=2;
176: 			}
177: 		else 
178: 			{
179: 			db(" Ping Test: Passed");
180: 			$pingpassed=true;
181: 			}
182: 		
183: 		// pingtest output bodge
184: 		// is there a test entry for ICMP
185: 		$fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
186: 		$fr=$NATS->DB->Query($fq);
187: 		$ltid_icmp="";
188: 		if ($irow=$NATS->DB->Fetch_Array($fr))
189: 			{ // exists
190: 			$uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid'];
191: 			$ltid_icmp=$irow['localtestid'];
192: 			//echo $uq;
193: 			$NATS->DB->Query($uq);
194: 			}
195: 		else
196: 			{ // doesn't exist
197: 			$uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")";
198: 			//echo $uq;
199: 			$NATS->DB->Query($uq);
200: 			$ltid_icmp=$NATS->DB->Insert_Id();
201: 			}
202: 		$NATS->DB->Free($fr);
203: 		
204: 		// record the ICMP bodge-test here
205: 		$rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")";
206: 		$NATS->DB->Query($rq);
207: 		//echo $rq." ".$NATS->DB->Affected_Rows()."\n";
208: 		
209: 		}
210: 	else
211: 		{ // further ICMP bodge - update to -1 or do nothing if the test doesn't exist
212: 		$uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
213: 		$NATS->DB->Query($uq);
214: 		}
215: 
216: 	if ($dotests&&($row['pingfatal'])&&($ptr<=0))
217: 		{
218: 		db(" Ping Fatal: Yes - Not Continuing");
219: 		$NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping");
220: 		$dotests=false;
221: 		}
222: 
223: 	 	// do the tests - only actually exec if dotests true
224: 
225: 	 	$first_test=true;
226: 	 	
227: 		db("Doing Local Tests");
228: 		$NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test");
229: 		$q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC";
230: 		$res=$NATS->DB->Query($q);
231: 		while ($lrow=$NATS->DB->Fetch_Array($res))
232: 			{
233: 			if ($lrow['nextrunx']<=time()) $testdue=true;
234: 			else $testdue=false;
235: 				
236: 			if ($first_test)
237: 				{
238: 				$first_test=false;
239: 				if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping
240: 				}
241: 			else test_sleep();
242: 			
243: 			if ($testdue)
244: 				{
245: 			
246: 				$eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']);
247: 				$NATS->EventHandler("localtest_start",$eventdata);
248: 					
249: 				db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")");
250: 				
251: 				// Build parameter array
252: 				$params=array();
253: 				$params[0]=$lrow['testparam']; // pass standard param in as 0
254: 				for ($a=1; $a<10; $a++)
255: 					{
256: 					$parstr="testparam".$a;
257: 					$params[$a]=$lrow[$parstr];
258: 					}
259: 				
260: 				if ($dotests)
261: 					{
262: 					$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
263: 					$result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
264: 					$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
265: 					db(" Result: ".$result);
266: 					}
267: 				else $result=0;
268: 				
269: 				if ($dotests)
270: 				{
271: 				// evaluation
272: 				if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
273: 				else $lvl=nats_eval("L".$lrow['localtestid'],$result);
274: 				db(" Eval: ".$lvl);
275: 				
276: 				// put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status
277: 				$att=$lrow['attempts'];
278: 				if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) )
279: 					{
280: 					for ($a=2; $a<=$att; $a++)
281: 						{
282: 						test_sleep();
283: 						db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a);
284: 						$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
285: 						$result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
286: 						db(" Result: ".$result);
287: 						if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
288: 						else $lvl=nats_eval("L".$lrow['localtestid'],$result);
289: 						db(" Eval: ".$lvl);
290: 						if ($lvl==0) $a=$att+1; // test passed
291: 						}
292: 					}
293: 				
294: 				// $lvl is now the last lvl regardless of where it came from
295: 						
296: 				if ($lvl>$alertlevel) $alertlevel=$lvl;
297: 				if ($lvl>0)
298: 					{
299: 					if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." ";
300: 					else $s=$lrow['testname']." ";
301: 					/*
302: 					if ($lvl>1) $s.=$NATS->Cfg->Get("site.text.failed","failed");
303: 					else $s.=$NATS->Cfg->Get("site.text.warning","warning");
304: 					*/
305: 					$s.=oText($lvl);
306: 					// site.alert.showvalue -- includes value in alert messages if numeric
307: 					// site.alert.showtext -- includes value in alert messages if textual
308: 					if (is_numeric($result))
309: 						{
310: 						if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")";
311: 						}
312: 					else // non-numeric
313: 						{
314: 						if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")";
315: 						}
316: 					$alerts[$alertc++]=$s;
317: 					}
318: 				} else $lvl=-1;
319: 					
320: 				// record it
321: 				if ($lrow['testrecord']==1)
322: 					{
323: 					$tid="L".$lrow['localtestid'];
324: 					$iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES(";
325: 					$iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")";
326: 					$NATS->DB->Query($iq);
327: 					db(" Recording Test");
328: 					}
329: 				if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net
330: 					
331: 				// update localtest record
332: 				$uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid'];
333: 				$NATS->DB->Query($uq);
334: 				
335: 				$eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl);
336: 				$NATS->EventHandler("localtest_finish",$eventdata);
337: 				}
338: 				
339: 			else // test not due so take pre-existing level for it
340: 				{
341: 				$lvl=$lrow['alertlevel'];
342: 				if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl;
343: 				}
344: 			
345: 			
346: 			}
347: 			
348: 	// Node-side testy magic
349: 	db("Nodeside Testing");
350: 	$freshdata=false;
351: 	if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull
352: 		{
353: 		$pullalert=$row['nspullalert']; // what happened the last time we tried
354: 		
355: 		if ($row['nsnextx']<=time()) // the time is right
356: 			{
357: 			db(" Pulling Data");
358: 			$pull_result=$NATS->Nodeside_Pull($row['nodeid']);
359: 			
360: 			if ($pull_result===false) // Pull Failed
361: 				{
362: 				db(" Pull Failed");
363: 				$pullalert=1; // alert
364: 				$alerts[$alertc++]="pull failed";
365: 				$alertlevel=2;
366: 				}
367: 			else // Pull Worked
368: 				{
369: 				$freshdata=true;
370: 				$pullalert=0; // ok
371: 				db(" Pull Succeeded");
372: 				}
373: 				
374: 				
375: 			db(" Updating Pull nslast/nextx and nspullalert");
376: 			$uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\"";
377: 			$NATS->DB->Query($uq);
378: 			if ($NATS->DB->Affected_Rows()<=0) db(" - Failed");
379: 			}
380: 		/*
381: 		// Process for alerts in here - whether pulled or not!
382: 		$tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
383: 		$tr=$NATS->DB->Query($tq);
384: 		while ($trow=$NATS->DB->Fetch_Array($tr))
385: 			{
386: 			if ($trow['testname']=="") $tname=$trow['testtype'];
387: 			else $tname=$trow['testname'];
388: 			if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
389: 			if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
390: 			}
391: 		*/
392: 			
393: 		// and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one
394: 		// if pull not scheduled yet
395: 		if ($pullalert>0) $alertlevel=2; // so mark a failure
396: 
397: 		}
398: 		
399: 	if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) ||		// pull and tests are on
400: 		(($row['nsenabled']==1)&&($row['nspushenabled']==1)) )	// or pushed
401: 		{
402: 		if ($row['nsfreshpush']==1)
403: 			{
404: 			$freshdata=true;
405: 			$uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\"";
406: 			$NATS->DB->Query($uq);
407: 			}
408: 		// Process for alerts in here - whether pulled or not!
409: 		$tq="SELECT testtype,testname,alertlevel,lastvalue FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
410: 		$tr=$NATS->DB->Query($tq);
411: 		while ($trow=$NATS->DB->Fetch_Array($tr))
412: 			{
413: 			if ($trow['testname']=="") $tname=$trow['testtype'];
414: 			else $tname=$trow['testname'];
415: 			if ($freshdata) // only record text to log if fresh
416: 				{
417: 				$s=$tname." ".oText($trow['alertlevel']); 
418: 				$result=$trow['lastvalue'];
419: 				if (is_numeric($result))
420: 						{
421: 						if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")";
422: 						}
423: 					else // non-numeric
424: 						{
425: 						if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")";
426: 						}
427: 				$alerts[$alertc++]=$s;
428: 				}
429: 			if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
430: 			}
431: 		}
432: 			
433: 	$NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node");
434: 
435: 	$eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel );
436: 	$NATS->EventHandler("node_test_finish",$eventdata);
437: 
438: 	db("Highest Alert Level: ".$alertlevel);
439: 	db("Alert Count        : ".$alertc);
440: 	$als="";
441: 	foreach($alerts as $al) $als.=$al.", ";
442: 	db("Alerts: ".$als);
443: 
444: 	$NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts);
445: 	
446: 	// This is where child/slave nodes spawn
447: 	
448: 	// $pingpassed bool holds if pingtest has passed
449: 	// $alertlevel holds the overall status
450: 	$chq="SELECT nodeid,masterjustping FROM fnnode WHERE masterid=\"".$row['nodeid']."\"";
451: 	//echo $chq;
452: 	
453: 	$chr=$NATS->DB->Query($chq);
454: 	$spawnlist=array();
455: 	
456: 	while ($child=$NATS->DB->Fetch_Array($chr))
457: 		{
458: 		if (($child['masterjustping']==1)&&($pingpassed)) $spawnlist[]=$child['nodeid'];
459: 		else if ($alertlevel==0) $spawnlist[]=$child['nodeid'];
460: 		// logic: if the child requires a ping and ping has passed ok then spawn it
461: 		// otherwise (pass on any alert) if everything has passed spawn it
462: 		}
463: 	$NATS->DB->Free($chr);
464: 	
465: 	if (count($spawnlist)>0)
466: 		{
467: 		$cmd="php ./test-threaded.php";
468: 		foreach($spawnlist as $child)
469: 			$cmd.=" ".$child;
470: 		$cmd.=" > /tmp/ns.master.".$row['nodeid']." &";
471: 		db("Children Spawning: ".$cmd);
472: 		exec($cmd);
473: 		}
474: 	
475: 	
476: 	// End of the node... carry forward highest level
477: 
478: 	db(" ");
479: 	
480: 	if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel;
481: 	$talertc+=$alertc;
482: 	
483: 	}
484: 
485: 
486: 
487: db("Finished Tests... Finishing Off");
488: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc);
489: if ($highalertlevel>-1)
490: 	{
491: 	$uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid;
492: 	$NATS->DB->Query($uq);
493: 	}
494: else
495: 	{
496: 	$uq="DELETE FROM fntestrun WHERE trid=".$trid;
497: 	$NATS->DB->Query($uq);
498: 	}
499: 
500: 
501: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat");
502: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop");
503: 
504: // in here for now...
505: $NATS->ActionFlush();
506: 
507: $NATS->Stop();
508: db("NATS Stopped... Finished");
509: ?>
510: 
511: