File: 1.00.11a/server/bin/tester.php (View as HTML)

  1: <?php
  2: /* -------------------------------------------------------------
  3: This file is part of FreeNATS
  4: 
  5: FreeNATS is (C) Copyright 2008 PurplePixie Systems
  6: 
  7: FreeNATS is free software: you can redistribute it and/or modify
  8: it under the terms of the GNU General Public License as published by
  9: the Free Software Foundation, either version 3 of the License, or
 10: (at your option) any later version.
 11: 
 12: FreeNATS is distributed in the hope that it will be useful,
 13: but WITHOUT ANY WARRANTY; without even the implied warranty of
 14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 15: GNU General Public License for more details.
 16: 
 17: You should have received a copy of the GNU General Public License
 18: along with FreeNATS.  If not, see www.gnu.org/licenses
 19: 
 20: For more information see www.purplepixie.org/freenats
 21: -------------------------------------------------------------- */
 22: 
 23: if ((isset($argc))&&(isset($argv)))
 24: 	{
 25: 	if ($argc>1)
 26: 		{
 27: 		$nfilter=$argv[1];
 28: 		}
 29: 	else $nfilter="";
 30: 	}
 31: 
 32: require("include.php");
 33: 
 34: $dbt="";
 35: 
 36: function db($txt,$nl=true)
 37: {
 38: global $dbt;
 39: echo $txt;
 40: $dbt.=$txt;
 41: if ($nl) 
 42: 	{
 43: 	echo "\n";
 44: 	$dbt.=" <br>\n";
 45: 	}
 46: }
 47: 
 48: $NATS->Start();
 49: if ($nfilter!="") $st=": Node ".$nfilter;
 50: else $st="";
 51: db("NATS Tester Script Starting".$st);
 52: 
 53: $highalertlevel=-1;
 54: $talertc=0;
 55: 
 56: // check if already running
 57: $cq="SELECT startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1";
 58: $cr=$NATS->DB->Query($cq);
 59: if ($NATS->DB->Num_Rows($cr)>0)
 60: 	{
 61: 	$NATS->Event("Tester Already Running: Aborted",1,"Tester","Error");
 62: 	db("Tester Already Running: Aborted");
 63: 	$NATS->Stop();
 64: 	exit();
 65: 	}
 66: $NATS->DB->Free($cr);
 67: 
 68: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")";
 69: $NATS->DB->Query($gq);
 70: $trid=$NATS->DB->Insert_Id();
 71: db("Test ID: ".$trid." (Started at ".nicedt(time()).")");
 72: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start");
 73: 
 74: db(" ");
 75: 
 76: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx)
 77: 
 78: $q="SELECT * FROM fnnode WHERE nodeenabled=1";
 79: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\"";
 80: $q.=" AND nextrunx<=".time();
 81: 
 82: $r=$NATS->DB->Query($q);
 83: 
 84: 
 85: while ($row=$NATS->DB->Fetch_Array($r))
 86: 	{
 87: 	$dotests=true;
 88: 	$alertlevel=0;
 89: 	$alerts=array();
 90: 	$alertc=0;
 91: 	db("NodeID: ".$row['nodeid']);
 92: 	$NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node");
 93: 
 94: 	// Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested
 95: 	if ($row['scheduleid']!=0) // has a schedule
 96: 		{
 97: 		db(" Has Schedule: Yes - Checking");
 98: 		$run=run_x_in_schedule(time(),$row['scheduleid']);
 99: 		if (!$run)
100: 			{
101: 			db(" In Schedule: No - Skipping Tests");
102: 			$NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node");
103: 			$dotests=false;
104: 			$alertlevel=-1;
105: 			}
106: 		else db(" In Schedule: Yes");
107: 		}
108: 	
109: 	$eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests );
110: 	$NATS->EventHandler("node_test_start",$eventdata);
111: 	
112: 	
113: 	$ptr=0;
114: 	$pal=0;
115: 	
116: 
117: 	// Update lastrun and nextrun regardless of dotests
118: 	$q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\"";
119: 	$NATS->DB->Query($q);
120: 
121: 	
122: 	if ($row['pingtest']&&$dotests)
123: 		{
124: 		db(" Ping Test: Yes");
125: 		$NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping");
126: 		$ptr=PingTest($row['hostname']);
127: 		$NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
128: 		db(" Ping Returned: ".$ptr);
129: 		if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) )
130: 			{
131: 			$att=$NATS->Cfg->Get("test.icmp.attempts","2");
132: 			for ($a=2; $a<=$att; $a++) // starting on second attempt
133: 				{
134: 				// try again...
135: 				test_sleep();
136: 				db(" Trying Ping Again - X".$a);
137: 				$NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping");
138: 				$ptr=PingTest($row['hostname']);
139: 				$NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
140: 				db(" Ping Returned: ".$ptr);
141: 				if ($ptr>0) $a=$att+1; // break out of the loop
142: 				}
143: 			}
144: 			
145: 		if ($ptr<=0) 
146: 			{
147: 			$alertlevel=2;
148: 			db(" Ping Test: Failed");
149: 			$alerts[$alertc++]="ping failed";
150: 			$pal=2;
151: 			}
152: 		else db(" Ping Test: Passed");
153: 		
154: 		// pingtest output bodge
155: 		// is there a test entry for ICMP
156: 		$fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
157: 		$fr=$NATS->DB->Query($fq);
158: 		$ltid_icmp="";
159: 		if ($irow=$NATS->DB->Fetch_Array($fr))
160: 			{ // exists
161: 			$uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid'];
162: 			$ltid_icmp=$irow['localtestid'];
163: 			//echo $uq;
164: 			$NATS->DB->Query($uq);
165: 			}
166: 		else
167: 			{ // doesn't exist
168: 			$uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")";
169: 			//echo $uq;
170: 			$NATS->DB->Query($uq);
171: 			$ltid_icmp=$NATS->DB->Insert_Id();
172: 			}
173: 		$NATS->DB->Free($fr);
174: 		
175: 		// record the ICMP bodge-test here
176: 		$rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")";
177: 		$NATS->DB->Query($rq);
178: 		//echo $rq." ".$NATS->DB->Affected_Rows()."\n";
179: 		
180: 		}
181: 	else
182: 		{ // further ICMP bodge - update to -1 or do nothing if the test doesn't exist
183: 		$uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
184: 		$NATS->DB->Query($uq);
185: 		}
186: 
187: 	if ($dotests&&($row['pingfatal'])&&($ptr<=0))
188: 		{
189: 		db(" Ping Fatal: Yes - Not Continuing");
190: 		$NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping");
191: 		$dotests=false;
192: 		}
193: 
194: 	 	// do the tests - only actually exec if dotests true
195: 
196: 	 	$first_test=true;
197: 	 	
198: 		db("Doing Local Tests");
199: 		$NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test");
200: 		$q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC";
201: 		$res=$NATS->DB->Query($q);
202: 		while ($lrow=$NATS->DB->Fetch_Array($res))
203: 			{
204: 			if ($lrow['nextrunx']<=time()) $testdue=true;
205: 			else $testdue=false;
206: 				
207: 			if ($first_test)
208: 				{
209: 				$first_test=false;
210: 				if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping
211: 				}
212: 			else test_sleep();
213: 			
214: 			if ($testdue)
215: 				{
216: 			
217: 				$eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']);
218: 				$NATS->EventHandler("localtest_start",$eventdata);
219: 					
220: 				db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")");
221: 				
222: 				// Build parameter array
223: 				$params=array();
224: 				$params[0]=$lrow['testparam']; // pass standard param in as 0
225: 				for ($a=1; $a<10; $a++)
226: 					{
227: 					$parstr="testparam".$a;
228: 					$params[$a]=$lrow[$parstr];
229: 					}
230: 				
231: 				$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
232: 				if ($testdue) $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
233: 				else $result=0;
234: 				$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
235: 				db(" Result: ".$result);
236: 				
237: 				if ($dotests)
238: 				{
239: 				// evaluation
240: 				if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
241: 				else $lvl=nats_eval("L".$lrow['localtestid'],$result);
242: 				db(" Eval: ".$lvl);
243: 				
244: 				// put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status
245: 				$att=$lrow['attempts'];
246: 				if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) )
247: 					{
248: 					for ($a=2; $a<=$att; $a++)
249: 						{
250: 						test_sleep();
251: 						db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a);
252: 						$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
253: 						$result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
254: 						db(" Result: ".$result);
255: 						if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
256: 						else $lvl=nats_eval("L".$lrow['localtestid'],$result);
257: 						db(" Eval: ".$lvl);
258: 						if ($lvl==0) $a=$att+1; // test passed
259: 						}
260: 					}
261: 				
262: 				// $lvl is now the last lvl regardless of where it came from
263: 						
264: 				if ($lvl>$alertlevel) $alertlevel=$lvl;
265: 				if ($lvl>0)
266: 					{
267: 					if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." ";
268: 					else $s=$lrow['testname']." ";
269: 					if ($lvl>1) $s.="failed";
270: 					else $s.="warning";
271: 					$alerts[$alertc++]=$s;
272: 					}
273: 				} else $lvl=-1;
274: 					
275: 				// record it
276: 				if ($lrow['testrecord']==1)
277: 					{
278: 					$tid="L".$lrow['localtestid'];
279: 					$iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES(";
280: 					$iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")";
281: 					$NATS->DB->Query($iq);
282: 					db(" Recording Test");
283: 					}
284: 				if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net
285: 					
286: 				// update localtest record
287: 				$uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid'];
288: 				$NATS->DB->Query($uq);
289: 				
290: 				$eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl);
291: 				$NATS->EventHandler("localtest_finish",$eventdata);
292: 				}
293: 				
294: 			else // test not due so take pre-existing level for it
295: 				{
296: 				$lvl=$lrow['alertlevel'];
297: 				if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl;
298: 				}
299: 			
300: 			
301: 			}
302: 			
303: 	// Node-side testy magic
304: 	db("Nodeside Testing");
305: 	$freshdata=false;
306: 	if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull
307: 		{
308: 		$pullalert=$row['nspullalert']; // what happened the last time we tried
309: 		
310: 		if ($row['nsnextx']<=time()) // the time is right
311: 			{
312: 			db(" Pulling Data");
313: 			$pull_result=$NATS->Nodeside_Pull($row['nodeid']);
314: 			
315: 			if ($pull_result===false) // Pull Failed
316: 				{
317: 				db(" Pull Failed");
318: 				$pullalert=1; // alert
319: 				$alerts[$alertc++]="pull failed";
320: 				$alertlevel=2;
321: 				}
322: 			else // Pull Worked
323: 				{
324: 				$freshdata=true;
325: 				$pullalert=0; // ok
326: 				db(" Pull Succeeded");
327: 				}
328: 				
329: 				
330: 			db(" Updating Pull nslast/nextx and nspullalert");
331: 			$uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\"";
332: 			$NATS->DB->Query($uq);
333: 			if ($NATS->DB->Affected_Rows()<=0) db(" - Failed");
334: 			}
335: 		/*
336: 		// Process for alerts in here - whether pulled or not!
337: 		$tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
338: 		$tr=$NATS->DB->Query($tq);
339: 		while ($trow=$NATS->DB->Fetch_Array($tr))
340: 			{
341: 			if ($trow['testname']=="") $tname=$trow['testtype'];
342: 			else $tname=$trow['testname'];
343: 			if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
344: 			if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
345: 			}
346: 		*/
347: 			
348: 		// and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one
349: 		// if pull not scheduled yet
350: 		if ($pullalert>0) $alertlevel=2; // so mark a failure
351: 
352: 		}
353: 		
354: 	if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) ||		// pull and tests are on
355: 		(($row['nsenabled']==1)&&($row['nspushenabled']==1)) )	// or pushed
356: 		{
357: 		if ($row['nsfreshpush']==1)
358: 			{
359: 			$freshdata=true;
360: 			$uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\"";
361: 			$NATS->DB->Query($uq);
362: 			}
363: 		// Process for alerts in here - whether pulled or not!
364: 		$tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
365: 		$tr=$NATS->DB->Query($tq);
366: 		while ($trow=$NATS->DB->Fetch_Array($tr))
367: 			{
368: 			if ($trow['testname']=="") $tname=$trow['testtype'];
369: 			else $tname=$trow['testname'];
370: 			if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
371: 			if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
372: 			}
373: 		}
374: 			
375: 	$NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node");
376: 
377: 	$eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel );
378: 	$NATS->EventHandler("node_test_finish",$eventdata);
379: 
380: 	db("Highest Alert Level: ".$alertlevel);
381: 	db("Alert Count        : ".$alertc);
382: 	$als="";
383: 	foreach($alerts as $al) $als.=$al.", ";
384: 	db("Alerts: ".$als);
385: 
386: 	$NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts);
387: 
388: 	db(" ");
389: 	
390: 	if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel;
391: 	$talertc+=$alertc;
392: 	
393: 	}
394: 
395: 
396: 
397: db("Finished Tests... Finishing Off");
398: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc);
399: if ($highalertlevel>-1)
400: 	{
401: 	$uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid;
402: 	$NATS->DB->Query($uq);
403: 	}
404: else
405: 	{
406: 	$uq="DELETE FROM fntestrun WHERE trid=".$trid;
407: 	$NATS->DB->Query($uq);
408: 	}
409: 
410: 
411: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat");
412: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop");
413: 
414: // in here for now...
415: $NATS->ActionFlush();
416: 
417: $NATS->Stop();
418: db("NATS Stopped... Finished");
419: ?>
420: 
421: