File: 1.10.1a/server/bin/tester.php (View as Code)

1: 2: /* ------------------------------------------------------------- 3: This file is part of FreeNATS 4: 5: FreeNATS is (C) Copyright 2008 PurplePixie Systems 6: 7: FreeNATS is free software: you can redistribute it and/or modify 8: it under the terms of the GNU General Public License as published by 9: the Free Software Foundation, either version 3 of the License, or 10: (at your option) any later version. 11: 12: FreeNATS is distributed in the hope that it will be useful, 13: but WITHOUT ANY WARRANTY; without even the implied warranty of 14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15: GNU General Public License for more details. 16: 17: You should have received a copy of the GNU General Public License 18: along with FreeNATS. If not, see www.gnu.org/licenses 19: 20: For more information see www.purplepixie.org/freenats 21: -------------------------------------------------------------- */ 22: 23: if ((isset($argc))&&(isset($argv))) // specific node or all nodes 24: { 25: if ($argc>1) 26: { 27: $nfilter=$argv[1]; 28: } 29: else $nfilter=""; 30: } 31: 32: require("include.php"); 33: 34: $dbt=""; 35: 36: function db($txt,$nl=true) // debug text 37: { 38: global $dbt; 39: echo $txt; 40: $dbt.=$txt; 41: if ($nl) 42: { 43: echo "\n"; 44: $dbt.="
\n";
45: } 46: } 47: 48: $NATS->Start(); 49: if ($nfilter!="") $st=": Node ".$nfilter; 50: else $st=""; 51: db("NATS Tester Script Starting".$st); 52: 53: $highalertlevel=-1; 54: $talertc=0; 55: 56: // check if already running 57: $still_running=false; 58: $cq="SELECT trid,startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1"; 59: $cr=$NATS->DB->Query($cq); 60: 61: if ($runrow=$NATS->DB->Fetch_Array($cr)) 62: { // yes there is a testrun session for this node(s) 63: $timelimit=$NATS->Cfg->Get("test.session.limit",60*60); 64: if ( (!is_numeric($timelimit)) || ($timelimit<0) ) $timelimit=60*60; // bogus config value 65: 66: // n.b. a timelimit of 0 means the session will never expire so... 67: if ( ($timelimit>0) && ((time()-$runrow['startx'])>$timelimit) ) 68: { 69: // valid time limit and the difference is more than the limit so close it 70: $uq="UPDATE fntestrun SET finishx=1 WHERE trid=".$runrow['trid']; 71: $NATS->DB->Query($uq); 72: if ($NATS->DB->Affected_Rows()>0) 73: { 74: $NATS->Event("Tester Already Running: Cleared",3,"Tester","Stale"); 75: db("Tester Already Running: Cleared"); 76: } 77: else $NATS->Event("Tester Already Running: Failed to Clear",1,"Tester","Stale"); // weirdness... run anyway 78: } 79: else $still_running=true; // either never timesout or newer than timelimit ago 80: } 81: $NATS->DB->Free($cr); 82: 83: // and if it is then don't continue 84: if ($still_running) 85: { 86: $NATS->Event("Tester Already Running: Aborted",1,"Tester","Error"); 87: db("Tester Already Running: Aborted"); 88: $NATS->Stop(); 89: exit(); 90: } 91: 92: 93: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")"; 94: $NATS->DB->Query($gq); 95: $trid=$NATS->DB->Insert_Id(); 96: db("Test ID: ".$trid." (Started at ".nicedt(time()).")"); 97: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start"); 98: 99: db(" "); 100: 101: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx) 102: 103: $q="SELECT * FROM fnnode WHERE nodeenabled=1"; 104: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\""; 105: $q.=" AND nextrunx<=".time(); 106: 107: $r=$NATS->DB->Query($q); 108: 109: 110: while ($row=$NATS->DB->Fetch_Array($r)) 111: { 112: $dotests=true; 113: $alertlevel=0; 114: $alerts=array(); 115: $alertc=0; 116: db("NodeID: ".$row['nodeid']); 117: $NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node"); 118: 119: // Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested 120: if ($row['scheduleid']!=0) // has a schedule 121: { 122: db(" Has Schedule: Yes - Checking"); 123: $run=run_x_in_schedule(time(),$row['scheduleid']); 124: if (!$run) 125: { 126: db(" In Schedule: No - Skipping Tests"); 127: $NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node"); 128: $dotests=false; 129: $alertlevel=-1; 130: } 131: else db(" In Schedule: Yes"); 132: } 133: 134: $eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests ); 135: $NATS->EventHandler("node_test_start",$eventdata); 136: 137: 138: $ptr=0; 139: $pal=0; 140: 141: 142: // Update lastrun and nextrun regardless of dotests 143: $q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\""; 144: $NATS->DB->Query($q); 145: 146: $pingpassed=false; // this will only be set to true if a test is done and passes - for the "child" nodes 147: if ($row['pingtest']&&$dotests) 148: { 149: db(" Ping Test: Yes"); 150: $NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping"); 151: $ptr=PingTest($row['hostname']); 152: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping"); 153: db(" Ping Returned: ".$ptr); 154: if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) ) 155: { 156: $att=$NATS->Cfg->Get("test.icmp.attempts","2"); 157: for ($a=2; $a<=$att; $a++) // starting on second attempt 158: { 159: // try again... 160: test_sleep(); 161: db(" Trying Ping Again - X".$a); 162: $NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping"); 163: $ptr=PingTest($row['hostname']); 164: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping"); 165: db(" Ping Returned: ".$ptr); 166: if ($ptr>0) $a=$att+1; // break out of the loop 167: } 168: } 169: 170: if ($ptr<=0) 171: { 172: $alertlevel=2; 173: db(" Ping Test: Failed"); 174: $alerts[$alertc++]="ping failed"; 175: $pal=2; 176: } 177: else 178: { 179: db(" Ping Test: Passed"); 180: $pingpassed=true; 181: } 182: 183: // pingtest output bodge 184: // is there a test entry for ICMP 185: $fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\""; 186: $fr=$NATS->DB->Query($fq); 187: $ltid_icmp=""; 188: if ($irow=$NATS->DB->Fetch_Array($fr)) 189: { // exists 190: $uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid']; 191: $ltid_icmp=$irow['localtestid']; 192: //echo $uq; 193: $NATS->DB->Query($uq); 194: } 195: else 196: { // doesn't exist 197: $uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")"; 198: //echo $uq; 199: $NATS->DB->Query($uq); 200: $ltid_icmp=$NATS->DB->Insert_Id(); 201: } 202: $NATS->DB->Free($fr); 203: 204: // record the ICMP bodge-test here 205: $rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")"; 206: $NATS->DB->Query($rq); 207: //echo $rq." ".$NATS->DB->Affected_Rows()."\n"; 208: 209: } 210: else 211: { // further ICMP bodge - update to -1 or do nothing if the test doesn't exist 212: $uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\""; 213: $NATS->DB->Query($uq); 214: } 215: 216: if ($dotests&&($row['pingfatal'])&&($ptr<=0)) 217: { 218: db(" Ping Fatal: Yes - Not Continuing"); 219: $NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping"); 220: $dotests=false; 221: } 222: 223: // do the tests - only actually exec if dotests true 224: 225: $first_test=true; 226: 227: db("Doing Local Tests"); 228: $NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test"); 229: $q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC"; 230: $res=$NATS->DB->Query($q); 231: while ($lrow=$NATS->DB->Fetch_Array($res)) 232: { 233: if ($lrow['nextrunx']<=time()) $testdue=true; 234: else $testdue=false; 235: 236: if ($first_test) 237: { 238: $first_test=false; 239: if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping 240: } 241: else test_sleep(); 242: 243: if ($testdue) 244: { 245: 246: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']); 247: $NATS->EventHandler("localtest_start",$eventdata); 248: 249: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")"); 250: 251: // Build parameter array 252: $params=array(); 253: $params[0]=$lrow['testparam']; // pass standard param in as 0 254: for ($a=1; $a<10; $a++) 255: { 256: $parstr="testparam".$a; 257: $params[$a]=$lrow[$parstr]; 258: } 259: 260: if ($dotests) 261: { 262: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 263: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']); 264: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 265: db(" Result: ".$result); 266: } 267: else $result=0; 268: 269: if ($dotests) 270: { 271: // evaluation 272: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result); 273: else $lvl=nats_eval("L".$lrow['localtestid'],$result); 274: db(" Eval: ".$lvl); 275: 276: // put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status 277: $att=$lrow['attempts']; 278: if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) ) 279: { 280: for ($a=2; $a<=$att; $a++) 281: { 282: test_sleep(); 283: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a); 284: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 285: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']); 286: db(" Result: ".$result); 287: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result); 288: else $lvl=nats_eval("L".$lrow['localtestid'],$result); 289: db(" Eval: ".$lvl); 290: if ($lvl==0) $a=$att+1; // test passed 291: } 292: } 293: 294: // $lvl is now the last lvl regardless of where it came from 295: 296: if ($lvl>$alertlevel) $alertlevel=$lvl; 297: if ($lvl>0) 298: { 299: if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." "; 300: else $s=$lrow['testname']." "; 301: /* 302: if ($lvl>1) $s.=$NATS->Cfg->Get("site.text.failed","failed"); 303: else $s.=$NATS->Cfg->Get("site.text.warning","warning"); 304: */ 305: $s.=oText($lvl); 306: // site.alert.showvalue -- includes value in alert messages if numeric 307: // site.alert.showtext -- includes value in alert messages if textual 308: if (is_numeric($result)) 309: { 310: if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")"; 311: } 312: else // non-numeric 313: { 314: if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")"; 315: } 316: $alerts[$alertc++]=$s; 317: } 318: } else $lvl=-1; 319: 320: // record it 321: if ($lrow['testrecord']==1) 322: { 323: $tid="L".$lrow['localtestid']; 324: $iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES("; 325: $iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")"; 326: $NATS->DB->Query($iq); 327: db(" Recording Test"); 328: } 329: if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net 330: 331: // update localtest record 332: $uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid']; 333: $NATS->DB->Query($uq); 334: 335: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl); 336: $NATS->EventHandler("localtest_finish",$eventdata); 337: } 338: 339: else // test not due so take pre-existing level for it 340: { 341: $lvl=$lrow['alertlevel']; 342: if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl; 343: } 344: 345: 346: } 347: 348: // Node-side testy magic 349: db("Nodeside Testing"); 350: $freshdata=false; 351: if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull 352: { 353: $pullalert=$row['nspullalert']; // what happened the last time we tried 354: 355: if ($row['nsnextx']<=time()) // the time is right 356: { 357: db(" Pulling Data"); 358: $pull_result=$NATS->Nodeside_Pull($row['nodeid']); 359: 360: if ($pull_result===false) // Pull Failed 361: { 362: db(" Pull Failed"); 363: $pullalert=1; // alert 364: $alerts[$alertc++]="pull failed"; 365: $alertlevel=2; 366: } 367: else // Pull Worked 368: { 369: $freshdata=true; 370: $pullalert=0; // ok 371: db(" Pull Succeeded"); 372: } 373: 374: 375: db(" Updating Pull nslast/nextx and nspullalert"); 376: $uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\""; 377: $NATS->DB->Query($uq); 378: if ($NATS->DB->Affected_Rows()<=0) db(" - Failed"); 379: } 380: /* 381: // Process for alerts in here - whether pulled or not! 382: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0"; 383: $tr=$NATS->DB->Query($tq); 384: while ($trow=$NATS->DB->Fetch_Array($tr)) 385: { 386: if ($trow['testname']=="") $tname=$trow['testtype']; 387: else $tname=$trow['testname']; 388: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh 389: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel']; 390: } 391: */ 392: 393: // and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one 394: // if pull not scheduled yet 395: if ($pullalert>0) $alertlevel=2; // so mark a failure 396: 397: } 398: 399: if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) || // pull and tests are on 400: (($row['nsenabled']==1)&&($row['nspushenabled']==1)) ) // or pushed 401: { 402: if ($row['nsfreshpush']==1) 403: { 404: $freshdata=true; 405: $uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\""; 406: $NATS->DB->Query($uq); 407: } 408: // Process for alerts in here - whether pulled or not! 409: $tq="SELECT testtype,testname,alertlevel,lastvalue FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0"; 410: $tr=$NATS->DB->Query($tq); 411: while ($trow=$NATS->DB->Fetch_Array($tr)) 412: { 413: if ($trow['testname']=="") $tname=$trow['testtype']; 414: else $tname=$trow['testname']; 415: if ($freshdata) // only record text to log if fresh 416: { 417: $s=$tname." ".oText($trow['alertlevel']); 418: $result=$trow['lastvalue']; 419: if (is_numeric($result)) 420: { 421: if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")"; 422: } 423: else // non-numeric 424: { 425: if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")"; 426: } 427: $alerts[$alertc++]=$s; 428: } 429: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel']; 430: } 431: } 432: 433: $NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node"); 434: 435: $eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel ); 436: $NATS->EventHandler("node_test_finish",$eventdata); 437: 438: db("Highest Alert Level: ".$alertlevel); 439: db("Alert Count : ".$alertc); 440: $als=""; 441: foreach($alerts as $al) $als.=$al.", "; 442: db("Alerts: ".$als); 443: 444: $NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts); 445: 446: // This is where child/slave nodes spawn 447: 448: // $pingpassed bool holds if pingtest has passed 449: // $alertlevel holds the overall status 450: $chq="SELECT nodeid,masterjustping FROM fnnode WHERE masterid=\"".$row['nodeid']."\""; 451: //echo $chq; 452: 453: $chr=$NATS->DB->Query($chq); 454: $spawnlist=array(); 455: 456: while ($child=$NATS->DB->Fetch_Array($chr)) 457: { 458: if (($child['masterjustping']==1)&&($pingpassed)) $spawnlist[]=$child['nodeid']; 459: else if ($alertlevel==0) $spawnlist[]=$child['nodeid']; 460: // logic: if the child requires a ping and ping has passed ok then spawn it 461: // otherwise (pass on any alert) if everything has passed spawn it 462: } 463: $NATS->DB->Free($chr); 464: 465: if (count($spawnlist)>0) 466: { 467: $cmd="php ./test-threaded.php"; 468: foreach($spawnlist as $child) 469: $cmd.=" ".$child; 470: $cmd.=" > /tmp/ns.master.".$row['nodeid']." &"; 471: db("Children Spawning: ".$cmd); 472: exec($cmd); 473: } 474: 475: 476: // End of the node... carry forward highest level 477: 478: db(" "); 479: 480: if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel; 481: $talertc+=$alertc; 482: 483: } 484: 485: 486: 487: db("Finished Tests... Finishing Off"); 488: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc); 489: if ($highalertlevel>-1) 490: { 491: $uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid; 492: $NATS->DB->Query($uq); 493: } 494: else 495: { 496: $uq="DELETE FROM fntestrun WHERE trid=".$trid; 497: $NATS->DB->Query($uq); 498: } 499: 500: 501: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat"); 502: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop"); 503: 504: // in here for now... 505: $NATS->ActionFlush(); 506: 507: $NATS->Stop(); 508: db("NATS Stopped... Finished"); 509: ?> 510: 511: