|
Viewing File server/bin/tester.php of 1.02.1a
|
1: <?php 2: /* ------------------------------------------------------------- 3: This file is part of FreeNATS 4: 5: FreeNATS is (C) Copyright 2008 PurplePixie Systems 6: 7: FreeNATS is free software: you can redistribute it and/or modify 8: it under the terms of the GNU General Public License as published by 9: the Free Software Foundation, either version 3 of the License, or 10: (at your option) any later version. 11: 12: FreeNATS is distributed in the hope that it will be useful, 13: but WITHOUT ANY WARRANTY; without even the implied warranty of 14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15: GNU General Public License for more details. 16: 17: You should have received a copy of the GNU General Public License 18: along with FreeNATS. If not, see www.gnu.org/licenses 19: 20: For more information see www.purplepixie.org/freenats 21: -------------------------------------------------------------- */ 22: 23: if ((isset($argc))&&(isset($argv))) 24: { 25: if ($argc>1) 26: { 27: $nfilter=$argv[1]; 28: } 29: else $nfilter=""; 30: } 31: 32: require("include.php"); 33: 34: $dbt=""; 35: 36: function db($txt,$nl=true) 37: { 38: global $dbt; 39: echo $txt; 40: $dbt.=$txt; 41: if ($nl) 42: { 43: echo "\n"; 44: $dbt.=" <br>\n"; 45: } 46: } 47: 48: $NATS->Start(); 49: if ($nfilter!="") $st=": Node ".$nfilter; 50: else $st=""; 51: db("NATS Tester Script Starting".$st); 52: 53: $highalertlevel=-1; 54: $talertc=0; 55: 56: // check if already running 57: $cq="SELECT startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1"; 58: $cr=$NATS->DB->Query($cq); 59: if ($NATS->DB->Num_Rows($cr)>0) 60: { 61: $NATS->Event("Tester Already Running: Aborted",1,"Tester","Error"); 62: db("Tester Already Running: Aborted"); 63: $NATS->Stop(); 64: exit(); 65: } 66: $NATS->DB->Free($cr); 67: 68: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")"; 69: $NATS->DB->Query($gq); 70: $trid=$NATS->DB->Insert_Id(); 71: db("Test ID: ".$trid." (Started at ".nicedt(time()).")"); 72: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start"); 73: 74: db(" "); 75: 76: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx) 77: 78: $q="SELECT * FROM fnnode WHERE nodeenabled=1"; 79: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\""; 80: $q.=" AND nextrunx<=".time(); 81: 82: $r=$NATS->DB->Query($q); 83: 84: 85: while ($row=$NATS->DB->Fetch_Array($r)) 86: { 87: $dotests=true; 88: $alertlevel=0; 89: $alerts=array(); 90: $alertc=0; 91: db("NodeID: ".$row['nodeid']); 92: $NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node"); 93: 94: // Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested 95: if ($row['scheduleid']!=0) // has a schedule 96: { 97: db(" Has Schedule: Yes - Checking"); 98: $run=run_x_in_schedule(time(),$row['scheduleid']); 99: if (!$run) 100: { 101: db(" In Schedule: No - Skipping Tests"); 102: $NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node"); 103: $dotests=false; 104: $alertlevel=-1; 105: } 106: else db(" In Schedule: Yes"); 107: } 108: 109: $eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests ); 110: $NATS->EventHandler("node_test_start",$eventdata); 111: 112: 113: $ptr=0; 114: $pal=0; 115: 116: 117: // Update lastrun and nextrun regardless of dotests 118: $q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\""; 119: $NATS->DB->Query($q); 120: 121: 122: if ($row['pingtest']&&$dotests) 123: { 124: db(" Ping Test: Yes"); 125: $NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping"); 126: $ptr=PingTest($row['hostname']); 127: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping"); 128: db(" Ping Returned: ".$ptr); 129: if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) ) 130: { 131: $att=$NATS->Cfg->Get("test.icmp.attempts","2"); 132: for ($a=2; $a<=$att; $a++) // starting on second attempt 133: { 134: // try again... 135: test_sleep(); 136: db(" Trying Ping Again - X".$a); 137: $NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping"); 138: $ptr=PingTest($row['hostname']); 139: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping"); 140: db(" Ping Returned: ".$ptr); 141: if ($ptr>0) $a=$att+1; // break out of the loop 142: } 143: } 144: 145: if ($ptr<=0) 146: { 147: $alertlevel=2; 148: db(" Ping Test: Failed"); 149: $alerts[$alertc++]="ping failed"; 150: $pal=2; 151: } 152: else db(" Ping Test: Passed"); 153: 154: // pingtest output bodge 155: // is there a test entry for ICMP 156: $fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\""; 157: $fr=$NATS->DB->Query($fq); 158: $ltid_icmp=""; 159: if ($irow=$NATS->DB->Fetch_Array($fr)) 160: { // exists 161: $uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid']; 162: $ltid_icmp=$irow['localtestid']; 163: //echo $uq; 164: $NATS->DB->Query($uq); 165: } 166: else 167: { // doesn't exist 168: $uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")"; 169: //echo $uq; 170: $NATS->DB->Query($uq); 171: $ltid_icmp=$NATS->DB->Insert_Id(); 172: } 173: $NATS->DB->Free($fr); 174: 175: // record the ICMP bodge-test here 176: $rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")"; 177: $NATS->DB->Query($rq); 178: //echo $rq." ".$NATS->DB->Affected_Rows()."\n"; 179: 180: } 181: else 182: { // further ICMP bodge - update to -1 or do nothing if the test doesn't exist 183: $uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\""; 184: $NATS->DB->Query($uq); 185: } 186: 187: if ($dotests&&($row['pingfatal'])&&($ptr<=0)) 188: { 189: db(" Ping Fatal: Yes - Not Continuing"); 190: $NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping"); 191: $dotests=false; 192: } 193: 194: // do the tests - only actually exec if dotests true 195: 196: $first_test=true; 197: 198: db("Doing Local Tests"); 199: $NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test"); 200: $q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC"; 201: $res=$NATS->DB->Query($q); 202: while ($lrow=$NATS->DB->Fetch_Array($res)) 203: { 204: if ($lrow['nextrunx']<=time()) $testdue=true; 205: else $testdue=false; 206: 207: if ($first_test) 208: { 209: $first_test=false; 210: if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping 211: } 212: else test_sleep(); 213: 214: if ($testdue) 215: { 216: 217: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']); 218: $NATS->EventHandler("localtest_start",$eventdata); 219: 220: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")"); 221: 222: // Build parameter array 223: $params=array(); 224: $params[0]=$lrow['testparam']; // pass standard param in as 0 225: for ($a=1; $a<10; $a++) 226: { 227: $parstr="testparam".$a; 228: $params[$a]=$lrow[$parstr]; 229: } 230: 231: if ($dotests) 232: { 233: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 234: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']); 235: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 236: db(" Result: ".$result); 237: } 238: else $result=0; 239: 240: if ($dotests) 241: { 242: // evaluation 243: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result); 244: else $lvl=nats_eval("L".$lrow['localtestid'],$result); 245: db(" Eval: ".$lvl); 246: 247: // put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status 248: $att=$lrow['attempts']; 249: if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) ) 250: { 251: for ($a=2; $a<=$att; $a++) 252: { 253: test_sleep(); 254: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a); 255: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 256: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']); 257: db(" Result: ".$result); 258: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result); 259: else $lvl=nats_eval("L".$lrow['localtestid'],$result); 260: db(" Eval: ".$lvl); 261: if ($lvl==0) $a=$att+1; // test passed 262: } 263: } 264: 265: // $lvl is now the last lvl regardless of where it came from 266: 267: if ($lvl>$alertlevel) $alertlevel=$lvl; 268: if ($lvl>0) 269: { 270: if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." "; 271: else $s=$lrow['testname']." "; 272: if ($lvl>1) $s.="failed"; 273: else $s.="warning"; 274: $alerts[$alertc++]=$s; 275: } 276: } else $lvl=-1; 277: 278: // record it 279: if ($lrow['testrecord']==1) 280: { 281: $tid="L".$lrow['localtestid']; 282: $iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES("; 283: $iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")"; 284: $NATS->DB->Query($iq); 285: db(" Recording Test"); 286: } 287: if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net 288: 289: // update localtest record 290: $uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid']; 291: $NATS->DB->Query($uq); 292: 293: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl); 294: $NATS->EventHandler("localtest_finish",$eventdata); 295: } 296: 297: else // test not due so take pre-existing level for it 298: { 299: $lvl=$lrow['alertlevel']; 300: if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl; 301: } 302: 303: 304: } 305: 306: // Node-side testy magic 307: db("Nodeside Testing"); 308: $freshdata=false; 309: if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull 310: { 311: $pullalert=$row['nspullalert']; // what happened the last time we tried 312: 313: if ($row['nsnextx']<=time()) // the time is right 314: { 315: db(" Pulling Data"); 316: $pull_result=$NATS->Nodeside_Pull($row['nodeid']); 317: 318: if ($pull_result===false) // Pull Failed 319: { 320: db(" Pull Failed"); 321: $pullalert=1; // alert 322: $alerts[$alertc++]="pull failed"; 323: $alertlevel=2; 324: } 325: else // Pull Worked 326: { 327: $freshdata=true; 328: $pullalert=0; // ok 329: db(" Pull Succeeded"); 330: } 331: 332: 333: db(" Updating Pull nslast/nextx and nspullalert"); 334: $uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\""; 335: $NATS->DB->Query($uq); 336: if ($NATS->DB->Affected_Rows()<=0) db(" - Failed"); 337: } 338: /* 339: // Process for alerts in here - whether pulled or not! 340: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0"; 341: $tr=$NATS->DB->Query($tq); 342: while ($trow=$NATS->DB->Fetch_Array($tr)) 343: { 344: if ($trow['testname']=="") $tname=$trow['testtype']; 345: else $tname=$trow['testname']; 346: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh 347: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel']; 348: } 349: */ 350: 351: // and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one 352: // if pull not scheduled yet 353: if ($pullalert>0) $alertlevel=2; // so mark a failure 354: 355: } 356: 357: if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) || // pull and tests are on 358: (($row['nsenabled']==1)&&($row['nspushenabled']==1)) ) // or pushed 359: { 360: if ($row['nsfreshpush']==1) 361: { 362: $freshdata=true; 363: $uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\""; 364: $NATS->DB->Query($uq); 365: } 366: // Process for alerts in here - whether pulled or not! 367: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0"; 368: $tr=$NATS->DB->Query($tq); 369: while ($trow=$NATS->DB->Fetch_Array($tr)) 370: { 371: if ($trow['testname']=="") $tname=$trow['testtype']; 372: else $tname=$trow['testname']; 373: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh 374: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel']; 375: } 376: } 377: 378: $NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node"); 379: 380: $eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel ); 381: $NATS->EventHandler("node_test_finish",$eventdata); 382: 383: db("Highest Alert Level: ".$alertlevel); 384: db("Alert Count : ".$alertc); 385: $als=""; 386: foreach($alerts as $al) $als.=$al.", "; 387: db("Alerts: ".$als); 388: 389: $NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts); 390: 391: // This is where child/slave nodes would spawn 392: 393: 394: // End of the node... carry forward highest level 395: 396: db(" "); 397: 398: if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel; 399: $talertc+=$alertc; 400: 401: } 402: 403: 404: 405: db("Finished Tests... Finishing Off"); 406: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc); 407: if ($highalertlevel>-1) 408: { 409: $uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid; 410: $NATS->DB->Query($uq); 411: } 412: else 413: { 414: $uq="DELETE FROM fntestrun WHERE trid=".$trid; 415: $NATS->DB->Query($uq); 416: } 417: 418: 419: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat"); 420: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop"); 421: 422: // in here for now... 423: $NATS->ActionFlush(); 424: 425: $NATS->Stop(); 426: db("NATS Stopped... Finished"); 427: ?> 428: 429: