File: 1.00.6a/server/bin/tester.php (View as Code)

1: 2: /* ------------------------------------------------------------- 3: This file is part of FreeNATS 4: 5: FreeNATS is (C) Copyright 2008 PurplePixie Systems 6: 7: FreeNATS is free software: you can redistribute it and/or modify 8: it under the terms of the GNU General Public License as published by 9: the Free Software Foundation, either version 3 of the License, or 10: (at your option) any later version. 11: 12: FreeNATS is distributed in the hope that it will be useful, 13: but WITHOUT ANY WARRANTY; without even the implied warranty of 14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15: GNU General Public License for more details. 16: 17: You should have received a copy of the GNU General Public License 18: along with FreeNATS. If not, see www.gnu.org/licenses 19: 20: For more information see www.purplepixie.org/freenats 21: -------------------------------------------------------------- */ 22: 23: if ((isset($argc))&&(isset($argv))) 24: { 25: if ($argc>1) 26: { 27: $nfilter=$argv[1]; 28: } 29: else $nfilter=""; 30: } 31: 32: require("include.php"); 33: 34: $dbt=""; 35: 36: function db($txt,$nl=true) 37: { 38: global $dbt; 39: echo $txt; 40: $dbt.=$txt; 41: if ($nl) 42: { 43: echo "\n"; 44: $dbt.="
\n";
45: } 46: } 47: 48: $NATS->Start(); 49: if ($nfilter!="") $st=": Node ".$nfilter; 50: else $st=""; 51: db("NATS Tester Script Starting".$st); 52: 53: $highalertlevel=-1; 54: $talertc=0; 55: 56: // check if already running 57: $cq="SELECT startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1"; 58: $cr=$NATS->DB->Query($cq); 59: if ($NATS->DB->Num_Rows($cr)>0) 60: { 61: $NATS->Event("Tester Already Running: Aborted",1,"Tester","Error"); 62: db("Tester Already Running: Aborted"); 63: $NATS->Stop(); 64: exit(); 65: } 66: $NATS->DB->Free($cr); 67: 68: $gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")"; 69: $NATS->DB->Query($gq); 70: $trid=$NATS->DB->Insert_Id(); 71: db("Test ID: ".$trid." (Started at ".nicedt(time()).")"); 72: $NATS->Event("Tester ".$trid." Started",5,"Tester","Start"); 73: 74: db(" "); 75: 76: // Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx) 77: 78: $q="SELECT * FROM fnnode WHERE nodeenabled=1"; 79: if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\""; 80: $q.=" AND nextrunx<=".time(); 81: 82: $r=$NATS->DB->Query($q); 83: 84: 85: while ($row=$NATS->DB->Fetch_Array($r)) 86: { 87: $dotests=true; 88: $alertlevel=0; 89: $alerts=array(); 90: $alertc=0; 91: db("NodeID: ".$row['nodeid']); 92: $NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node"); 93: 94: // Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested 95: if ($row['scheduleid']!=0) // has a schedule 96: { 97: db(" Has Schedule: Yes - Checking"); 98: $run=run_x_in_schedule(time(),$row['scheduleid']); 99: if (!$run) 100: { 101: db(" In Schedule: No - Skipping Tests"); 102: $NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node"); 103: $dotests=false; 104: $alertlevel=-1; 105: } 106: else db(" In Schedule: Yes"); 107: } 108: 109: $eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests ); 110: $NATS->EventHandler("node_test_start",$eventdata); 111: 112: 113: $ptr=0; 114: $pal=0; 115: 116: 117: // Update lastrun and nextrun regardless of dotests 118: $q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\""; 119: $NATS->DB->Query($q); 120: 121: 122: if ($row['pingtest']&&$dotests) 123: { 124: db(" Ping Test: Yes"); 125: $NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping"); 126: $ptr=PingTest($row['hostname']); 127: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping"); 128: db(" Ping Returned: ".$ptr); 129: if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) ) 130: { 131: $att=$NATS->Cfg->Get("test.icmp.attempts","2"); 132: for ($a=2; $a<=$att; $a++) // starting on second attempt 133: { 134: // try again... 135: test_sleep(); 136: db(" Trying Ping Again - X".$a); 137: $NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping"); 138: $ptr=PingTest($row['hostname']); 139: $NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping"); 140: db(" Ping Returned: ".$ptr); 141: if ($ptr>0) $a=$att+1; // break out of the loop 142: } 143: } 144: 145: if ($ptr<=0) 146: { 147: $alertlevel=2; 148: db(" Ping Test: Failed"); 149: $alerts[$alertc++]="ping failed"; 150: $pal=2; 151: } 152: else db(" Ping Test: Passed"); 153: 154: // pingtest output bodge 155: // is there a test entry for ICMP 156: $fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\""; 157: $fr=$NATS->DB->Query($fq); 158: $ltid_icmp=""; 159: if ($irow=$NATS->DB->Fetch_Array($fr)) 160: { // exists 161: $uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid']; 162: $ltid_icmp=$irow['localtestid']; 163: //echo $uq; 164: $NATS->DB->Query($uq); 165: } 166: else 167: { // doesn't exist 168: $uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")"; 169: //echo $uq; 170: $NATS->DB->Query($uq); 171: $ltid_icmp=$NATS->DB->Insert_Id(); 172: } 173: $NATS->DB->Free($fr); 174: 175: // record the ICMP bodge-test here 176: $rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")"; 177: $NATS->DB->Query($rq); 178: //echo $rq." ".$NATS->DB->Affected_Rows()."\n"; 179: 180: } 181: else 182: { // further ICMP bodge - update to -1 or do nothing if the test doesn't exist 183: $uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\""; 184: $NATS->DB->Query($uq); 185: } 186: 187: if ($dotests&&($row['pingfatal'])&&($ptr<=0)) 188: { 189: db(" Ping Fatal: Yes - Not Continuing"); 190: $NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping"); 191: $dotests=false; 192: } 193: 194: // do the tests - only actually exec if dotests true 195: 196: $first_test=true; 197: 198: db("Doing Local Tests"); 199: $NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test"); 200: $q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC"; 201: $res=$NATS->DB->Query($q); 202: while ($lrow=$NATS->DB->Fetch_Array($res)) 203: { 204: if ($lrow['nextrunx']<=time()) $testdue=true; 205: else $testdue=false; 206: 207: if ($first_test) 208: { 209: $first_test=false; 210: if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping 211: } 212: else test_sleep(); 213: 214: if ($testdue) 215: { 216: 217: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']); 218: $NATS->EventHandler("localtest_start",$eventdata); 219: 220: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")"); 221: 222: // Build parameter array 223: $params=array(); 224: $params[0]=$lrow['testparam']; // pass standard param in as 0 225: for ($a=1; $a<10; $a++) 226: { 227: $parstr="testparam".$a; 228: $params[$a]=$lrow[$parstr]; 229: } 230: 231: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 232: if ($testdue) $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']); 233: else $result=0; 234: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 235: db(" Result: ".$result); 236: 237: if ($dotests) 238: { 239: // evaluation 240: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result); 241: else $lvl=nats_eval("L".$lrow['localtestid'],$result); 242: db(" Eval: ".$lvl); 243: 244: // put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status 245: $att=$lrow['attempts']; 246: if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) ) 247: { 248: for ($a=2; $a<=$att; $a++) 249: { 250: test_sleep(); 251: db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a); 252: $NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test"); 253: $result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']); 254: db(" Result: ".$result); 255: if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result); 256: else $lvl=nats_eval("L".$lrow['localtestid'],$result); 257: db(" Eval: ".$lvl); 258: if ($lvl==0) $a=$att+1; // test passed 259: } 260: } 261: 262: // $lvl is now the last lvl regardless of where it came from 263: 264: if ($lvl>$alertlevel) $alertlevel=$lvl; 265: if ($lvl>0) 266: { 267: if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." "; 268: else $s=$lrow['testname']." "; 269: if ($lvl>1) $s.="failed"; 270: else $s.="warning"; 271: $alerts[$alertc++]=$s; 272: } 273: } else $lvl=-1; 274: 275: // record it 276: if ($lrow['testrecord']==1) 277: { 278: $tid="L".$lrow['localtestid']; 279: $iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES("; 280: $iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")"; 281: $NATS->DB->Query($iq); 282: db(" Recording Test"); 283: } 284: if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net 285: 286: // update localtest record 287: $uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid']; 288: $NATS->DB->Query($uq); 289: 290: $eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl); 291: $NATS->EventHandler("localtest_finish",$eventdata); 292: } 293: 294: else // test not due so take pre-existing level for it 295: { 296: $lvl=$lrow['alertlevel']; 297: if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl; 298: } 299: 300: 301: } 302: 303: // Node-side testy magic 304: db("Nodeside Testing"); 305: $freshdata=false; 306: if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull 307: { 308: $pullalert=$row['nspullalert']; // what happened the last time we tried 309: 310: if ($row['nsnextx']<=time()) // the time is right 311: { 312: db(" Pulling Data"); 313: $pull_result=$NATS->Nodeside_Pull($row['nodeid']); 314: 315: if ($pull_result===false) // Pull Failed 316: { 317: db(" Pull Failed"); 318: $pullalert=1; // alert 319: $alerts[$alertc++]="pull failed"; 320: $alertlevel=2; 321: } 322: else // Pull Worked 323: { 324: $freshdata=true; 325: $pullalert=0; // ok 326: db(" Pull Succeeded"); 327: } 328: 329: 330: db(" Updating Pull nslast/nextx and nspullalert"); 331: $uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\""; 332: $NATS->DB->Query($uq); 333: if ($NATS->DB->Affected_Rows()<=0) db(" - Failed"); 334: } 335: /* 336: // Process for alerts in here - whether pulled or not! 337: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0"; 338: $tr=$NATS->DB->Query($tq); 339: while ($trow=$NATS->DB->Fetch_Array($tr)) 340: { 341: if ($trow['testname']=="") $tname=$trow['testtype']; 342: else $tname=$trow['testname']; 343: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh 344: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel']; 345: } 346: */ 347: 348: // and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one 349: // if pull not scheduled yet 350: if ($pullalert>0) $alertlevel=2; // so mark a failure 351: 352: } 353: 354: if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) || // pull and tests are on 355: (($row['nsenabled']==1)&&($row['nspushenabled']==1)) ) // or pushed 356: { 357: if ($row['nsfreshpush']==1) 358: { 359: $freshdata=true; 360: $uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\""; 361: $NATS->DB->Query($uq); 362: } 363: // Process for alerts in here - whether pulled or not! 364: $tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0"; 365: $tr=$NATS->DB->Query($tq); 366: while ($trow=$NATS->DB->Fetch_Array($tr)) 367: { 368: if ($trow['testname']=="") $tname=$trow['testtype']; 369: else $tname=$trow['testname']; 370: if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh 371: if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel']; 372: } 373: } 374: 375: $NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node"); 376: 377: $eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel ); 378: $NATS->EventHandler("node_test_finish",$eventdata); 379: 380: db("Highest Alert Level: ".$alertlevel); 381: db("Alert Count : ".$alertc); 382: $als=""; 383: foreach($alerts as $al) $als.=$al.", "; 384: db("Alerts: ".$als); 385: 386: $NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts); 387: 388: db(" "); 389: 390: if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel; 391: $talertc+=$alertc; 392: 393: } 394: 395: 396: 397: db("Finished Tests... Finishing Off"); 398: db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc); 399: if ($highalertlevel>-1) 400: { 401: $uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid; 402: $NATS->DB->Query($uq); 403: } 404: else 405: { 406: $uq="DELETE FROM fntestrun WHERE trid=".$trid; 407: $NATS->DB->Query($uq); 408: } 409: 410: 411: $NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat"); 412: $NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop"); 413: 414: // in here for now... 415: $NATS->ActionFlush(); 416: 417: $NATS->Stop(); 418: db("NATS Stopped... Finished"); 419: ?> 420: 421: