1: <?php
  2: /* -------------------------------------------------------------
  3: This file is part of FreeNATS
  4: 
  5: FreeNATS is (C) Copyright 2008-2017 PurplePixie Systems
  6: 
  7: FreeNATS is free software: you can redistribute it and/or modify
  8: it under the terms of the GNU General Public License as published by
  9: the Free Software Foundation, either version 3 of the License, or
 10: (at your option) any later version.
 11: 
 12: FreeNATS is distributed in the hope that it will be useful,
 13: but WITHOUT ANY WARRANTY; without even the implied warranty of
 14: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 15: GNU General Public License for more details.
 16: 
 17: You should have received a copy of the GNU General Public License
 18: along with FreeNATS.  If not, see www.gnu.org/licenses
 19: 
 20: For more information see www.purplepixie.org/freenats
 21: -------------------------------------------------------------- */
 22: 
 23: if ((isset($argc))&&(isset($argv))) // specific node or all nodes
 24: 	{
 25: 	if ($argc>1)
 26: 		{
 27: 		$nfilter=$argv[1];
 28: 		}
 29: 	else $nfilter="";
 30: 	}
 31: 
 32: require("include.php");
 33: 
 34: $dbt="";
 35: 
 36: function db($txt,$nl=true) // debug text
 37: {
 38: global $dbt;
 39: echo $txt;
 40: $dbt.=$txt;
 41: if ($nl) 
 42: 	{
 43: 	echo "\n";
 44: 	$dbt.=" <br>\n";
 45: 	}
 46: }
 47: 
 48: $NATS->Start();
 49: if ($nfilter!="") $st=": Node ".$nfilter;
 50: else $st="";
 51: db("NATS Tester Script Starting".$st);
 52: 
 53: $suspend = $NATS->Cfg->Get("site.tester.suspended",0);
 54: 
 55: if ($suspend != 1)
 56: {
 57: 
 58: 	$highalertlevel=-1;
 59: 	$talertc=0;
 60: 
 61: 	// check if already running
 62: 	$still_running=false;
 63: 	$cq="SELECT trid,startx FROM fntestrun WHERE fnode=\"".ss($nfilter)."\" AND finishx=0 LIMIT 0,1";
 64: 	$cr=$NATS->DB->Query($cq);
 65: 
 66: 	if ($runrow=$NATS->DB->Fetch_Array($cr))
 67: 		{ // yes there is a testrun session for this node(s)
 68: 		$timelimit=$NATS->Cfg->Get("test.session.limit",60*60);
 69: 		if ( (!is_numeric($timelimit)) || ($timelimit<0) ) $timelimit=60*60; // bogus config value
 70: 		
 71: 		// n.b. a timelimit of 0 means the session will never expire so...
 72: 		if ( ($timelimit>0) && ((time()-$runrow['startx'])>$timelimit) )
 73: 			{
 74: 			// valid time limit and the difference is more than the limit so close it
 75: 			$uq="UPDATE fntestrun SET finishx=1 WHERE trid=".$runrow['trid'];
 76: 			$NATS->DB->Query($uq);
 77: 			if ($NATS->DB->Affected_Rows()>0) 
 78: 				{
 79: 				$NATS->Event("Tester Already Running: Cleared",3,"Tester","Stale");
 80: 				db("Tester Already Running: Cleared");
 81: 				}
 82: 			else $NATS->Event("Tester Already Running: Failed to Clear",1,"Tester","Stale"); // weirdness... run anyway
 83: 			}
 84: 		else $still_running=true; // either never timesout or newer than timelimit ago
 85: 		}
 86: 	$NATS->DB->Free($cr);
 87: 
 88: 	// and if it is then don't continue
 89: 	if ($still_running)
 90: 		{
 91: 		$NATS->Event("Tester Already Running: Aborted",1,"Tester","Error");
 92: 		db("Tester Already Running: Aborted");
 93: 		$NATS->Stop();
 94: 		exit();
 95: 		}
 96: 
 97: 
 98: 	$gq="INSERT INTO fntestrun(startx,fnode) VALUES(".time().",\"".ss($nfilter)."\")";
 99: 	$NATS->DB->Query($gq);
100: 	$trid=$NATS->DB->Insert_Id();
101: 	db("Test ID: ".$trid." (Started at ".nicedt(time()).")");
102: 	$NATS->Event("Tester ".$trid." Started",5,"Tester","Start");
103: 
104: 	db(" ");
105: 
106: 	// Find node to test - must be enabled, have id if set, and be due to be tested (nextrunx)
107: 
108: 	$q="SELECT * FROM fnnode WHERE nodeenabled=1";
109: 	if ($nfilter!="") $q.=" AND nodeid=\"".ss($nfilter)."\"";
110: 	$q.=" AND nextrunx<=".time();
111: 
112: 	$r=$NATS->DB->Query($q);
113: 
114: 
115: 	while ($row=$NATS->DB->Fetch_Array($r))
116: 		{
117: 		$dotests=true;
118: 		$alertlevel=0;
119: 		$alerts=array();
120: 		$alertc=0;
121: 		db("NodeID: ".$row['nodeid']);
122: 		$NATS->Event("Tester ".$trid." Node ".$row['nodeid'],10,"Tester","Node");
123: 
124: 		// Scheduling Test In Here - sets dotests to false and alertlevel to -1 untested
125: 		if ($row['scheduleid']!=0) // has a schedule
126: 			{
127: 			db(" Has Schedule: Yes - Checking");
128: 			$run=run_x_in_schedule(time(),$row['scheduleid']);
129: 			if (!$run)
130: 				{
131: 				db(" In Schedule: No - Skipping Tests");
132: 				$NATS->Event("Tester ".$trid." Skipped by Schedule",5,"Tester","Node");
133: 				$dotests=false;
134: 				$alertlevel=-1;
135: 				}
136: 			else db(" In Schedule: Yes");
137: 			}
138: 		
139: 		$eventdata=array( "nodeid" => $row['nodeid'], "in_schedule" => $dotests );
140: 		$NATS->EventHandler("node_test_start",$eventdata);
141: 		
142: 		
143: 		$ptr=0;
144: 		$pal=0;
145: 		
146: 
147: 		// Update lastrun and nextrun regardless of dotests
148: 		$q="UPDATE fnnode SET lastrunx=".time().",nextrunx=".next_run_x($row['testinterval'])." WHERE nodeid=\"".ss($row['nodeid'])."\"";
149: 		$NATS->DB->Query($q);
150: 
151: 		$pingpassed=false; // this will only be set to true if a test is done and passes - for the "child" nodes
152: 		if ($row['pingtest']&&$dotests)
153: 			{
154: 			db(" Ping Test: Yes");
155: 			$NATS->Event("Tester ".$trid." Pinging Node ".$row['nodeid'],10,"Tester","Ping");
156: 			$ptr=PingTest($row['hostname']);
157: 			$NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
158: 			db(" Ping Returned: ".$ptr);
159: 			if ( ($ptr<=0) && ($NATS->Cfg->Get("test.icmp.attempts","2")>1) )
160: 				{
161: 				$att=$NATS->Cfg->Get("test.icmp.attempts","2");
162: 				for ($a=2; $a<=$att; $a++) // starting on second attempt
163: 					{
164: 					// try again...
165: 					test_sleep();
166: 					db(" Trying Ping Again - X".$a);
167: 					$NATS->Event("Tester ".$trid." Ping X".$a." Node ".$row['nodeid'],10,"Tester","Ping");
168: 					$ptr=PingTest($row['hostname']);
169: 					$NATS->Event("Tester ".$trid." Ping Node ".$row['nodeid']." Returned ".$ptr,10,"Tester","Ping");
170: 					db(" Ping Returned: ".$ptr);
171: 					if ($ptr>0) $a=$att+1; // break out of the loop
172: 					}
173: 				}
174: 				
175: 			if ($ptr<=0) 
176: 				{
177: 				$alertlevel=2;
178: 				db(" Ping Test: Failed");
179: 				$alerts[$alertc++]="ping failed";
180: 				$pal=2;
181: 				}
182: 			else 
183: 				{
184: 				db(" Ping Test: Passed");
185: 				$pingpassed=true;
186: 				}
187: 			
188: 			// pingtest output bodge
189: 			// is there a test entry for ICMP
190: 			$fq="SELECT localtestid FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
191: 			$fr=$NATS->DB->Query($fq);
192: 			$ltid_icmp="";
193: 			if ($irow=$NATS->DB->Fetch_Array($fr))
194: 				{ // exists
195: 				$uq="UPDATE fnlocaltest SET alertlevel=".$pal.",lastrunx=".time().",lastvalue=".$ptr.",testrecord=1,testinterval=0 WHERE localtestid=".$irow['localtestid'];
196: 				$ltid_icmp=$irow['localtestid'];
197: 				//echo $uq;
198: 				$NATS->DB->Query($uq);
199: 				}
200: 			else
201: 				{ // doesn't exist
202: 				$uq="INSERT INTO fnlocaltest(nodeid,testrecord,testinterval,testtype,alertlevel,lastrunx,lastvalue) VALUES(\"".$row['nodeid']."\",1,0,\"ICMP\",".$pal.",".time().",".$ptr.")";
203: 				//echo $uq;
204: 				$NATS->DB->Query($uq);
205: 				$ltid_icmp=$NATS->DB->Insert_Id();
206: 				}
207: 			$NATS->DB->Free($fr);
208: 			
209: 			// record the ICMP bodge-test here
210: 			$rq="INSERT INTO fnrecord(testid,recordx,testvalue,alertlevel,nodeid) VALUES(\"L".$ltid_icmp."\",".time().",".$ptr.",".$pal.",\"".$row['nodeid']."\")";
211: 			$NATS->DB->Query($rq);
212: 			//echo $rq." ".$NATS->DB->Affected_Rows()."\n";
213: 			
214: 			}
215: 		else
216: 			{ // further ICMP bodge - update to -1 or do nothing if the test doesn't exist
217: 			$uq="UPDATE fnlocaltest SET alertlevel=-1,lastrunx=".time()." WHERE nodeid=\"".$row['nodeid']."\" AND testtype=\"ICMP\"";
218: 			$NATS->DB->Query($uq);
219: 			}
220: 
221: 		if ($dotests&&($row['pingfatal'])&&($ptr<=0))
222: 			{
223: 			db(" Ping Fatal: Yes - Not Continuing");
224: 			$NATS->Event("Tester ".$trid." Ping Fatal for Node ".$row['nodeid'],10,"Tester","Ping");
225: 			$dotests=false;
226: 			}
227: 
228: 		 	// do the tests - only actually exec if dotests true
229: 
230: 		 	$first_test=true;
231: 		 	
232: 			db("Doing Local Tests");
233: 			$NATS->Event("Tester ".$trid." Testing Node ".$row['nodeid'],10,"Tester","Test");
234: 			$q="SELECT * FROM fnlocaltest WHERE nodeid=\"".$row['nodeid']."\" AND testtype!=\"ICMP\" AND testenabled=1 ORDER BY localtestid ASC";
235: 			$res=$NATS->DB->Query($q);
236: 			while ($lrow=$NATS->DB->Fetch_Array($res))
237: 				{
238: 				if ($lrow['nextrunx']<=time()) $testdue=true;
239: 				else $testdue=false;
240: 					
241: 				if ($first_test)
242: 					{
243: 					$first_test=false;
244: 					if ($row['pingtest']==1) test_sleep(); // sleep if has done a ping
245: 					}
246: 				else test_sleep();
247: 				
248: 				if ($testdue)
249: 					{
250: 				
251: 					$eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype']);
252: 					$NATS->EventHandler("localtest_start",$eventdata);
253: 						
254: 					db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].")");
255: 					
256: 					// Build parameter array
257: 					$params=array();
258: 					$params[0]=$lrow['testparam']; // pass standard param in as 0
259: 					for ($a=1; $a<10; $a++)
260: 						{
261: 						$parstr="testparam".$a;
262: 						$params[$a]=$lrow[$parstr];
263: 						}
264: 					
265: 					if ($dotests)
266: 						{
267: 						$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
268: 						$result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
269: 						$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." Result ".$result." from ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
270: 						db(" Result: ".$result);
271: 						}
272: 					else $result=0;
273: 					
274: 					if ($dotests)
275: 					{
276: 					// evaluation
277: 					if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
278: 					else $lvl=nats_eval("L".$lrow['localtestid'],$result);
279: 					db(" Eval: ".$lvl);
280: 					
281: 					// put in the custom retries based on attempts here - we KNOW dotests is on so don't need to worry about untested status
282: 					$att=$lrow['attempts'];
283: 					if ( ($lvl!=0) && (is_numeric($att)) && ($att>1) )
284: 						{
285: 						for ($a=2; $a<=$att; $a++)
286: 							{
287: 							test_sleep();
288: 							db(" Test: ".$lrow['testtype']." (".$lrow['testparam'].") X".$a);
289: 							$NATS->Event("Tester ".$trid." Node ".$row['nodeid']." X".$a." Doing ".$lrow['testtype']."(".$lrow['testparam'].")",10,"Tester","Test");
290: 							$result=DoTest($lrow['testtype'],$lrow['testparam'],$row['hostname'],$lrow['timeout'],$params,$row['nodeid']);
291: 							db(" Result: ".$result);
292: 							if ($lrow['simpleeval']==1) $lvl=SimpleEval($lrow['testtype'],$result);
293: 							else $lvl=nats_eval("L".$lrow['localtestid'],$result);
294: 							db(" Eval: ".$lvl);
295: 							if ($lvl==0) $a=$att+1; // test passed
296: 							}
297: 						}
298: 					
299: 					// $lvl is now the last lvl regardless of where it came from
300: 							
301: 					if ($lvl>$alertlevel) $alertlevel=$lvl;
302: 					if ($lvl>0)
303: 						{
304: 						if ($lrow['testname']=="") $s=$lrow['testtype']."/".substr($lrow['testparam'],0,5)." ";
305: 						else $s=$lrow['testname']." ";
306: 						/*
307: 						if ($lvl>1) $s.=$NATS->Cfg->Get("site.text.failed","failed");
308: 						else $s.=$NATS->Cfg->Get("site.text.warning","warning");
309: 						*/
310: 						$s.=oText($lvl);
311: 						// site.alert.showvalue -- includes value in alert messages if numeric
312: 						// site.alert.showtext -- includes value in alert messages if textual
313: 						if (is_numeric($result))
314: 							{
315: 							if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")";
316: 							}
317: 						else // non-numeric
318: 							{
319: 							if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")";
320: 							}
321: 						$alerts[$alertc++]=$s;
322: 						}
323: 					} else $lvl=-1;
324: 						
325: 					// record it
326: 					if ($lrow['testrecord']==1)
327: 						{
328: 						$tid="L".$lrow['localtestid'];
329: 						$iq="INSERT INTO fnrecord(testid,nodeid,alertlevel,testvalue,recordx) VALUES(";
330: 						$iq.="\"".$tid."\",\"".$row['nodeid']."\",".$lvl.",".$result.",".time().")";
331: 						$NATS->DB->Query($iq);
332: 						db(" Recording Test");
333: 						}
334: 					if ((!isset($result))||(!is_numeric($result))) $result=0; // safety net
335: 						
336: 					// update localtest record
337: 					$uq="UPDATE fnlocaltest SET lastrunx=".time().",nextrunx=".next_run_x($lrow['testinterval']).",alertlevel=".$lvl.",lastvalue=".$result." WHERE localtestid=".$lrow['localtestid'];
338: 					$NATS->DB->Query($uq);
339: 					
340: 					$eventdata=array("nodeid"=>$row['nodeid'],"testid"=>"L".$lrow['testparam'],"testtype"=>$lrow['testtype'],"alertlevel"=>$lvl);
341: 					$NATS->EventHandler("localtest_finish",$eventdata);
342: 					}
343: 					
344: 				else // test not due so take pre-existing level for it
345: 					{
346: 					$lvl=$lrow['alertlevel'];
347: 					if (($lvl>0)&&($lvl>$alertlevel)) $alertlevel=$lvl;
348: 					}
349: 				
350: 				
351: 				}
352: 				
353: 		// Node-side testy magic
354: 		db("Nodeside Testing");
355: 		$freshdata=false;
356: 		if ( $dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1) ) // should be doing a pull
357: 			{
358: 			$pullalert=$row['nspullalert']; // what happened the last time we tried
359: 			
360: 			if ($row['nsnextx']<=time()) // the time is right
361: 				{
362: 				db(" Pulling Data");
363: 				$pull_result=$NATS->Nodeside_Pull($row['nodeid']);
364: 				
365: 				if ($pull_result===false) // Pull Failed
366: 					{
367: 					db(" Pull Failed");
368: 					$pullalert=1; // alert
369: 					$alerts[$alertc++]="pull failed";
370: 					$alertlevel=2;
371: 					}
372: 				else // Pull Worked
373: 					{
374: 					$freshdata=true;
375: 					$pullalert=0; // ok
376: 					db(" Pull Succeeded");
377: 					}
378: 					
379: 					
380: 				db(" Updating Pull nslast/nextx and nspullalert");
381: 				$uq="UPDATE fnnode SET nsnextx=".next_run_x($row['nsinterval']).",nspullalert=".$pullalert.",nslastx=".time()." WHERE nodeid=\"".$row['nodeid']."\"";
382: 				$NATS->DB->Query($uq);
383: 				if ($NATS->DB->Affected_Rows()<=0) db(" - Failed");
384: 				}
385: 			/*
386: 			// Process for alerts in here - whether pulled or not!
387: 			$tq="SELECT testtype,testname,alertlevel FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
388: 			$tr=$NATS->DB->Query($tq);
389: 			while ($trow=$NATS->DB->Fetch_Array($tr))
390: 				{
391: 				if ($trow['testname']=="") $tname=$trow['testtype'];
392: 				else $tname=$trow['testname'];
393: 				if ($freshdata) $alerts[$alertc++]=$tname." ".oText($trow['alertlevel']); // only record text to log if fresh
394: 				if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
395: 				}
396: 			*/
397: 				
398: 			// and finally again use pullalert - this is either the new value if a pull was attempted or just remains the same as the old one
399: 			// if pull not scheduled yet
400: 			if ($pullalert>0) $alertlevel=2; // so mark a failure
401: 
402: 			}
403: 			
404: 		if ( ($dotests && ($row['nsenabled']==1) && ($row['nspullenabled']==1)) ||		// pull and tests are on
405: 			(($row['nsenabled']==1)&&($row['nspushenabled']==1)) )	// or pushed
406: 			{
407: 			if ($row['nsfreshpush']==1)
408: 				{
409: 				$freshdata=true;
410: 				$uq="UPDATE fnnode SET nsfreshpush=0 WHERE nodeid=\"".$row['nodeid']."\"";
411: 				$NATS->DB->Query($uq);
412: 				}
413: 			// Process for alerts in here - whether pulled or not!
414: 			$tq="SELECT testtype,testname,alertlevel,lastvalue FROM fnnstest WHERE nodeid=\"".$row['nodeid']."\" AND testenabled=1 AND testalerts=1 AND alertlevel>0";
415: 			$tr=$NATS->DB->Query($tq);
416: 			while ($trow=$NATS->DB->Fetch_Array($tr))
417: 				{
418: 				if ($trow['testname']=="") $tname=$trow['testtype'];
419: 				else $tname=$trow['testname'];
420: 				if ($freshdata) // only record text to log if fresh
421: 					{
422: 					$s=$tname." ".oText($trow['alertlevel']); 
423: 					$result=$trow['lastvalue'];
424: 					if (is_numeric($result))
425: 							{
426: 							if ($NATS->Cfg->Get("site.alert.showvalue",0)==1) $s.=" (".$result.")";
427: 							}
428: 						else // non-numeric
429: 							{
430: 							if ($NATS->Cfg->Get("site.alert.showtext",0)==1) $s.=" (".$result.")";
431: 							}
432: 					$alerts[$alertc++]=$s;
433: 					}
434: 				if ($trow['alertlevel']>$alertlevel) $alertlevel=$trow['alertlevel'];
435: 				}
436: 			}
437: 				
438: 		$NATS->Event("Tester ".$trid." Finished Node ".$row['nodeid'],10,"Tester","Node");
439: 
440: 		$eventdata=array( "nodeid" => $row['nodeid'], "alertlevel" => $alertlevel );
441: 		$NATS->EventHandler("node_test_finish",$eventdata);
442: 
443: 		db("Highest Alert Level: ".$alertlevel);
444: 		db("Alert Count        : ".$alertc);
445: 		$als="";
446: 		foreach($alerts as $al) $als.=$al.", ";
447: 		db("Alerts: ".$als);
448: 
449: 		$NATS->SetAlerts($row['nodeid'],$alertlevel,$alerts);
450: 		
451: 		// This is where child/slave nodes spawn
452: 		
453: 		// $pingpassed bool holds if pingtest has passed
454: 		// $alertlevel holds the overall status
455: 		$chq="SELECT nodeid,masterjustping FROM fnnode WHERE masterid=\"".$row['nodeid']."\"";
456: 		//echo $chq;
457: 		
458: 		$chr=$NATS->DB->Query($chq);
459: 		$spawnlist=array();
460: 		
461: 		while ($child=$NATS->DB->Fetch_Array($chr))
462: 			{
463: 			if (($child['masterjustping']==1)&&($pingpassed)) $spawnlist[]=$child['nodeid'];
464: 			else if ($alertlevel==0) $spawnlist[]=$child['nodeid'];
465: 			// logic: if the child requires a ping and ping has passed ok then spawn it
466: 			// otherwise (pass on any alert) if everything has passed spawn it
467: 			}
468: 		$NATS->DB->Free($chr);
469: 		
470: 		if (count($spawnlist)>0)
471: 			{
472: 			$cmd="php ./test-threaded.php";
473: 			foreach($spawnlist as $child)
474: 				$cmd.=" ".$child;
475: 			$cmd.=" > /tmp/ns.master.".$row['nodeid']." &";
476: 			db("Children Spawning: ".$cmd);
477: 			exec($cmd);
478: 			}
479: 		
480: 		
481: 		// End of the node... carry forward highest level
482: 
483: 		db(" ");
484: 		
485: 		if ($alertlevel>$highalertlevel) $highalertlevel=$alertlevel;
486: 		$talertc+=$alertc;
487: 		
488: 		}
489: 
490: 
491: 
492: 	db("Finished Tests... Finishing Off");
493: 	db("Summary: Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc);
494: 	if ($highalertlevel>-1)
495: 		{
496: 		$uq="UPDATE fntestrun SET finishx=".time().",routput=\"".ss($dbt)."\" WHERE trid=".$trid;
497: 		$NATS->DB->Query($uq);
498: 		}
499: 	else
500: 		{
501: 		$uq="DELETE FROM fntestrun WHERE trid=".$trid;
502: 		$NATS->DB->Query($uq);
503: 		}
504: 
505: 
506: 	$NATS->Event("Tester ".$trid." Highest Level ".$highalertlevel.", Alerts ".$talertc,7,"Tester","Stat");
507: 	$NATS->Event("Tester ".$trid." Finished",5,"Tester","Stop");
508: 
509: 	// in here for now...
510: 	$NATS->ActionFlush();
511: 
512: } // end of suspend check
513: else
514: {
515: 	// Suspended
516: 	db("NATS Testing Suspended");
517: }
518: 
519: $NATS->Stop();
520: db("NATS Stopped... Finished");
521: ?>
522: 
523: