perfsonar development work

Author: maxim
Date: 2007-07-19 14:16:27 -0400 (Thu, 19 Jul 2007)
New Revision: 2554

Modified:
   trunk/perfsonar/contrib/CricketParser/cricket2SONAR.pl
Log:
fixed numerous bugs and replace old utilization event type with new one


Modified: trunk/perfsonar/contrib/CricketParser/cricket2SONAR.pl
===================================================================
--- trunk/perfsonar/contrib/CricketParser/cricket2SONAR.pl      2007-07-19 
14:28:49 UTC (rev 2553)
+++ trunk/perfsonar/contrib/CricketParser/cricket2SONAR.pl      2007-07-19 
18:16:27 UTC (rev 2554)
@@ -2,8 +2,8 @@
 # -*- perl -*-
 #-------------------------------------------------------------------------
 #         former dump-targets script converted to create perfSONAR 
-#            MA XML configuration files
-#             Maxim Grigoriev, 
,
  03.2006
+#             XML configuration files
+#                  Maxim Grigoriev, 
,
  09.2006
 #
 # See embedded POD below for further information
 #
@@ -25,11 +25,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#######################################################################
-#
-#    This script lives in cricket util directory
-#  
-### $programdir is a default value of cricket  base - location of 
cricket-conf.pl 
+
+
+
+  
+### ddefault value of cricket  base - location of cricket-conf.pl
 BEGIN {
         my $programdir = (($0 =~ m:^(.*/):)[0] || "./") . "..";
         eval "require '$programdir/cricket-conf.pl'";
@@ -37,50 +37,44 @@
                                         unless $Common::global::gInstallRoot;
         $Common::global::gInstallRoot ||= $programdir;
 }
-########
-
 use lib "$Common::global::gInstallRoot/lib";
 
 use strict; 
 use Getopt::Long;
-use Net::Netmask; 
-use POSIX qw(strftime);
-use Sys::Hostname;
-use Pod::Usage;
-
-my $help = ''; 
-#-------------------- Default values for command line options
+ 
 my $fname = '';
+my $help = '';
 my $dss = 'ifInOctets,ifOutOctets';
 my $nets = '';
 my $domain = "Fermilab";
 my $ctree = '/';
 my $vvv = '';
-my %hopts= (  "ctree" => \$ctree, 
-             "help" => \$help, 
-             "fname" => \$fname, 
+
+GetOptions (   
+             "ctree=s" => \$ctree, 
+             "help|?|h" => \$help, 
+             "fname=s" => \$fname, 
             "verbose" => \$vvv, 
-            "domain" => \$domain,
-            "dss" => \$dss, 
-            "nets" => \$nets );
-GetOptions ( \%hopts,  "ctree=s", 
"help|?|h","fname=s","verbose","domain=s","dss=s", "nets=s" ) or 
pod2usage(2); 
-pod2usage(1) if $help; 
+            "domain=s" => \$domain,
+            "dss=s" => \$dss, 
+            "nets=s" => \$nets) or pod2usage(2) ; 
+       
  
-
-#---------------------- packages needed for parsing Cricket config tree  
-use Common::global;
+use Net::Netmask; 
+use Pod::Usage;
 use Common::Log;
+use Common::global;
 use ConfigTree::Cache;
-
-Common::Log::setLevel('warn');                  
+                
 $Common::global::gCT = new ConfigTree::Cache;
 my $gCT = $Common::global::gCT;
 $gCT->Base($Common::global::gConfigRoot);
- 
-#------------------------------------------------------------------------
- 
+$gCT->Warn(\&Warn);
 
-###### storage hash with CIDRs as keys and Net objects  
+ 
+pod2usage(1) if $help;
+pod2usage(2)  unless $fname;
+###### create hash with CIDRs as keys and Net objects  
 my $subnets = {};
 
 if($nets) {
@@ -95,75 +89,59 @@
     Die("Failed to open compiled config tree from " .
                "$Common::global::gConfigRoot/config.db: $!");
 }
-my $now = strftime "%Y-%m-%d %H:%M:%S",localtime();
-my $opts_string = $0; 
-foreach my $k ( keys %hopts) {
- $opts_string .= " $k=" . ${$hopts{$k}} if ${$hopts{$k}} ;
-}
-my $host = hostname;
 
-#-------------------------- MA config header 
----------------------------------------
+
 my $header =  qq(<?xml version="1.0" encoding="UTF-8"?>
- 
 
+
 <!-- ===================================================================
 <description>
    MA RRD configuration file
 
-   Template \$Id$fname,v 1.10 2005/10/25 05:39:08 zurawski Exp \$
+   \$Id$fname,v 1.10 2005/10/25 05:39:08 zurawski Exp \$
    project: perfSONAR
-   Created: $now by  $opts_string  
-   On: host=$host  OS=$^O
+
 Notes:
    This is the configuration file which contains the information 
-   about RRD files 
-   
+   about RRD files from  $domain
 
 </description>
 ==================================================================== -->
 
-<perfsonar:store xmlns="http://ggf.org/ns/nmwg/tools/org/perfsonar/1.0/"; 
-                 
xmlns:perfsonar="http://ggf.org/ns/nmwg/tools/org/perfsonar/1.0/"; 
+<nmwg:store xmlns="http://ggf.org/ns/nmwg/base/2.0/"; 
                  xmlns:nmwg="http://ggf.org/ns/nmwg/base/2.0/"; 
-                 xmlns:nmtm="http://ggf.org/ns/nmwg/time/2.0/"; 
-                 xmlns:nmwgt="http://ggf.org/ns/nmwg/topology/2.0/";>
+                 
xmlns:netutil="http://ggf.org/ns/nmwg/characteristic/utilization/2.0/"; 
+                 xmlns:nmwgt="http://ggf.org/ns/nmwg/topology/2.0/";> 
                 
                 <!--  metadata section  -->
 );
-#---------------------------------------------------------------------- 
+ 
 my $targetCnt = 1;
-my %dss_check = map {$_, 1} split ",", $dss; ## converting DataSource string 
into hash for later check
+my %dss_check = map {$_, 1} split ",", $dss; ## converting string into hash 
for later check
     
 my @ctrees =   split ",", $ctree; 
 my %dss_h = ();
-foreach my $subtree (@ctrees) {                   # over every tree supplied
+foreach my $subtree (@ctrees) {
     if ($gCT->nodeExists( $subtree )) {
-        $gCT->visitLeafs($subtree, \&makePerfSONAR ); # parsing the tree
+        $gCT->visitLeafs($subtree, \&makePerfSONAR );
     } else {
         Warn("Unknown subtree: $subtree.");
     }
 }
-&printPerfSONAR($header, $fname );
+printPerfSONAR($header, $fname );
 exit;
 
-
-
 ####################################
 #
-#     print MA XML into file or STDOUT
-#     input:  $header, $fname - header of the XML file, name of the 
resulting XML file ( if missed then its STDOUT)
-#     return: nothing
-#     global variables: 
-#               %dss_h - hash with parsed cricket tree leafes
-#               $nets - string with subnets 
-#               $subnets - reference to hash of Netblocks ( subnets )
+#     print XML into file or STDOUT
 #
+#
 sub printPerfSONAR {
    my($header, $fname) = @_;
    if($fname) {
        open OUT, ">$fname" or die " Could not open the file: $fname $! "; 
    } else {
-      *OUT = *STDOUT;
+    *OUT = *STDOUT;
    }
    print OUT  $header;
    my $cnt = 1;
@@ -173,61 +151,61 @@
        my   $host = $dss_h{$rrd_file}{$rrd_ds_name}{host};
        my   $ipaddr =  $dss_h{$rrd_file}{$rrd_ds_name}{ipv4} ;
        
-       next unless($ipaddr and $host);
+        unless($ipaddr and $host) {Warn("Unknown IP=$ipaddr or 
Hostname=$host");next;}
  
-       next if( $nets && !(findNetblock($ipaddr, [$subnets])));
+        next if( $nets && !(findNetblock($ipaddr, [$subnets])));
        
        my   $direction  = lc($rrd_ds_name) ;
-            $direction =~ s/^if(in|out).+/$1/;                    #  
identifying direction by  the DataSource name
+            $direction =~ s/^if(in|out).+/$1/;
        my   $descr = $dss_h{$rrd_file}{$rrd_ds_name}{descr};
-            $descr =~ s/\<\/?\w+\>//g;
-           $descr = "N/A" unless $descr;           
+            $descr = "N/A" unless $descr;
        my   $interface = $dss_h{$rrd_file}{$rrd_ds_name}{interface};
        my   $rrd_ds= $dss_h{$rrd_file}{$rrd_ds_name}{rrd_ds};
        my   $rrd_max = $dss_h{$rrd_file}{$rrd_ds_name}{rrd_max};    
        print OUT qq(
-       <metadata id="meta$cnt">
-        <subject id="subj$cnt">
-               <nmwgt:interface>
+       <nmwg:metadata  xmlns:nmwg="http://ggf.org/ns/nmwg/base/2.0/"; 
id="meta$cnt">
+        <netutil:subject  
xmlns:netutil="http://ggf.org/ns/nmwg/characteristic/utilization/2.0/"; 
id="subj$cnt">
+               <nmwgt:interface 
xmlns:nmwgt="http://ggf.org/ns/nmwg/topology/2.0/";> 
+                 <nmwgt:hostName>$host</nmwgt:hostName>
                    <nmwgt:ifAddress type="ipv4">$ipaddr</nmwgt:ifAddress>
-                   <nmwgt:hostName>$host</nmwgt:hostName>
                    <nmwgt:ifName>$interface</nmwgt:ifName>
                    <nmwgt:ifDescription>$descr</nmwgt:ifDescription>
-                   <nmwgt:direction>$direction</nmwgt:direction>
-                   <nmwgt:capacity>$rrd_max</nmwgt:capacity>                
-                   <nmwgt:authRealm>$domain</nmwgt:authRealm>
+                  <nmwgt:direction>$direction</nmwgt:direction>\n);
+          $rrd_max?print OUT "                   
<nmwgt:capacity>$rrd_max</nmwgt:capacity>\n":
+           $interface =~ /tengigabit/i?print OUT "                   
<nmwgt:capacity>10000000000</nmwgt:capacity>\n":
+             $interface =~ /gigabit/i?print OUT "                   
<nmwgt:capacity>1000000000</nmwgt:capacity>\n":print;              
+       print OUT qq(                   
<nmwgt:authRealm>$domain</nmwgt:authRealm>
                </nmwgt:interface>
-             </subject>
-           <nmwg:characteristic>utilization</nmwg:characteristic>
-
-       </metadata>
-         <perfsonar:data id="data$cnt" metadataIdRef="meta$cnt">
+             </netutil:subject>  
+            
<nmwg:eventType>http://ggf.org/ns/nmwg/characteristic/utilization/2.0</nmwg:eventType>
 
+            <nmwg:parameters>
+                   <nmwg:parameter 
name="supportedEventType">http://ggf.org/ns/nmwg/characteristic/utilization/2.0</nmwg:parameter>
+          </nmwg:parameters>
+       </nmwg:metadata>
+         <nmwg:data xmlns:nmwg="http://ggf.org/ns/nmwg/base/2.0/"; 
id="data$cnt" metadataIdRef="meta$cnt">
            <nmwg:key>
-               <nmwg:parameters>
+               <nmwg:parameters> 
+                  <nmwg:parameter name="type">rrd</nmwg:parameter>
+                   <nmwg:parameter name="valueUnits">Bps</nmwg:parameter>
                    <nmwg:parameter name="file">$rrd_file</nmwg:parameter>
                    <nmwg:parameter name="dataSource">$rrd_ds</nmwg:parameter>
+                  <nmwg:parameter name="unit">bps</nmwg:parameter> 
                </nmwg:parameters>
            </nmwg:key>
-       </perfsonar:data>
+       </nmwg:data>
 
        );
     $cnt++;     
    } 
     
    }
-   print OUT "</perfsonar:store>\n";
+   print OUT "</nmwg:store>\n";
    close OUT;
 }
-##################################################
+#########
 #
-#  callback to parse  every Cricket  config tree
-#   input:  $name - name of the subtree
-#   return: nothing
-#     global variables: 
-#               %dss_h - resulted hash with parsed cricket tree leafes
-#               %dss_check -    hash
-#               $vvv - verbose flag
-#    
+#  callback for every config tree
+#
 sub makePerfSONAR {
     my($name) = @_;
 
@@ -244,23 +222,34 @@
                                            lc($target->{'target-type'}),
                                           $target );
        print  " $type dictionary for $tname\n" if $vvv;
-       next unless $tmp_dict ->{'ds'};
         my($Counter) = 0;
         my(%dsMap) = map { $_ => $Counter++ } split(/\s*,\s*/,$tmp_dict 
->{'ds'});
             
-    
-        (my $rrd_fl = $target->{'rrd-datafile'}) =~ 
s/\/([^\/]+\/\.\.\/|\/)/\//g;
-        
+     #    foreach my $k (sort (keys(%{$tmp_dict}))) {
+     #   my $v = $tmp_dict->{$k};
+     #   print "\t$k = $v\n" if $vvv;
+     #    } 
+         (my $rrd_fl = $target->{'rrd-datafile'}) =~ 
s/\/([^\/]+\/\.\.\/|\/)/\//g;
+        my %router_names = ();
        foreach my $k (sort  (keys(%dsMap))) {
                 my $v = $dsMap{$k};
                 if($dss_check{$k}) {
                  $dss_h{$rrd_fl}{$k}{host} =  $target->{'router'};
-                 $dss_h{$rrd_fl}{$k}{ipv4} =  $target->{'ip'};
+                 $dss_h{$rrd_fl}{$k}{ipv4} =  $target->{'ip'};  
+                 unless($target->{'ip'}) {
+                    if($router_names{$target->{'router'}}) {
+                        $dss_h{$rrd_fl}{$k}{ipv4} = 
$router_names{$target->{'router'}};
+                    } else {
+                        $dss_h{$rrd_fl}{$k}{ipv4} = 
get_nameip($target->{'router'});
+                        $router_names{$target->{'router'}} = 
$dss_h{$rrd_fl}{$k}{ipv4};
+                   }
+                 } 
                  $dss_h{$rrd_fl}{$k}{rrd_ds} =   "ds$v";
                  $dss_h{$rrd_fl}{$k}{interface} =  $tname;
-                 $dss_h{$rrd_fl}{$k}{rrd_max} =   $target->{'rrd-max'};
-                
-                 $dss_h{$rrd_fl}{$k}{descr} = 
($target->{'short-desc'})?$target->{'short-desc'}:$target->{'long-desc'};
+                 $dss_h{$rrd_fl}{$k}{rrd_max} =  $target->{'rrd-max'};
+                  
+                 
+                 $dss_h{$rrd_fl}{$k}{descr} = $target->{'short-desc'};
                  }
                   
                print "\t$k = $v\n" if $vvv;
@@ -271,6 +260,16 @@
        print "\n" if $vvv;
 
 }
+
+sub get_nameip {
+ my $ipname = shift;
+  my $s2_ip = qx($Common::global::gInstallRoot/util/rname.sh  $ipname);
+  chomp $s2_ip;
+  return  undef if($s2_ip eq "NODNS");
+  return $s2_ip;
+ 
+}
+
 __END__
 
 
@@ -313,7 +312,7 @@
 
  
  
-=item B<--ctree>=F<CVS LIST>
+ =item B<--ctree>=F<CVS LIST>
 
  Comma separated list of   Cricket config trees ( example: 
'/switches,/routers' )
  DEFAULT: /  
@@ -323,7 +322,7 @@
  Full filename of the resulting perfSONAR config file.
  DEFAULT: all will be printed to STDOUT
  
-=item B<--dss>=F<CSV LIST>
+=item <--dss>=F<CSV LIST>
 
  Comma separated list of datasources to look for in the Cricket 
targets.Example: 'ifInOctets,ifOutOctets'
  DEFAULT: Accept all supported datasources ( 'ifInOctets,ifOutOctets' only 
for now)