perfsonar development work

Author: mac
Date: 2008-05-30 08:22:32 -0400 (Fri, 30 May 2008)
New Revision: 3921

Modified:
   
trunk/perfsonar_base/src/main/java/org/perfsonar/client/base/requests/lookupService/LSRegisterRequestGenerator.java
   
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/GenericServiceLSRegistrator.java
   
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
Log:
Changes for hLS registration to gLS.

1. Data are read from local XML DB database (with using HTTP connector)
2. Dummy summarization is used for now (all IPs are used with /32 mask), all 
domain names are used
3. For hLS registration "summary" eventType is used automatically

And some other changes such as multiple URL registration (comma separated 
list of LSes)




Modified: 
trunk/perfsonar_base/src/main/java/org/perfsonar/client/base/requests/lookupService/LSRegisterRequestGenerator.java
===================================================================
--- 
trunk/perfsonar_base/src/main/java/org/perfsonar/client/base/requests/lookupService/LSRegisterRequestGenerator.java
 2008-05-29 14:14:30 UTC (rev 3920)
+++ 
trunk/perfsonar_base/src/main/java/org/perfsonar/client/base/requests/lookupService/LSRegisterRequestGenerator.java
 2008-05-30 12:22:32 UTC (rev 3921)
@@ -7,6 +7,7 @@
 
 import org.ggf.ns.nmwg.base.v2_0.Data;
 import org.ggf.ns.nmwg.base.v2_0.Element;
+import org.ggf.ns.nmwg.base.v2_0.EventType;
 import org.ggf.ns.nmwg.base.v2_0.Message;
 import org.ggf.ns.nmwg.base.v2_0.Metadata;
 import org.ggf.ns.nmwg.base.v2_0.Parameters;
@@ -35,6 +36,11 @@
 
     private static final String METADATA_ID = "serviceLookupInfo";
 
+    public final static String SERVICE_EVENT_TYPE = 
"http://ogf.org/ns/nmwg/tools/org/perfsonar/service/lookup/registration/service/2.0";;
+    public final static String SUMMARY_EVENT_TYPE = 
"http://ogf.org/ns/nmwg/tools/org/perfsonar/service/lookup/registration/summary/2.0";;
+
+    String eventType = SERVICE_EVENT_TYPE;
+    
     // ----------------------------------------------------- Instance 
variables
     
     /**
@@ -72,6 +78,12 @@
 //        data.setMetadataIdRef(METADATA_ID);
 //        message.setData(data);
 
+        if (eventType != null) {
+            EventType evt = new EventType();
+            evt.setEventType(eventType);
+            metadata.setEventType(evt);
+        }
+        
         Subject subject = new Subject();
         subject.setId("commonParameters");
         metadata.setSubject(subject);
@@ -271,6 +283,10 @@
         this.parserFile = parserFile;
     }
     
+    
+    public void setEventType(String evt) {
+        eventType = evt;
+    }
 
 //    public static void main(String[] s) throws PerfSONARException {
 //        String parserFile = 
"/home/mac/projects/perfsonar/perfsonar-trunk/src/objects.config";

Modified: 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/GenericServiceLSRegistrator.java
===================================================================
--- 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/GenericServiceLSRegistrator.java
 2008-05-29 14:14:30 UTC (rev 3920)
+++ 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/GenericServiceLSRegistrator.java
 2008-05-30 12:22:32 UTC (rev 3921)
@@ -30,6 +30,9 @@
     protected ConfigurationComponent configuration;
     protected ServiceDescription serviceDescription;
     protected ServiceContent serviceContent;
+    private boolean ls;
+    
+    
    
     // -------------------------------------------------- Constructors
     
@@ -52,7 +55,20 @@
         configuration = (ConfigurationComponent)
             AuxiliaryComponentManager.getInstance().
                 getComponent(ComponentNames.CONFIG); 
-
+        
+        //get service.r.type - test if ls
+        try {
+            String glsStr = configuration.getProperty("service.ls.gls");
+            if ("ls".equalsIgnoreCase(glsStr))
+                ls=true;
+            
+        } catch (Exception ex) {
+            ls=false;
+        }       
+        
+        
+        
+        
     }
         
     public Message register(String lsUrl) throws PerfSONARException {
@@ -74,21 +90,22 @@
         logger.debug("GenericServiceLSRegistrator: sending via Axis");
         //send request Document to url
         try {
-               logger.debug("GenericServiceLSRegistrator: Creating Axis 
Client"); 
+               logger.debug("GenericServiceLSRegistrator: Getting parser 
configuration"); 
             //return response as Message
             String parserFile = 
configuration.getProperty("service.sax_parser.config");
             File file = AuxiliaryComponentManager.lookupFile(parserFile);
             if(file!=null) {
                parserFile = file.getAbsolutePath();
             }                        
+            logger.debug("GenericServiceLSRegistrator: Creating Axis Client 
and sending request"); 
             AxisClient client = new AxisClient();
             Message responseMsg = client.sendRequest(lsUrl,message, 
parserFile);
             return responseMsg;
         } catch (ParserConfigurationException e) {
-            logger.error("GenericServiceLSRegistrator: "+e.getMessage());
+            logger.error("GenericServiceLSRegistrator: 
ParserConfigurationException: "+e.getMessage());
             throw new 
PerfSONARException("error.common.parse_error","GenericServiceLSRegistrator: 
"+e.getMessage());
         } catch (FactoryConfigurationError e) {
-            logger.error("GenericServiceLSRegistrator: "+e.getMessage());
+            logger.error("GenericServiceLSRegistrator: 
FactoryConfigurationError: "+e.getMessage());
             throw new 
PerfSONARException("error.common.parse_error","GenericServiceLSRegistrator: 
"+e.getMessage());
         } catch (Exception e) {
             logger.error("GenericServiceLSRegistrator: "+e.getMessage());
@@ -132,10 +149,19 @@
         
         LSRegisterRequestGenerator lsRegisterGenerator = new 
LSRegisterRequestGenerator();
         lsRegisterGenerator.setCommonParameters(commonParameters);
+
+        //for gLS - if service is LS it should register summary
+        if (ls) {
+            
lsRegisterGenerator.setEventType(LSRegisterRequestGenerator.SUMMARY_EVENT_TYPE);
+            logger.debug(
+                    "GenericServiceLSRegistrator: " +
+                    "send registration from hLS to gLS - eventType "+
+                    LSRegisterRequestGenerator.SUMMARY_EVENT_TYPE);
+        }
        
                if (serviceContent!=null)  {    
-                       String[] metadata = 
serviceContent.getServiceContent();
-                       lsRegisterGenerator.setMetadataElements(metadata);
+               String[] metadata = serviceContent.getServiceContent();
+               lsRegisterGenerator.setMetadataElements(metadata);
                } else {
                        //String[] metadata = new String[1];
                        //metadata[0] = "<nmwg:metadata id=\"empty\" />";

Modified: 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
===================================================================
--- 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
     2008-05-29 14:14:30 UTC (rev 3920)
+++ 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
     2008-05-30 12:22:32 UTC (rev 3921)
@@ -1,10 +1,18 @@
 package org.perfsonar.service.base.registration;
 
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
 import org.perfsonar.base.auxiliary.AuxiliaryComponentManager;
 import org.perfsonar.base.auxiliary.ComponentNames;
 import 
org.perfsonar.base.auxiliary.components.configuration.ConfigurationComponent;
 import org.perfsonar.base.auxiliary.components.logger.LoggerComponent;
 import org.perfsonar.base.exceptions.PerfSONARException;
+import 
org.perfsonar.service.base.storage.xmldb.exist.rest.ExistDbHttpXmlDataRepository;
+import 
org.perfsonar.service.base.storage.xmldb.exist.rest.ExistDbHttpXmlStorageManager;
 
 /**
  * This class get data and summarize
@@ -14,9 +22,28 @@
 public class LSSummaryServiceContent implements ServiceContent {
 
 
+    private static final int IP = 100;
+    private static final int DOMAIN = 200;
+    private static final int EVENT_TYPE = 300;
+
+    private static final String XMLNS_DEF = 
+            " xmlns:nmwg=\"http://ggf.org/ns/nmwg/base/2.0/\"; "+
+            " 
xmlns:nmtb=\"http://ogf.org/schema/network/topology/base/20070707/\"; " +
+            " 
xmlns:nmtl3=\"http://ogf.org/schema/network/topology/l3/20070707/\"; " +
+            " 
xmlns:summary=\"http://ggf.org/ns/nmwg/tools/org/perfsonar/service/lookup/summarization/2.0/\";
 ";
+    
     private LoggerComponent logger;
-    private ConfigurationComponent configuration;
+    private ConfigurationComponent config;
+
+    private String dbCollection = null;
+    private String dbUsername = null;
+    private String dbPassword = null;
+    private ExistDbHttpXmlStorageManager storage = null;
     
+    
+    
+    // ----------------------------------------------------------------------
+        
     public LSSummaryServiceContent() {
         
         try {
@@ -25,32 +52,346 @@
         } catch (PerfSONARException e) { }
         
         try {
-            this.configuration = (ConfigurationComponent) 
AuxiliaryComponentManager
+            this.config = (ConfigurationComponent) AuxiliaryComponentManager
                     .getInstance().getComponent(ComponentNames.CONFIG);
         } catch (PerfSONARException e) { }        
         
     }
     
+    /**
+     * Translate to HTTP uri if necessary
+     */
+    private String getHTTPUri(String uri) {
+
+        if (uri.startsWith("xmldb:exist://")) {
+            uri = uri.replace("xmldb:exist://", "http://";);
+            uri = uri.replace("/exist/xmlrpc/", "/exist/rest/");
+        }
+        return uri;
+    }
+
+    /**
+     * Initializes Database access (from configuration)
+     * @throws PerfSONARException
+     */
+    private void init() {
+        
+        try {
+            //get data from configuration        
+            dbCollection = 
getHTTPUri(config.getProperty("service.ls.db_uri"));
+            dbUsername   = config.getProperty("service.ls.db_username");
+            dbPassword   = config.getProperty("service.ls.db_password");
+            if (dbCollection == null || dbUsername == null || dbPassword == 
null)
+                throw new PerfSONARException();
+
+            logger.debug(
+                    "LSSummaryServiceContent: database access details: "+
+                    "(url="+dbCollection+", username="+dbUsername+", 
"+dbPassword+")"); 
+            
+            //initialize data repository object
+            ExistDbHttpXmlDataRepository dataRepository = 
+                new ExistDbHttpXmlDataRepository(dbCollection, dbUsername, 
dbPassword);
+            //initialize storage manager
+            storage = new ExistDbHttpXmlStorageManager();
+            
storage.setResultType(ExistDbHttpXmlStorageManager.STRING_ARRAY_TYPE);
+            storage.initStorage(dataRepository);
+            logger.debug("LSSummaryServiceContent: storage initiated");
+            
+        } catch (PerfSONARException e) {
+            logger.error("LSSummaryServiceContent: Wrong XML DB settings 
(url="+
+                    dbCollection+", username="+dbUsername+", "+dbPassword+
+                    ") or cannot instantiate StorageManager");
+        }
+        
+    }
     
+    
+    /**
+     * Method that is executed by Registration component, MAIN method of the 
class
+     */
     public String[] getServiceContent() throws PerfSONARException {
 
-        logger.info("LSSummaryServiceConetent STARTED");
+        //initialize
+        logger.info("LSSummaryServiceConetent: init");
         
+        init();
+        
+        //wait 10 sec;
+        
+        synchronized (this) {
+            try {
+                Thread.sleep(10000);
+            } catch (InterruptedException e) {
+
+            }
+        }
+        
         //get all topology information (IPs)
-        /*
-             declare namespace nmwg="http://ggf.org/ns/nmwg/base/2.0/";;
-             declare namespace nmwgt="http://ggf.org/ns/nmwg/topology/2.0/";;
-             
/nmwg:store[@type='LSStore']/nmwg:data/nmwg:metadata/*:subject//nmwgt:ifAddress/text()
-         */
+
+        logger.info("LSSummaryServiceConetent: get services info from DB");
+
+        Map<String, ServiceSummary> summaries = getAllServicesSummaries();
         
-        //summarize information
+        //TEMPORARY:
+        showSummary(summaries);
         
-            //build IPTrie conaining all IPs and find compressed set of 
IPs/masks
+        //now having all data in "summaries" structure and need to summarize
+        // IPs, domains separately. EventTypes will remain the same.
         
-        //pack into String[] 
-        String[] resultArray = {""};
+        //So, summarize information
+        logger.debug("LSSummaryServiceContent: Summarize...");
         
+        
+        
+            //TODO: do summarization
+            //work on the same data - summarize and store in "summaries" 
structure
+            //
+        
+            // ...
+
+        
+        
+        
+        String[] resultArray = new String[summaries.size()];
+
+        int i=0;
+        //for every single summary get XML
+        for (ServiceSummary s :summaries.values()) {
+            
+            StringBuffer sb = new StringBuffer();
+            
+            sb.append("<nmwg:metadata id=\"summmary.md.1\" ");
+            sb.append(XMLNS_DEF); //add namespace definition (important!)
+            sb.append(">\n");
+            sb.append("  <summary:subject id=\"summmary.md.1\">\n");
+
+            for (String ip : s.ipAddresses) {
+                sb.append("    <nmtl3:network>\n");
+                sb.append("      <nmtl3:subnet>\n");
+                sb.append("        <nmtl3:address 
type=\"ipv4\">"+ip+"</nmtl3:address>\n");
+                sb.append("        <nmtl3:netmask>32</nmtl3:netmask>\n");
+                sb.append("      </nmtl3:subnet>\n");
+                sb.append("    </nmtl3:network>\n");
+            }
+            
+            for (String dom : s.domains) {
+                sb.append("    <nmtb:domain>\n");
+                sb.append("      <nmtb:name 
type=\"dns\">"+dom+"</nmtb:name>\n");
+                sb.append("    </nmtb:domain>\n");
+            }
+            
+            sb.append("  </summary:subject>\n");
+            
+            sb.append("  
<nmwg:eventType>http://ggf.org/ns/nmwg/tools/org/perfsonar/service/lookup/summarization/2.0</nmwg:eventType>\n");
+
+            if (s.eventTypes.size()>0) {
+                sb.append("  <summary:parameters id=\"summmary.md.1\">\n");
+                for (String evt : s.eventTypes) {
+                    sb.append("    <nmwg:parameter name=\"eventType\" 
value=\""+evt+"\" />\n");
+                }
+                sb.append("  </summary:parameters>\n");
+            }
+            sb.append("</nmwg:metadata>\n");
+
+            //convert StringBuffer to String
+            String res = sb.toString();
+            
+            logger.debug("LSSummaryServicecontent: Summary data is:\n"+res);
+
+            //store in array
+            resultArray[i] = res;
+            i++;
+        }
+        
         return resultArray;
+        
     }
+    
+    /**
+     * Get topology data and eventTypes from all services
+     * store it in Map<String, ServiceSummary>
+     * @return
+     */
+    private Map<String, ServiceSummary> getAllServicesSummaries() {
+        
+        Map<String, ServiceSummary> summaries = new HashMap<String, 
ServiceSummary>();
 
-}
+        String[] ipResults = queryIpResults();
+        String[] domainResults = queryDomainResults();
+        String[] eventTypeResults = queryEventTypeResults();
+        
+        populateSummary(summaries, ipResults, IP);
+        populateSummary(summaries, domainResults, DOMAIN);
+        populateSummary(summaries, eventTypeResults, EVENT_TYPE);
+        
+        return summaries;    
+        
+    }
+    
+        
+    /**
+     * Extracts data from lines (DB results) and put them into the right 
place
+     * in summaries
+     * @param summaries
+     * @param lines results from DB
+     * @param type type of field to be populated
+     */
+    private void populateSummary(Map<String, ServiceSummary> summaries, 
String[] lines, int type) {
+
+        //split and store results
+        for (String line : lines) {
+            
+            //split result
+            String[] params = line.split(",",2);
+            String key = params[0];
+            String content = params[1];
+            //System.out.println(" -- parse results: "+key+" : "+content);
+            
+            //get or create summary for the service
+            ServiceSummary summary = summaries.get(key);
+            if (summary==null) { 
+                summary = new ServiceSummary();
+                summaries.put(key, summary); 
+            }
+            
+            //fill the right field
+            switch (type) {
+                case IP:
+                    //System.out.println("       # ip: "+content);
+                    summary.ipAddresses.add(content);
+                    break;
+                case DOMAIN:
+                    //System.out.println("       # domain: "+content);
+                    summary.domains.add(content);
+                    break;
+                case EVENT_TYPE:
+                    //System.out.println("       # evt: "+content);
+                    summary.eventTypes.add(content);
+                    break;
+            }
+            
+        }
+        
+    }
+
+    /**
+     * Perform XQuery with storage manager
+     * @param xq
+     * @return
+     */
+    private String[] xquery(String xq) {
+        
+        try {
+            String[] s = (String[]) storage.fetch(xq);
+            return s;
+        } catch (Exception e) {
+            logger.error("LSSummaryServiceContent: Cannot perform xquery: 
"+e.getMessage());  
+        }
+        return new String[0];
+        
+    }
+    
+    /**
+     * Return set of IP addresses used by services
+     * @return
+     */
+    private String[] queryIpResults() {
+
+        String xq = 
+        " declare namespace nmwg=\"http://ggf.org/ns/nmwg/base/2.0/\";; \n"+
+        " declare namespace nmwgt=\"http://ggf.org/ns/nmwg/topology/2.0/\";; 
\n"+
+        " for $d in 
/nmwg:store[@type='LSStore']/nmwg:data
 \n"+
+        " let $val := $d/nmwg:metadata/*:subject//nmwgt:ifAddress/text() \n"+
+        " let $metaIdRef := data($d/@metadataIdRef) \n"+
+        " return concat($metaIdRef,',',$val) \n";
+        return xquery(xq);
+    }
+
+    /**
+     * Return set of domains used by services
+     * @return
+     */
+    private String[] queryDomainResults() {
+
+        String xq = 
+            " declare namespace nmwg=\"http://ggf.org/ns/nmwg/base/2.0/\";; 
\n"+
+            " declare namespace 
nmwgt=\"http://ggf.org/ns/nmwg/topology/2.0/\";; \n"+
+            " for $d in 
/nmwg:store[@type='LSStore']/nmwg:data
 \n"+
+            " let $val := $d/nmwg:metadata/*:subject//nmwgt:hostName/text() 
\n"+
+            " let $metaIdRef := data($d/@metadataIdRef) \n"+
+            " return concat($metaIdRef,',',$val) \n";
+        return xquery(xq);
+    }    
+
+    /**
+     * Return set of eventTypes used by services
+     * @return
+     */
+    private String[] queryEventTypeResults() {
+
+        String xq = 
+            " declare namespace nmwg=\"http://ggf.org/ns/nmwg/base/2.0/\";; 
\n"+
+            " declare namespace 
nmwgt=\"http://ggf.org/ns/nmwg/topology/2.0/\";; \n"+
+            " for $d in 
/nmwg:store[@type='LSStore']/nmwg:data
 \n"+
+            " let $val := $d/nmwg:metadata/nmwg:eventType \n"+
+            " let $metaIdRef := data($d/@metadataIdRef) \n"+
+            " return concat($metaIdRef,',',$val) \n";        
+        return xquery(xq);
+    }        
+
+    
+    private void showSummary(Map<String, ServiceSummary> summaries) {
+        Iterator<Entry<String, ServiceSummary>> i = 
summaries.entrySet().iterator();
+        while (i.hasNext()) {
+            Map.Entry e = i.next();
+            logger.debug("\n==== "+e.getKey()+" ====\n");
+            logger.debug(e.getValue().toString());
+            
+        }
+    }
+    
+
+    
+    // ----------------------------------------------------------------------
+    
+    /**
+     * Contains summary from a single service
+     * @author Maciej Glowiak
+     *
+     */
+    class ServiceSummary {
+        
+        public HashSet<String> eventTypes = new HashSet<String>();
+        public HashSet<String> domains  = new HashSet<String>();
+        public HashSet<String> ipAddresses = new HashSet<String>();
+        
+        public String toString() {
+            
+            StringBuffer sb = new StringBuffer();
+            
+            sb.append("1. eventTypes:\n");
+            for (String s:eventTypes) { 
+                sb.append("  -- ");
+                sb.append(s);
+                sb.append("\n");
+            }
+            sb.append("2. domains:\n");
+            for (String s:domains) { 
+                sb.append("  -- ");
+                sb.append(s);
+                sb.append("\n");
+            }
+            sb.append("3. IPs:\n");
+            for (String s:ipAddresses) { 
+                sb.append("  -- ");
+                sb.append(s);
+                sb.append("\n");
+            }
+
+            return sb.toString();
+        }
+                
+    } //ServiceSummary
+    
+    
+}//LSSummaryServiceContent