perfsonar development work

Author: mac
Date: 2008-07-03 10:19:03 -0400 (Thu, 03 Jul 2008)
New Revision: 4116

Modified:
   
trunk/perfsonar_base/src/main/java/org/perfsonar/base/auxiliary/components/simplescheduler/Scheduler.java
   
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSRegistrationComponent.java
   
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
Log:
Several changes.
LSRegistrationComponent fixes. Now it takes interval time in seconds instead 
of millis.
LSSummaryServiceComponent: new xqueries, XML RPC supported, some bug fixed
Some other minor changes in other classes

Modified: 
trunk/perfsonar_base/src/main/java/org/perfsonar/base/auxiliary/components/simplescheduler/Scheduler.java
===================================================================
--- 
trunk/perfsonar_base/src/main/java/org/perfsonar/base/auxiliary/components/simplescheduler/Scheduler.java
   2008-07-03 14:18:15 UTC (rev 4115)
+++ 
trunk/perfsonar_base/src/main/java/org/perfsonar/base/auxiliary/components/simplescheduler/Scheduler.java
   2008-07-03 14:19:03 UTC (rev 4116)
@@ -42,7 +42,7 @@
     
     private boolean paused = false;
     
-    private int wakeupInterval = 1000; //milliseconds
+    private int wakeupInterval = 10000; //milliseconds
     
     private Set<SchedulerTask> tasks = Collections.synchronizedSet(new 
HashSet<SchedulerTask>());
     

Modified: 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSRegistrationComponent.java
===================================================================
--- 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSRegistrationComponent.java
     2008-07-03 14:18:15 UTC (rev 4115)
+++ 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSRegistrationComponent.java
     2008-07-03 14:19:03 UTC (rev 4116)
@@ -28,9 +28,9 @@
 
     // ----------------------------------------------------- constants
 
-    private static final int DEFAULT_RUN_INTERVAL = 1800000;
+    private static final int DEFAULT_RUN_INTERVAL = 1800000; //ms
 
-    private static final int MINIMAL_RUN_INTERVAL = 300000;
+    private static final int MINIMAL_RUN_INTERVAL = 300000; //ms
 
     /**
      * Posible values: <empty> "xmldb" "context" or "file" 
@@ -241,7 +241,7 @@
             contentType = configuration
                     
.getProperty(LSRegistrationComponent.SERVICE_CONTENT_TYPE);
         } catch (PerfSONARException e) {
-            logger.debug("service.ls.service_content_type could not be 
read.");
+            logger.debug("LSRegistrationComponent: 
service.ls.service_content_type could not be read.");
         }
         if (contentType == null || contentType.trim().equals("")) {
             return null;
@@ -256,7 +256,10 @@
     }
 
 
-    // ---------------------- component that loads action to scheduler
+    /**
+     * Inits compontnt
+     * Takes interval in seconds from config file 
+     */
     public void initComponent() throws PerfSONARException {
 
         // Get configuration
@@ -264,9 +267,10 @@
         int interval = LSRegistrationComponent.DEFAULT_RUN_INTERVAL;
 
         try {
+            //get interval in seconds
             String intervalVal = configuration.getProperty("component."
-                    + componentName + ".interval");
-            interval = Integer.parseInt(intervalVal);
+                    + componentName + ".interval"); 
+            interval = Integer.parseInt(intervalVal) * 1000; //multiply to 
have it in ms
         } catch (RuntimeException e) {
             // possibly not an integer - keep default interval
         } catch (PerfSONARException e) {
@@ -275,18 +279,21 @@
 
         // if too small take default?
         if (interval < LSRegistrationComponent.MINIMAL_RUN_INTERVAL) {
-            interval = LSRegistrationComponent.DEFAULT_RUN_INTERVAL;
+
+            logger.debug("LSRegistrationComponent.initComponent: parameter 
RUN_INTERVAL="
+                    + interval+" smaller than 
MINIMAL="+LSRegistrationComponent.MINIMAL_RUN_INTERVAL);
+            
+            interval = LSRegistrationComponent.MINIMAL_RUN_INTERVAL;
         }
 
-        logger
-                .debug("LSRegistrationComponent.initComponent: 
LSCleanupLoader: parameter RUN_INTERVAL = "
+        logger.debug("LSRegistrationComponent.initComponent: 
LSRegistrationComponent: parameter RUN_INTERVAL = "
                         + interval);
 
         // get scheduler - parameter "component.CName.scheduler_component"
         try {
             String schedulerVal = configuration.getProperty("component."
                     + componentName + ".scheduler_component");
-            logger.debug("LSRegistrationComponent.initComponent: 
LSCleanupLoader: parameter SCHEDULER = "
+            logger.debug("LSRegistrationComponent.initComponent: 
LSRegistrationComponent: parameter SCHEDULER = "
                             + schedulerVal);
             scheduler = (Scheduler) AuxiliaryComponentManager.getInstance()
                     .getComponent(schedulerVal);

Modified: 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
===================================================================
--- 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
     2008-07-03 14:18:15 UTC (rev 4115)
+++ 
trunk/perfsonar_base/src/main/java/org/perfsonar/service/base/registration/LSSummaryServiceContent.java
     2008-07-03 14:19:03 UTC (rev 4116)
@@ -1,5 +1,6 @@
 package org.perfsonar.service.base.registration;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -11,8 +12,12 @@
 import 
org.perfsonar.base.auxiliary.components.configuration.ConfigurationComponent;
 import org.perfsonar.base.auxiliary.components.logger.LoggerComponent;
 import org.perfsonar.base.exceptions.PerfSONARException;
+import org.perfsonar.service.base.storage.xmldb.XmlDbStorageManager;
+import org.perfsonar.service.base.storage.xmldb.exist.ExistDbFactory;
+import 
org.perfsonar.service.base.storage.xmldb.exist.ExistDbGenericStorageManager;
 import 
org.perfsonar.service.base.storage.xmldb.exist.rest.ExistDbHttpXmlDataRepository;
 import 
org.perfsonar.service.base.storage.xmldb.exist.rest.ExistDbHttpXmlStorageManager;
+import 
org.perfsonar.service.base.storage.xmldb.exist.xmlrpc.ExistDbXmlrpcXmlDataRepository;
 
 /**
  * This class get data and summarize
@@ -21,6 +26,7 @@
  */
 public class LSSummaryServiceContent implements ServiceContent {
 
+    private static int counter = 0;
 
     private static final int IP = 100;
     private static final int DOMAIN = 200;
@@ -32,14 +38,20 @@
             " 
xmlns:nmtl3=\"http://ogf.org/schema/network/topology/l3/20070828/\"; " +
             " 
xmlns:summary=\"http://ggf.org/ns/nmwg/tools/org/perfsonar/service/lookup/summarization/2.0/\";
 ";
     
+    private static final int XMLRPC = 0;
+    private static final int HTTP = 1;
+    
     private LoggerComponent logger;
     private ConfigurationComponent config;
 
     private String dbCollection = null;
     private String dbUsername = null;
     private String dbPassword = null;
-    private ExistDbHttpXmlStorageManager storage = null;
     
+    //private ExistDbHttpXmlStorageManager storage = null;
+    //private ExistDbXmlrpcXmlStorageManager storage = null;
+    private ExistDbGenericStorageManager storage = null;
+    private int connectionType = HTTP; //XMLRPC or HTTP
     
     
     // ----------------------------------------------------------------------
@@ -71,30 +83,91 @@
     }
 
     /**
+     * Translate to HTTP uri if necessary
+     */
+    private String getXMLRPCUri(String uri) {
+
+        if (uri.startsWith("http://";)) {
+            uri = uri.replace("http://";, "xmldb:exist://");
+            uri = uri.replace("/exist/rest/", "/exist/xmlrpc/");
+        }
+        return uri;
+    }    
+    
+    /**
      * Initializes Database access (from configuration)
      * @throws PerfSONARException
      */
     private void init() {
+
+        //wait 60 sec; to avoid first failed registration (when service is 
+        //starting)
+
+        long time = 60000;
+        if (LSSummaryServiceContent.counter==0) 
         
+            logger.debug("LSSummaryServiceContent: Wait ["+
+                    (time/1000)+
+                    "] seconds until other components are initialized...");
+        
+            synchronized (this) {
+                try {
+                    Thread.sleep(60000);
+                } catch (InterruptedException e) {
+    
+                }
+            }
+
+        LSSummaryServiceContent.counter++;
+        
+        // ----
+        
         try {
             //get data from configuration        
-            dbCollection = 
getHTTPUri(config.getProperty("service.ls.db_uri"));
+            switch (connectionType) {
+                case HTTP:
+                    logger.debug("Connection type for 
LSSUmmaryServiceContent is HTTP");
+                    dbCollection = 
getHTTPUri(config.getProperty("service.ls.db_uri"));
+                    break;
+                case XMLRPC:
+                    logger.debug("Connection type for 
LSSUmmaryServiceContent is XMLRPC");
+                    dbCollection = 
getXMLRPCUri(config.getProperty("service.ls.db_uri"));
+                    break;
+            }
+            
             dbUsername   = config.getProperty("service.ls.db_username");
             dbPassword   = config.getProperty("service.ls.db_password");
+
             if (dbCollection == null || dbUsername == null || dbPassword == 
null)
                 throw new PerfSONARException();
-
             logger.debug(
                     "LSSummaryServiceContent: database access details: "+
                     "(url="+dbCollection+", username="+dbUsername+", 
"+dbPassword+")"); 
+
+            switch (connectionType) {
+                case HTTP:
+                    ExistDbHttpXmlDataRepository httpDataRepository = 
+                    new ExistDbHttpXmlDataRepository(dbCollection, 
dbUsername, dbPassword);
+                    //initialize storage manager
+                    storage = new ExistDbHttpXmlStorageManager();
+                    
storage.setResultType(ExistDbHttpXmlStorageManager.STRING_ARRAY_TYPE);
+                    storage.initStorage(httpDataRepository);
+                    break;
+                case XMLRPC:
+                    //initialize data repository object - XML RPC
+                    ExistDbXmlrpcXmlDataRepository xmlrpcDataRepository
+                        = new ExistDbXmlrpcXmlDataRepository(
+                            dbCollection, dbUsername, dbPassword);
+                    storage = (ExistDbGenericStorageManager) 
+                        ExistDbFactory.getXMLStorageManager(
+                            ExistDbFactory.EXIST_XMLRPC, 
xmlrpcDataRepository);
+                    
storage.setResultType(XmlDbStorageManager.STRING_ARRAY_TYPE);
+                    
+                    logger.debug("XML RPC initialized");
+
+                    break;
+            }
             
-            //initialize data repository object
-            ExistDbHttpXmlDataRepository dataRepository = 
-                new ExistDbHttpXmlDataRepository(dbCollection, dbUsername, 
dbPassword);
-            //initialize storage manager
-            storage = new ExistDbHttpXmlStorageManager();
-            
storage.setResultType(ExistDbHttpXmlStorageManager.STRING_ARRAY_TYPE);
-            storage.initStorage(dataRepository);
             logger.debug("LSSummaryServiceContent: storage initiated");
             
         } catch (PerfSONARException e) {
@@ -112,29 +185,16 @@
     public String[] getServiceContent() throws PerfSONARException {
 
         //initialize
-        logger.info("LSSummaryServiceConetent: init");
+        logger.debug("LSSummaryServiceConetent: init");
         
         init();
         
-        //wait 10 sec;
-        
-        synchronized (this) {
-            try {
-                Thread.sleep(10000);
-            } catch (InterruptedException e) {
-
-            }
-        }
-        
         //get all topology information (IPs)
 
-        logger.info("LSSummaryServiceConetent: get services info from DB");
+        logger.debug("LSSummaryServiceConetent: get services info from DB");
 
         Map<String, ServiceSummary> summaries = getAllServicesSummaries();
         
-        //TEMPORARY:
-        showSummary(summaries);
-        
         //now having all data in "summaries" structure and need to summarize
         // IPs, domains separately. EventTypes will remain the same.
         
@@ -142,16 +202,27 @@
         logger.debug("LSSummaryServiceContent: Summarize...");
         
         
-        
-            //TODO: do summarization
-            //work on the same data - summarize and store in "summaries" 
structure
-            //
-        
-            // ...
+        //TODO: do summarization
+        //work on the same data - summarize and store in "summaries" 
structure
+        for (ServiceSummary s :summaries.values()) {
+            
+            //summarize IPs
+            //...
+           
+            //summarize domains
+            logger.debug("Summarize domains");
+            s.domains = summarizeDomains(s.domains);
 
+            //summarize eventTypes
+            //...
+            
+            //summarize keywords
+            //...
+        }
         
+        //TEMPORARY:
+        showSummary(summaries);
         
-        
         String[] resultArray = new String[summaries.size()];
 
         int i=0;
@@ -220,6 +291,11 @@
         String[] domainResults = queryDomainResults();
         String[] eventTypeResults = queryEventTypeResults();
         
+        logger.debug("LSSummaryServiceContent.getAllServicesSummaries: got 
summary from database. Number of: " );
+        try {  logger.debug("        - IP: ["+ipResults.length+"]"); } catch 
(Exception ex) {}
+        try {  logger.debug("        - domains: 
["+domainResults.length+"]"); } catch (Exception ex) {}
+        try {  logger.debug("        - eventvTypes: 
["+eventTypeResults.length+"]"); } catch (Exception ex) {}
+        
         populateSummary(summaries, ipResults, IP);
         populateSummary(summaries, domainResults, DOMAIN);
         populateSummary(summaries, eventTypeResults, EVENT_TYPE);
@@ -228,8 +304,52 @@
         
     }
     
+
+    /**
+     * Get subdomains for domain. For example for input string 
"www.man.poznan.pl"
+     * and limit=3 it will return collection of "pl", "poznan.pl" and 
"man.poznan.pl" 
+     * @param domain input domain e.g. "www.man.poznan.pl"
+     * @param limit number of subdomains to be returned. Value -1 means 
"all" 
+     * @param list collection where results are to be added
+     * @return updated Collection
+     */
+    public Collection<String> getSubDomains(String domain, int limit, 
Collection<String> list) {
         
+        if (domain==null) return null;
+        
+        int last = domain.length();
+        
+        for (int current=last-1; current>=0; current--) {
+            
+            if (limit==0) break;
+            
+            if (domain.charAt(current) == '.') { 
+                String subdomain = domain.substring(current+1);
+                if (!"".equals(subdomain))
+                    list.add(subdomain);
+                    limit--;
+            }
+        }
+        return list;
+    }    
+
     /**
+     * Do domains summarization. Returns new, summarized set of data
+     * Input data is not changed.
+     * @param data
+     * @return
+     */
+    private HashSet<String> summarizeDomains(HashSet<String> data) {
+
+        HashSet<String> summarizedData = new HashSet<String>();
+        for (String s : data) {
+            getSubDomains(s, 3, summarizedData);
+        }
+        return summarizedData;
+                
+    }
+
+    /**
      * Extracts data from lines (DB results) and put them into the right 
place
      * in summaries
      * @param summaries
@@ -269,6 +389,7 @@
                 case DOMAIN:
                     //System.out.println("       # domain: "+content);
                     summary.domains.add(content);
+                    //summary.domains.add(content);
                     break;
                 case EVENT_TYPE:
                     //System.out.println("       # evt: "+content);
@@ -312,8 +433,9 @@
         " for $d in 
/nmwg:store[@type='LSStore']/nmwg:data
 \n"+
         " let $val := $d/nmwg:metadata/*:subject//nmwgt:ifAddress/text() \n"+
         " let $metaIdRef := data($d/@metadataIdRef) \n"+
-        " return concat($metaIdRef,',',$val) \n";
+        " return if (empty($val)) then $val else concat($metaIdRef,',',$val) 
\n";
         return xquery(xq);
+        
     }
 
     /**
@@ -328,7 +450,7 @@
             " for $d in 
/nmwg:store[@type='LSStore']/nmwg:data
 \n"+
             " let $val := $d/nmwg:metadata/*:subject//nmwgt:hostName/text() 
\n"+
             " let $metaIdRef := data($d/@metadataIdRef) \n"+
-            " return concat($metaIdRef,',',$val) \n";
+            " return if (empty($val)) then $val else 
concat($metaIdRef,',',$val) \n";
         return xquery(xq);
     }    
 
@@ -344,7 +466,7 @@
             " for $d in 
/nmwg:store[@type='LSStore']/nmwg:data
 \n"+
             " let $val := $d/nmwg:metadata/nmwg:eventType \n"+
             " let $metaIdRef := data($d/@metadataIdRef) \n"+
-            " return concat($metaIdRef,',',$val) \n";        
+            " return if (empty($val)) then $val else 
concat($metaIdRef,',',$val) \n";        
         return xquery(xq);
     }        
 
@@ -359,7 +481,6 @@
         }
     }
     
-
     
     // ----------------------------------------------------------------------