perfsonar development work

Author: michael.bischoff
Date: 2009-11-24 07:42:34 -0500 (Tue, 24 Nov 2009)
New Revision: 5354

Modified:
   trunk/ps-mdm-flowsa-ma/.classpath
   trunk/ps-mdm-flowsa-ma/.project
   trunk/ps-mdm-flowsa-ma/conf/axis-1.4/WEB-INF/web.xml
   trunk/ps-mdm-flowsa-ma/conf/service.properties
   trunk/ps-mdm-flowsa-ma/pom.xml
   
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaGetRequest.java
   
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaMAServiceEngine.java
   
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/Exporter.java
   
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/NfcapdControll.java
Log:
fixed handling Z timezone (needs verify)
fixed handling : in timezone (needs verify)
Improved information provided with errors


Modified: trunk/ps-mdm-flowsa-ma/.classpath
===================================================================
--- trunk/ps-mdm-flowsa-ma/.classpath   2009-11-18 09:57:12 UTC (rev 5353)
+++ trunk/ps-mdm-flowsa-ma/.classpath   2009-11-24 12:42:34 UTC (rev 5354)
@@ -1,13 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
-       <classpathentry including="**/*.java" kind="src" 
output="target/test-classes" path="src/test/java"/>
-       <classpathentry including="**/*.java" kind="src" 
path="src/main/java"/>
-       <classpathentry kind="con" 
path="org.devzuz.q.maven.jdt.core.mavenClasspathContainer">
+       <classpathentry kind="src" output="bin" path="src/main/java"/>
+       <classpathentry kind="src" output="build/test-classes" 
path="src/test/java"/>
+       <classpathentry kind="con" 
path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
+       <classpathentry kind="con" 
path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER">
                <attributes>
                        <attribute 
name="org.eclipse.jst.component.dependency" value="/WEB-INF/lib"/>
                </attributes>
        </classpathentry>
-       <classpathentry kind="con" 
path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-       <classpathentry kind="con" 
path="org.eclipse.jdt.junit.JUNIT_CONTAINER/3"/>
        <classpathentry kind="output" path="bin"/>
 </classpath>

Modified: trunk/ps-mdm-flowsa-ma/.project
===================================================================
--- trunk/ps-mdm-flowsa-ma/.project     2009-11-18 09:57:12 UTC (rev 5353)
+++ trunk/ps-mdm-flowsa-ma/.project     2009-11-24 12:42:34 UTC (rev 5354)
@@ -6,30 +6,40 @@
        </projects>
        <buildSpec>
                <buildCommand>
-                       <name>org.eclipse.jdt.core.javabuilder</name>
+                       
<name>org.eclipse.wst.jsdt.core.javascriptValidator</name>
                        <arguments>
                        </arguments>
                </buildCommand>
                <buildCommand>
-                       <name>org.maven.ide.eclipse.maven2Builder</name>
+                       
<name>org.eclipse.wst.common.project.facet.core.builder</name>
                        <arguments>
                        </arguments>
                </buildCommand>
                <buildCommand>
-                       
<name>org.devzuz.q.maven.jdt.core.mavenIncrementalBuilder</name>
+                       <name>org.eclipse.jdt.core.javabuilder</name>
                        <arguments>
                        </arguments>
                </buildCommand>
                <buildCommand>
-                       
<name>edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder</name>
+                       
<name>org.eclipse.wst.validation.validationbuilder</name>
                        <arguments>
                        </arguments>
                </buildCommand>
+               <buildCommand>
+                       <name>org.maven.ide.eclipse.maven2Builder</name>
+                       <arguments>
+                       </arguments>
+               </buildCommand>
        </buildSpec>
        <natures>
+               <nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
+               
<nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
+               <nature>org.maven.ide.eclipse.maven2Nature</nature>
                <nature>org.devzuz.q.maven.jdt.core.mavenNature</nature>
-               <nature>org.maven.ide.eclipse.maven2Nature</nature>
+               <nature>org.eclipse.iam.jdt.core.mavenNature</nature>
                <nature>org.eclipse.jdt.core.javanature</nature>
                
<nature>edu.umd.cs.findbugs.plugin.eclipse.findbugsNature</nature>
+               
<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
+               <nature>org.eclipse.wst.jsdt.core.jsNature</nature>
        </natures>
 </projectDescription>

Modified: trunk/ps-mdm-flowsa-ma/conf/axis-1.4/WEB-INF/web.xml
===================================================================
--- trunk/ps-mdm-flowsa-ma/conf/axis-1.4/WEB-INF/web.xml        2009-11-18 
09:57:12 UTC (rev 5353)
+++ trunk/ps-mdm-flowsa-ma/conf/axis-1.4/WEB-INF/web.xml        2009-11-24 
12:42:34 UTC (rev 5354)
@@ -6,7 +6,7 @@
 <web-app>
        <display-name>perfSONAR - Flow Selection &amp; Aggregation 
MA</display-name>
 
-       <filter>
+       <!--  <filter>
                <filter-name>ipfilter</filter-name>
                <filter-class>nl.surfnet.web.IpAccessFilter</filter-class>
                <init-param>
@@ -27,7 +27,7 @@
        <filter-mapping>
                <filter-name>ipfilter</filter-name>
                <url-pattern>/services/*</url-pattern>
-       </filter-mapping>
+       </filter-mapping> -->
 
        <listener>
                <listener-class>

Modified: trunk/ps-mdm-flowsa-ma/conf/service.properties
===================================================================
--- trunk/ps-mdm-flowsa-ma/conf/service.properties      2009-11-18 09:57:12 
UTC (rev 5353)
+++ trunk/ps-mdm-flowsa-ma/conf/service.properties      2009-11-24 12:42:34 
UTC (rev 5354)
@@ -118,7 +118,7 @@
 
 # Group=Query properties
 # Description= if more then <value> results are encountered stop processing
-service.ma.flowsa.query.max-results=1000
+service.ma.flowsa.query.max-results=5000
 
 # --- nfdump properties ---
 # nfdump.source - location of nfcapd files

Modified: trunk/ps-mdm-flowsa-ma/pom.xml
===================================================================
--- trunk/ps-mdm-flowsa-ma/pom.xml      2009-11-18 09:57:12 UTC (rev 5353)
+++ trunk/ps-mdm-flowsa-ma/pom.xml      2009-11-24 12:42:34 UTC (rev 5354)
@@ -7,8 +7,7 @@
        <name>Flow Selection and Aggregation Measurement Archive</name>
        <version>1.0.5</version>
        <description>
-               Flow selection and aggregation Measurement Archive for
-               perfSONAR. For more information see the wiki-page.
+               Flow selection and aggregation Measurement Archive for 
perfSONAR. For more information see the wiki-page.
        </description>
        <url>
                
http://wiki.perfsonar.net/jra1-wiki/index.php/Flow_Selection_and_Aggregation_MA

Modified: 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaGetRequest.java
===================================================================
--- 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaGetRequest.java
  2009-11-18 09:57:12 UTC (rev 5353)
+++ 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaGetRequest.java
  2009-11-24 12:42:34 UTC (rev 5354)
@@ -107,9 +107,9 @@
                        if (flowSubject.getNetwork() != null) {
                                this.nodes = 
flowSubject.getNetwork().getNodeArray();
                        }
-               } else {
-                       logger.error("Wrong subject type encountered.");
-               }
+               } 
+               if(!(subject instanceof 
org.ggf.ns.nmwg.ops.select.v2_0.Subject)) return;
+               logger.warn("unexpected subject type encountered.");
        }
 
        /**
@@ -148,38 +148,37 @@
        /**
         * returns the startTime contained in this request
         * @return StartTime
+        * @throws PerfSONARException 
         */
-       public Date getStartTime() {
+       public Date getStartTime() throws PerfSONARException {
                if (startTime == null) {
                        String value = parameters.get("starttime");
                        if(value==null) {
                            logger.error("no startTime specified.");
                            return null;
                        }
-                       DateFormat dateFormat = new 
SimpleDateFormat(ISO8601_PATTERN);
                        try {
-                               startTime = dateFormat.parse(value);
+                               startTime = parseISODate(value);
                        } catch (ParseException e) {
                                logger.error("Could not parse DateTime. 
(startTime)");
-                               startTime = null;
+                               throw new 
PerfSONARException("error.ma.fetching","Could not parse DateTime. (startTime 
supplied: '"+value+"')");
                        }
                }
                return startTime == null ? null : new 
Date(startTime.getTime());
        }
 
-       public Date getEndTime() {
+       public Date getEndTime() throws PerfSONARException {
                if (endTime == null) {
                        String value = parameters.get("endtime");
                 if(value==null) {
                     logger.error("no endTime specified.");
                     return null;
                 }
-                       DateFormat dateFormat = new 
SimpleDateFormat(ISO8601_PATTERN);
                        try {
-                               endTime = dateFormat.parse(value);
+                               endTime = parseISODate(value);
                        } catch (ParseException e) {
-                               logger.error("Could not parse DateTime. 
(endTime)");
-                               endTime = null;
+                               logger.error("Could not parse DateTime. 
(endTime supplied: '"+value+"')");
+                               throw new 
PerfSONARException("error.ma.fetching", "Could not parse DateTime. (endTime 
supplied: '"+value+"')");
                        }
                }
                return endTime == null ? null : new Date(endTime.getTime());
@@ -298,4 +297,41 @@
         }
         return set;
     }
+    
+       private Date parseISODate(String value) throws ParseException, 
PerfSONARException {
+               DateFormat dateFormat = new SimpleDateFormat(ISO8601_PATTERN);
+               value = value.trim();
+               if(!replaceZAsTimeZone(value)) stripColonFromTimeZone(value);
+               return dateFormat.parse(value);
+       }
+       
+       private boolean replaceZAsTimeZone(String value) {
+               if(value.endsWith("Z")) {
+                       value = value.substring(0, value.length()-1) + 
"+0000";
+                       return true;
+               }
+               return false;
+       }
+
+       private boolean stripColonFromTimeZone(String value) {
+               if(value.length() > 5 && value.charAt(value.length()-3) == 
':') {
+                       value = value.substring(0, value.length()-4) + 
value.substring(value.length()-3, value.length()-1);
+                       return true;
+               }
+               return false;
+       }
+       
+       public void setStartTime(Date startTime) {
+               for (Metadata metadata : request.getMetadataArray()) {
+                       for(Parameter parameter : 
metadata.getParameters().getParameterArray()) {
+                               String name = 
parameter.getParameterName().toLowerCase();
+                               if ("starttime".equals(name)) {
+                                       parameter.setParameterValue(new 
SimpleDateFormat(ISO8601_PATTERN).format(startTime));
+                                       processTimeParameter(parameter);
+                                       break;
+                               }
+                       }
+               }               
+               this.startTime = startTime;
+       }
 }

Modified: 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaMAServiceEngine.java
===================================================================
--- 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaMAServiceEngine.java
     2009-11-18 09:57:12 UTC (rev 5353)
+++ 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/FlowsaMAServiceEngine.java
     2009-11-24 12:42:34 UTC (rev 5354)
@@ -23,6 +23,7 @@
 import org.perfsonar.service.base.engine.ServiceEngine;
 import org.perfsonar.service.measurementArchive.flowsa.nfcapd.Exporter;
 import org.perfsonar.service.measurementArchive.flowsa.nfcapd.NfcapdControll;
+import 
org.perfsonar.service.measurementArchive.flowsa.nfcapd.StartDateBeforeFirstCaptureException;
 import 
org.perfsonar.service.measurementArchive.flowsa.nfcapd.NfcapdControll.FileData;
 import org.perfsonar.service.measurementArchive.flowsa.nfdump.Flow;
 import 
org.perfsonar.service.measurementArchive.flowsa.nfdump.IllegalAggregationRuleException;
@@ -342,14 +343,22 @@
                if(startTime==null || endTime==null) throw new 
PerfSONARException("error.ma.fetching","Time range not specified correctly 
startTime or EndTime is absent.");
                if(!startTime.before(endTime)) throw new 
PerfSONARException("error.ma.fetching","Specified StartTime isn't before 
EndTime");
                FileData fileData;
-               try {                   
-                       fileData = nfcapd.getFileData(hostNames, startTime, 
endTime);
+               try {
+                       try {
+                               fileData = nfcapd.getFileData(hostNames, 
startTime, endTime);
+                       } catch (StartDateBeforeFirstCaptureException e) {
+                               request.setStartTime(e.getFirstCaptureDate());
+                               fileData = nfcapd.getFileData(hostNames, 
e.getFirstCaptureDate(), endTime);
+                       } 
                } catch (FileNotFoundException e) {
                        logger.error(e.getLocalizedMessage());
                        throw new PerfSONARException("error.ma.query","No 
data avail for specified range");
                } catch (IOException e) {
                        logger.error(e.getLocalizedMessage());
                        throw new PerfSONARException("error.ma.query","Query 
contained a subject not known to this MA");
+               } catch (StartDateBeforeFirstCaptureException e) {
+                       logger.error("Setting startDate to firstCaptureDate 
failed");
+                       throw new 
PerfSONARException("error.ma.query","Setting startDate to firstCaptureDate 
failed");
                }
                return fileData;
        }

Modified: 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/Exporter.java
===================================================================
--- 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/Exporter.java
   2009-11-18 09:57:12 UTC (rev 5353)
+++ 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/Exporter.java
   2009-11-24 12:42:34 UTC (rev 5354)
@@ -1,6 +1,7 @@
 package org.perfsonar.service.measurementArchive.flowsa.nfcapd;
 
 import java.io.File;
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
@@ -123,7 +124,7 @@
                        Arrays.sort(sortedFiles);
                        for(String file : sortedFiles) {                      
          
                                if(file.compareTo(value)>0) {
-                                       return selected;
+                                       break;
                                }
                                selected = file;
                        }
@@ -252,4 +253,21 @@
        public String getLongtitude() {
                return longtitude;
        }
+
+       public Date getFirstCaptureDate() {
+               if(myDirectory.exists()) {                      
+                       String[] sortedFiles = myDirectory.list();
+                       Arrays.sort(sortedFiles);                       
+                       for(String file : sortedFiles) {                      
          
+                               if(file.startsWith("nfcapd.") && 
!file.startsWith("nfcapd.c")) {
+                                       try {
+                                               return new 
SimpleDateFormat("'nfcapd.'yyyyMMddHHmm").parse(file);
+                                       } catch (ParseException e) {
+                                               return null;
+                                       }
+                               }
+                       }
+               }
+               return null;
+       }
 }

Modified: 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/NfcapdControll.java
===================================================================
--- 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/NfcapdControll.java
     2009-11-18 09:57:12 UTC (rev 5353)
+++ 
trunk/ps-mdm-flowsa-ma/src/main/java/org/perfsonar/service/measurementArchive/flowsa/nfcapd/NfcapdControll.java
     2009-11-24 12:42:34 UTC (rev 5354)
@@ -111,8 +111,9 @@
         * @return FileData
         * @throws IOException when it cannot read from an exporter because 
it is unknown.
         * @throws FileNotFoundException when one of the source files cannot 
be found.
+        * @throws StartDateBeforeFirstCaptureException 
         */
-       public FileData getFileData(Collection<String> hostnames, Date 
startTime, Date endTime) throws IOException, FileNotFoundException {
+       public FileData getFileData(Collection<String> hostnames, Date 
startTime, Date endTime) throws IOException, FileNotFoundException, 
StartDateBeforeFirstCaptureException {
                String startFile = null;
                String endFile = null;
                Set<String> directories = new TreeSet<String>();
@@ -123,18 +124,21 @@
                        if(exporter==null) {
                                throw new IOException("Cannot find / unkown 
Exporter("+hostName+"), no data avail.");
                        }
-                       if(startFile!=null) {
+                       if(startFile!=null && endFile!=null) {
                                // TODO document: here we presume that files 
for exporters are all written at the same interval leading 
                                // to the same names among the directories 
this might cause problems
-                               if(!exporter.hasFiles(startFile, endFile)) 
throw new FileNotFoundException("Data not avail for specified interval");
+                               if(!exporter.hasFiles(startFile, endFile)) 
throw new FileNotFoundException("Data not avail for specified interval and 
exporter("+exporter.getHostName()+")");
                                directories.add(exporter.getDirectory());
                                continue;
                        }
                        startFile = exporter.getClosestFileBefore(startTime);
+                       if(startFile == null || 
!startFile.startsWith("nfcapd.")) {
+                               throw new 
StartDateBeforeFirstCaptureException("Data not avail for specified 
interval",exporter.getFirstCaptureDate());
+                       } else if(startFile.equals("nfcapd.current")) {
+                               throw new FileNotFoundException("Data not 
avail for specified intervaland exporter("+exporter.getHostName()+")");
+                       }
                        endFile = exporter.getFileContaining(endTime);
-                       if(startFile==null) {
-                               throw new FileNotFoundException("Data not 
avail for specified interval");
-                       }
+
                        directories.add(exporter.getDirectory());
                }               
                return new FileData(startFile,endFile,directories);