Skip to Content.
Sympa Menu

perfsonar-dev - [pS-dev] [GEANT/SA2/ps-java-services] r5904 - in trunk/perfsonar-java-xml-ls: . log4j/development src/main/java/org/perfsonar/service/lookupservice src/main/java/org/perfsonar/service/lookupservice/storage src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao src/test/java/org/perfsonar/tests/actions src/test/java/org/perfsonar/tests/components src/test/java/org/perfsonar/tests/dao src/test/java/org/perfsonar/tests/suite src/test/java/org/perfsonar/tests/utilities

Subject: perfsonar development work

List archive

[pS-dev] [GEANT/SA2/ps-java-services] r5904 - in trunk/perfsonar-java-xml-ls: . log4j/development src/main/java/org/perfsonar/service/lookupservice src/main/java/org/perfsonar/service/lookupservice/storage src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao src/test/java/org/perfsonar/tests/actions src/test/java/org/perfsonar/tests/components src/test/java/org/perfsonar/tests/dao src/test/java/org/perfsonar/tests/suite src/test/java/org/perfsonar/tests/utilities


Chronological Thread 
  • From:
  • To:
  • Subject: [pS-dev] [GEANT/SA2/ps-java-services] r5904 - in trunk/perfsonar-java-xml-ls: . log4j/development src/main/java/org/perfsonar/service/lookupservice src/main/java/org/perfsonar/service/lookupservice/storage src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao src/test/java/org/perfsonar/tests/actions src/test/java/org/perfsonar/tests/components src/test/java/org/perfsonar/tests/dao src/test/java/org/perfsonar/tests/suite src/test/java/org/perfsonar/tests/utilities
  • Date: Thu, 15 Mar 2012 20:08:04 +0000 (GMT)
Author: dante.delvaux
Date: 2012-03-15 20:08:04 +0000 (Thu, 15 Mar 2012)
New Revision: 5904

Added:

trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/MultipleFilesDAOImplTests.java
Removed:

trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/SingleFileDAOImpl.java

trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/CommonExistDaoTests.java
Modified:
trunk/perfsonar-java-xml-ls/log4j/development/log4j.properties
trunk/perfsonar-java-xml-ls/pom.xml

trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/AbstractRegisterServiceEngine.java

trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/DiscoveryQueryGenerator.java

trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/LookupServiceDAO.java

trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/MultipleFilesDAOImpl.java

trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/actions/DiscoveryActionTests.java

trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/components/ModificationCounterComponentTest.java

trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/suite/LSTestSuite.java

trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/utilities/AsynchTester.java
Log:
Using XPaths expressions instead of WHERE clauses in internal XQueries to
improve performance, refs PSSERVICES-558. Improving documentation and unit
testing coverage on the modified XQueries.

Removing old and unused SingleFileDAOImpl class (only MultipleFilesDAOImpl is
needed now).

Adding GEANT 3rd party products maven repository to solve the eXist
dependency not being found.

Modified: trunk/perfsonar-java-xml-ls/log4j/development/log4j.properties
===================================================================
--- trunk/perfsonar-java-xml-ls/log4j/development/log4j.properties
2012-03-12 20:31:39 UTC (rev 5903)
+++ trunk/perfsonar-java-xml-ls/log4j/development/log4j.properties
2012-03-15 20:08:04 UTC (rev 5904)
@@ -9,6 +9,7 @@
#log4j.logger.org.perfsonar.service.lookupservice.components=DEBUG

#log4j.logger.org.perfsonar.service.lookupservice.components.XMLDBComponent=TRACE
#log4j.logger.org.perfsonar.service.lookupservice.storage=DEBUG
+#log4j.logger.org.perfsonar.service.lookupservice.storage.exist.dao=DEBUG
log4j.logger.org.perfsonar.tests=INFO
#log4j.logger.org.perfsonar.base2.service.configuration=DEBUG
#log4j.logger.org.exist.xmldb=DEBUG

Modified: trunk/perfsonar-java-xml-ls/pom.xml
===================================================================
--- trunk/perfsonar-java-xml-ls/pom.xml 2012-03-12 20:31:39 UTC (rev 5903)
+++ trunk/perfsonar-java-xml-ls/pom.xml 2012-03-15 20:08:04 UTC (rev 5904)
@@ -231,6 +231,11 @@
<name>GEANT repository</name>
<url>http://maven.geant.net/repo</url>
</repository>
+ <repository>
+ <id>geant-repository-3rdparty</id>
+ <name>GEANT repository 3rd party products</name>
+ <url>http://maven.geant.net/repo/net/geant/3rdparty</url>
+ </repository>
</repositories>

<distributionManagement>

Modified:
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/AbstractRegisterServiceEngine.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/AbstractRegisterServiceEngine.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/AbstractRegisterServiceEngine.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -4,8 +4,6 @@

import org.perfsonar.base2.service.exceptions.PerfSONARException;
import org.perfsonar.base2.service.requesthandler.ServiceEngine;
-import org.perfsonar.service.lookupservice.storage.LookupServiceDAO;
-import org.perfsonar.service.lookupservice.storage.LookupServiceDAOFactory;

import org.apache.log4j.Logger;

@@ -14,6 +12,7 @@
* Class that has commons methods for registration, updates, and keepalives
*
* @author Maciej Glowiak
+ * @author Antoine Delvaux - DANTE
*
*/
public abstract class AbstractRegisterServiceEngine
@@ -36,16 +35,12 @@
protected long updateTime(String key)
throws PerfSONARException {

- LookupServiceDAO dao = LookupServiceDAOFactory.getDAO();
-
- //get current time
- long now = System.currentTimeMillis() / 1000; //in seconds
- logger.debug("[LS Registration] Update time... ["+now+" sec]");
+ // Updating to the current time in seconds
+ long now = System.currentTimeMillis() / 1000;
+ logger.debug("Update time... [" + now + " sec]");

//run setControlParameter, modify control schema of the LS
dao.setControlParameter(key, "timestamp", Long.toString(now));
-
return now;
-
}
}

Modified:
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/DiscoveryQueryGenerator.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/DiscoveryQueryGenerator.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/DiscoveryQueryGenerator.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -9,16 +9,16 @@
import org.perfsonar.base2.xml.Element;

/**
- * The Discovery Query is built from a subject Element containing all the
search conditions.
- * Search conditions can be any number of the followings:
- * - IP addresses (address Elements)
- * - Domains (domain Elements)
- * - Event Types (eventType Elements)
- * - Parameters (pair of parameters/parameter Elements containing keyword
and eventType Elements)
+ * The Discovery Query is built from a subject Element containing all the
search conditions.</br>
+ * Search conditions can be any number of the followings:<ul>
+ * <li> IP addresses (address Elements)
+ * <li> Domains (domain Elements)
+ * <li> Event Types (eventType Elements)
+ * <li> Parameters (pair of parameters/parameter Elements containing keyword
and eventType Elements)
+ * </ul>
+ * The last two search Elements can be grouped inside a parameters/parameter
Element.</br>
+ * Subject Element syntax and namespaces are checked.
*
- * The last two search Elements can be grouped inside a parameters/parameter
Element
- * Subject Element syntax and namespaces are checked
- *
* @author Maciej Glowiak
* @author Antoine Delvaux - DANTE
*
@@ -60,12 +60,12 @@
}

/**
- * Explodes the subject Element and extracts:
- * - address
- * - domain
- * - eventType
- * - parameters (each with name, keyword and eventType)
- *
+ * Explodes the subject Element and extracts:<ul>
+ * <li> address
+ * <li> domain
+ * <li> eventType
+ * <li> parameters (each with name, keyword and eventType)
+ * </ul>
* Check the syntax of the subject Element and throws an exception
in case the inside elements are not supported or the namespace is wrong.
*
* @throws PerfSONARException
@@ -189,9 +189,8 @@
}

/**
- * Generates query and stores to query property
+ * Generates the discovery query and stores it in the query property
*
- * @return
* @throws PerfSONARException
*/
private void generateQuery() throws PerfSONARException {
@@ -209,24 +208,15 @@
q.append("declare namespace
nmwgt='http://ggf.org/ns/nmwg/topology/2.0/'; \n");

// Start the query with a for expression
- q.append("for $d in
/nmwg:store[@type='LSStore']/nmwg:data
\n");
-
- // generate query conditions
- // TODO: we should transform this where clause, it
brings bad performance with eXist, xpath expression should be preferred
- q.append("where \n");
-
+ q.append("for $d in
/nmwg:store[@type='LSStore']/nmwg:data[\n");
+ // Add search clauses
int i = 0;
- i += putCollectionToQueryBuffer(q, ipAddresses,
- "$d/nmwg:metadata//nmwgt:ifAddress",
(i > 0), null);
- i += putCollectionToQueryBuffer(q, domains,
- "$d/nmwg:metadata//nmwgt:hostName",
(i > 0), "ends-with");
- i += putCollectionToQueryBuffer(q, eventTypes,
- "$d/nmwg:metadata/nmwg:eventType", (i
> 0), null);
- i += putCollectionToQueryBuffer(q, keywords,
-
"$d/nmwg:metadata//nmwg:parameter[@name='keyword']",
- (i > 0), null);
+ i += addSearchClause(q, ipAddresses,
"descendant::nmwgt:ifAddress", (i > 0), null);
+ i += addSearchClause(q, domains,
"descendant::nmwgt:hostName", (i > 0), "ends-with");
+ i += addSearchClause(q, eventTypes,
"descendant::nmwg:eventType", (i > 0), null);
+ i += addSearchClause(q, keywords,
"descendant::nmwg:parameter[@name='keyword']",
(i > 0), null);

- q.append("return data($d/@metadataIdRef)");
+ q.append("]\nreturn data($d/@metadataIdRef)");

query = q.toString();
logger.debug("Generated XQuery is:\n" + query);
@@ -238,53 +228,48 @@
}

/**
- * Builds an AND/OR query clause and add it to the existing XQuery
+ * Builds an AND/OR query clause and add it to the existing XQuery
discovery query
*
- * @param q
- * buffer to be appended
- * @param col
- * collection where to take data from
- * @param element
- * name of an element to be checked
- * @param addAND
- * indicates whether to add AND at the beginning of the
- * expression
- * @param function
- * name of function, if null then equal "=" will be used
+ * @param q The query buffer to be
appended
+ * @param searchItems A collection items values to search for
+ * @param element Name of the element to be search for
+ * @param addAnd Indicates whether to add AND at the
beginning of the expression
+ * @param searchFunction Name of function to use in the search
clause, if null then equal "=" will be used
* @return number of elements added
*/
- private int putCollectionToQueryBuffer(StringBuffer q,
- Collection<String> col, String element, boolean
addAND,
- String function) {
-
- if ((col.size() > 0) && (addAND == true))
+ private int addSearchClause(StringBuffer q, Collection<String>
searchItems, String element, boolean addAnd, String searchFunction) {
+ int i = 0;
+ if ((searchItems.size() > 0) && (addAnd == true)) {
q.append(" and \n");
+ }
+ // Loop on all searchItems
+ for (String item : searchItems) {
+ if (i == 0) {
+ q.append(" (");
+ } else {
+ q.append(" or");
+ }

- int i = 0;
- for (String c : col) {
- if (i == 0)
- q.append("(");
- else
- q.append(" or ");
- if (function == null) {
+ if (searchFunction == null) {
q.append(" ");
q.append(element);
q.append(" = '");
- q.append(c);
+ q.append(item);
q.append("' ");
} else {
q.append(" ");
- q.append(function);
+ q.append(searchFunction);
q.append("(");
q.append(element);
q.append(",'");
- q.append(c);
+ q.append(item);
q.append("') ");
}
i++;
}
- if (i > 0)
+ if (i > 0) {
q.append(")\n");
+ }
return i;

}

Modified:
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/LookupServiceDAO.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/LookupServiceDAO.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/LookupServiceDAO.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -13,47 +13,51 @@
public void putLookupInfo(Metadata m, Data[] d) throws
PerfSONARException;

/**
- * Removes Lookup Information from LSStore database
- * @param metadataId
+ * Removes a Lookup Information entry from LSStore database, both in the
/store collection and in the control file
+ * @param metadataId The Id of the entry to be removed
* @return number of items removed, -1 if error
* @throws PerfSONARException
*/
public int removeLookupInfo(String metadataId) throws
PerfSONARException;

/**
- * Checks whether lookup information is registered
- * @param metadataId
+ * Checks whether a Lookup Information entry is registered
+ * @param metadataId The Id of the entry to be checked
* @return true if lookup information exists, false otherwise
* @throws PerfSONARException
*/
public boolean isLookupInfo(String metadataId) throws
PerfSONARException;

/**
- * Performs LS Cleanup operation
- * @param timestamp current (or other) unix timestamp
- * @param defaultInterval interval in millis
+ * Cleanup the old entries in the LSStore.</br>
+ * Old entries are the ones older than:<ul>
+ * <li> timestamp + defaultInterval
+ * <li> timestamp + lsTTL parameter defined for the entry
+ * </ul>
+ * @param timestamp UNIX timestamp defining the validity time (value in
seconds)
+ * @param defaultInterval Default LsTTL interval in milliseconds (used
if none was given for an entry)
* @return number of elements removed
*/
public int cleanupLookupInfo(long timestamp, long defaultInterval)
throws PerfSONARException;

/**
- * Sets control parameter in LSStore-control
- * @param metadataId
- * @param parameter
- * @param value
+ * Sets control parameter in LSStore-control file
+ * @param metadataId The Id of the entry whose parameter need to be
modified
+ * @param parameter The name of the parameter to modify
+ * @param value The new value to assign to the parameter
* @throws PerfSONARException
*/
public void setControlParameter(String metadataId, String parameter,
String value) throws PerfSONARException;

/**
- * checks whether collection exists
- * @param collection
+ * Checks whether collection exists
+ * @param collection The collection to test
* @return
*/
public boolean isDBCollection(String collection) throws
PerfSONARException;

/**
- * checks connection to the DB
+ * Checks connection to the DB
*/
public boolean isDBConnectivity() throws PerfSONARException;


Modified:
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/MultipleFilesDAOImpl.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/MultipleFilesDAOImpl.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/MultipleFilesDAOImpl.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -39,6 +39,9 @@

private static final String LS_STORE_FOOTER = "\n</nmwg:store>";

+ /* (non-Javadoc)
+ * @see
org.perfsonar.service.lookupservice.storage.LookupServiceDAO#putLookupInfo(org.perfsonar.base2.xml.nmwg.Metadata,
org.perfsonar.base2.xml.nmwg.Data[])
+ */
public void putLookupInfo(Metadata m, Data[] d) throws
PerfSONARException {

StringBuffer sb = new StringBuffer();
@@ -51,44 +54,42 @@
try {
ser.write(os, m);
for (int i = 0; i < d.length; i++) {
- logger.debug("Serialize data: " + d[i]);
+ logger.trace("Serialize data: " + d[i]);
ser.write(os, d[i]);
}
} catch (XMLStreamException e1) {
- logger.error("Cannot serialize elements in
MultipleFilesExistDAOImpl: " + e1.getMessage());
- throw new PerfSONARException("error/parse", "Cannot
serialize elements in MultipleFilesExistDAOImpl");
+ logger.error("Cannot serialize elements: " +
e1.getMessage());
+ throw new PerfSONARException("error/parse", "Cannot
serialize elements: " + e1.getMessage());
} catch (IOException e1) {
- logger.error("Cannot serialize elements in
MultipleFilesExistDAOImpl: " + e1.getMessage());
- throw new PerfSONARException("error/parse",
"Cannot serialize elements in MultipleFilesExistDAOImpl");
+ logger.error("Cannot serialize elements: " +
e1.getMessage());
+ throw new PerfSONARException("error/parse",
"Cannot serialize elements: " + e1.getMessage());
}
sb.append(LS_STORE_FOOTER);

// Create an md5 ID to identify the XML document and store it
- try {
- String XMLFileName = md5(m.getId()) + ".xml";
- logger.debug("Now uploading XML file to the store: "
+ XMLFileName);
- uploadStoreFile(XMLFileName, sb.toString());
- } catch (NoSuchAlgorithmException e) {
- throw new PerfSONARException("md5 - algorithm not
found",e);
- }
+ String XMLFileName = md5(m.getId()) + ".xml";
+ logger.debug("Now uploading XML file to the store: " +
XMLFileName);
+ uploadStoreFile(XMLFileName, sb.toString());

}

+ /* (non-Javadoc)
+ * @see
org.perfsonar.service.lookupservice.storage.LookupServiceDAO#removeLookupInfo(java.lang.String)
+ */
public int removeLookupInfo(String metadataId) throws
PerfSONARException {
-
int result = 0;
-
- try {
- removeFile(md5(metadataId) + ".xml");
- } catch (NoSuchAlgorithmException e) {
- throw new PerfSONARException("md5 - algorithm not
found",e);
- }

- String xquery = XQUERY_DECLARE_NAMESPACES + " for $i in " +
"("
- +
"/nmwg:store[@type='"
+ LS_STORE_CONTROL_COLLECTION_TYPE
- +
"']/nmwg:metadata[@id='"
+ metadataId + "']" + ")"
- + " return (update delete $i, 'r') ";
+ // TODO: This should be done in an atomic way so that we
cannot remove the file without also removing the control information

+ removeFile(md5(metadataId) + ".xml");
+
+ // We now remove the references in the LS control store
+ // The return statement, also returns an 'r' per removed row
to count the removed entries
+ String xquery = DECLARE_NMWG_NAMESPACE +
+ " for $meta in
/nmwg:store[@type='"
+ LS_STORE_CONTROL_COLLECTION_TYPE + "']" +
+
"/nmwg:metadata[@id='"
+ metadataId + "']" +
+ " return (update delete $meta, 'r') ";
+
XMLDBResult res = performXQuery(xquery);
result = res.getTotal();

@@ -96,55 +97,54 @@

}

- private void removeControlElement(String metadataId)
- throws PerfSONARException {
- String xquery = DECLARE_NMWG_NAMESPACE
- + " for $meta in
/nmwg:store[@type='LSStore-control']/nmwg:metadata[@id='"
- + metadataId + "'] \n" + " return (update
delete $meta)";
-
- performXQuery(xquery);
- }
-
+ /* (non-Javadoc)
+ * @see
org.perfsonar.service.lookupservice.storage.LookupServiceDAO#cleanupLookupInfo(long,
long)
+ */
public int cleanupLookupInfo(long timestamp, long defaultInterval)
throws PerfSONARException {
int result;
- String xquery = DECLARE_NMWG_NAMESPACE
- +
+
+ // First look for all the entry to be removed, based on
timestamp, defaultInterval or existing lsTTL
+ // We delete the entries in the control file at the same time
we find them
+ String xquery = DECLARE_NMWG_NAMESPACE +
+ "for $meta in " +
+
"/nmwg:store[@type='LSStore-control']/nmwg:metadata"
+
+
"[not(descendant::nmwg:parameter[@name='lsTTL'])]"
+
+
"[descendant::nmwg:parameters/nmwg:parameter[@name='timestamp'][@value
+ " + defaultInterval + " < " + timestamp + "]]\n" +
+ "|\n" +
+
"/nmwg:store[@type='LSStore-control']/nmwg:metadata"
+
+
"[descendant::nmwg:parameters/nmwg:parameter[@name='lsTTL']]"
+
+
"[descendant::nmwg:parameters/nmwg:parameter[@name='timestamp'][(@value
+
data(../nmwg:parameter[@name='lsTTL']/@value))
< (" + timestamp + ")]]\n" +
+ "return (data($meta/@id), update delete
$meta)";

- "let $now:="
- + timestamp
- + "\n"
- + "let $interval:="
- + defaultInterval
- + "\n"
- +
- "for $meta in
/nmwg:store[@type='LSStore-control']/nmwg:metadata
\n"
- + " let $ts :=
data($meta/nmwg:parameters/nmwg:parameter[@name='timestamp']/@value)
cast as xs:integer\n"
- + " let $metaid := data($meta/@id)\n "
- + "where ("
- + "
if(exists($meta/nmwg:parameters/nmwg:parameter[@name='lsTTL']))"
- + " then ( let $serviceLsTTL :=
data($meta/nmwg:parameters/nmwg:parameter[@name='lsTTL']/@value)
cast as xs:double"
- + " where ($ts +
$serviceLsTTL) < $now return true()"
- + " )"
- + " else"
- + " ($ts + $interval) <
$now"
- + " )"
- + "return $metaid\n";
-
XMLDBResult res = performXQuery(xquery);
result = res.getResults().size();
+ logger.debug(result + " files will be removed from the
store");
+
+ // Then we remove the XML files in the store
for (Element e : res.getResults()) {
String metadataId = e.getText();
- removeLookupInfo(metadataId);
- removeControlElement(metadataId);
+ removeFile(md5(metadataId) + ".xml");
}

return result;
}

- private String md5(String text) throws NoSuchAlgorithmException {
+ /**
+ * Generates an MD5 hash of the given text
+ * @param text The text to compute an MD5 hash for
+ * @return The MD5 hash
+ * @throws PerfSONARException
+ */
+ private String md5(String text) throws PerfSONARException {
MessageDigest md = null;
+ try {
md = MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ logger.error("md5 algorithm not found: " +
e.getMessage());
+ throw new PerfSONARException("md5 - algorithm not
found", e);
+ }
+
md.update(text.getBytes());
byte messageDigest[] = md.digest();
StringBuffer hexString = new StringBuffer();

Deleted:
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/SingleFileDAOImpl.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/SingleFileDAOImpl.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao/SingleFileDAOImpl.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -1,176 +0,0 @@
-package org.perfsonar.service.lookupservice.storage.exist.dao;
-
-import java.io.IOException;
-
-import javax.xml.stream.XMLStreamException;
-
-import org.apache.log4j.Logger;
-import org.perfsonar.base2.service.exceptions.PerfSONARException;
-import org.perfsonar.base2.service.storage.xml.XMLDBResult;
-import org.perfsonar.base2.xml.ElementSerializer;
-import org.perfsonar.base2.xml.StringBufferOutputStream;
-import org.perfsonar.base2.xml.nmwg.Data;
-import org.perfsonar.base2.xml.nmwg.Metadata;
-import org.perfsonar.service.lookupservice.storage.LookupServiceDAO;
-import
org.perfsonar.service.lookupservice.storage.exist.XQueryDAOSupportImpl;
-
-/**
- * Implementation of LookupServiceDAO: eXist HTTP connection using XQuery for
- * modification using "+LS_STORE_COLLECTION_TYPE+".xml,
- * "+LS_STORE_COLLECTION_TYPE+"-control.xml files in one collection
- *
- * @author Maciej Glowiak
- * @author Antoine Delvaux - DANTE
- *
- */
-public class SingleFileDAOImpl extends XQueryDAOSupportImpl implements
LookupServiceDAO {
-
- protected static final String UPDATE_LSSTORE = "
/nmwg:store[@type='"
- + LS_STORE_COLLECTION_TYPE + "'] ";
- private Logger logger = Logger.getLogger(SingleFileDAOImpl.class);
-
- public void putLookupInfo(Metadata m, Data[] d) throws
PerfSONARException {
-
- if (m == null) {
- String msg = "No metadata. Cannot put into database";
- logger.error(msg);
- // TODO: url event types
- throw new PerfSONARException("error.ls.no_metadata",
msg);
- }
- if (d == null) {
- String msg = "No data (null). Cannot put into
database";
- logger.error(msg);
- // TODO: url event types
- throw new PerfSONARException("error.ls.no_data", msg);
- }
- if (d.length < 1) {
- String msg = "No data (0). Cannot put into database";
- logger.error(msg);
- // TODO: url event types
- throw new PerfSONARException("error.ls.no_data", msg);
- }
-
- // serialize Metadata and Data
- logger.debug("Serialize metadata and data");
-
- // send xquery update expression containing all metadata and
data
- // elements in one xquery
- // instruction; should work faster than code above (commented)
- ElementSerializer ser = new ElementSerializer();
- ser.setStartingDocumentDeclaration(false);
- StringBuffer xmls = new StringBuffer();
- StringBufferOutputStream os = new
StringBufferOutputStream(xmls);
-
- // add xquery namespaces and UPDATE instruction
- xmls.append(XQUERY_DECLARE_NAMESPACES);
- xmls.append(UPDATE_INSERT);
- xmls.append(" (\n"); // IMPORTANT - begin Xquery collection
(m, d, d,
- // ...)
-
- // add metadata to StringBuffer
- try {
- ser.write(os, m);
- // convert data to Strings
- for (int i = 0; i < d.length; i++) {
- xmls.append(", \n"); // IMPORTANT - separator
for next element
- // of Xquery collection (m, d, d, ...)
- ser.write(os, d[i]);
- }
- } catch (XMLStreamException e) {
- // TODO: url event types
- throw new PerfSONARException("error.parse",
- "Cannot serialize elements in
XQueryExistDAOImpl");
- } catch (IOException e) {
- // TODO: url event types
- throw new PerfSONARException("error.parse",
- "Cannot serialize elements in
XQueryExistDAOImpl");
- }
-
- // send XQuery
- xmls.append(" )\n"); // IMPORTANT - end of Xquery collection
(m, d, d,
- // ...)
- xmls.append(UPDATE_INTO);
- xmls.append(UPDATE_LSSTORE);
-
- String query = xmls.toString();
- logger.debug("We insert the data using an XQuery call");
- performXQuery(query);
- }
-
- public int removeLookupInfo(String metadataId) throws
PerfSONARException {
-
- String xquery = XQUERY_DECLARE_NAMESPACES + UPDATE_FOR
- + getRemoveLookupInfoXQueryPart(metadataId) +
UPDATE_REMOVE;
- // remove by xquery
- XMLDBResult res = performXQuery(xquery);
- logger.debug("REMOVE LOOKUP QUERY:\n" + xquery);
- logger.debug("REMOVE LOOKUP RESULT:\n" + res);
- if (res != null)
- return res.getTotal();
- else
- return -1; // if error
-
- }
-
- public int cleanupLookupInfo(long timestamp, long defaultInterval)
throws PerfSONARException {
-
- String xquery = DECLARE_NMWG_NAMESPACE
- +
-
- "let $now:="
- + timestamp
- + "\n"
- + "let $interval:="
- + defaultInterval
- + "\n"
- +
- // to be changed - get interval from parameter
-
- "for $meta in
/nmwg:store[@type='LSStore-control']/nmwg:metadata
\n"
- + " let $ts :=
data($meta/nmwg:parameters/nmwg:parameter[@name='timestamp']/@value)
cast as xs:integer\n"
- + " let $metaid := data($meta/@id)\n "
- + "where ("
- + "
if(exists($meta/nmwg:parameters/nmwg:parameter[@name='lsTTL']))"
- + " then ( let $serviceLsTTL :=
data($meta/nmwg:parameters/nmwg:parameter[@name='lsTTL']/@value)
cast as xs:double"
- + " where ($ts +
$serviceLsTTL) < $now return true()"
- + " )"
- + " else"
- + " ($ts + $interval) <
$now"
- + " )"
- + "return \n"
- +
-
- "(\n"
- + " $metaid,\n"
- + " update delete
/nmwg:store[@type='LSStore-control']/nmwg:metadata[@id=$metaid],\n"
- + " update delete
/nmwg:store[@type='LSStore']/nmwg:metadata[@id=$metaid],
\n"
- + " update delete
/nmwg:store[@type='LSStore']/nmwg:data[@metadataIdRef=$metaid]\n"
- + " )\n";
-
- // System.out.println("---\n"+xquery+"\n---\n");
-
- XMLDBResult res = performXQuery(xquery);
-
- return res.getTextResults().length;
- }
-
- protected String getRemoveLookupInfoXQueryPart(String metadataId) {
- return
- // a set of ()
- "("
- +
- // metadata from
"+LS_STORE_CONTROL_COLLECTION_TYPE+"
-
"/nmwg:store[@type='"
+ LS_STORE_CONTROL_COLLECTION_TYPE +
"']/nmwg:metadata[@id='"
- + metadataId + "']"
- + ", "
- +
- // metadata from "+LS_STORE_COLLECTION_TYPE+"
-
"/nmwg:store[@type='"
+ LS_STORE_COLLECTION_TYPE +
"']/nmwg:metadata[@id='"
- + metadataId + "']" + ", "
- +
- // data from "+LS_STORE_COLLECTION_TYPE+"
-
"/nmwg:store[@type='"
+ LS_STORE_COLLECTION_TYPE +
"']/nmwg:data[@metadataIdRef='"
- + metadataId + "']" + ")";
- }
-
-}

Modified:
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/actions/DiscoveryActionTests.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/actions/DiscoveryActionTests.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/actions/DiscoveryActionTests.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -102,10 +102,11 @@

assertTrue(eventType.getText().contains("http://ogf.org/ns/nmwg/tools/org/perfsonar/service/lookup/discovery/summary/2.0";));

// Check result
-
logger.debug(serviceResponse.getElement().getChildren("data").iterator().next());
+
logger.trace(serviceResponse.getElement().getChildren("data").iterator().next());
assertNotNull("No data/metadata part was returned",
serviceResponse.getElement().getChildren("data").iterator().next().getChildren("metadata"));
Collection<Element> dataSubject =
serviceResponse.getElement().getChildren("data").iterator().next().getChildren("metadata").iterator().next().getChildren("subject");
Collection<Element> serviceResults =
dataSubject.iterator().next().getChildren("service");
+ logger.debug("Now checking every property of the returned
service");

assertTrue(serviceResults.iterator().next().getChildren("serviceName").iterator().next().getText().contains(RegistrationActionTests.testingServiceName));

assertTrue(serviceResults.iterator().next().getChildren("accessPoint").iterator().next().getText().contains(RegistrationActionTests.testingAccessPoint));

assertTrue(serviceResults.iterator().next().getChildren("serviceType").iterator().next().getText().contains(RegistrationActionTests.testingServiceType));

Modified:
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/components/ModificationCounterComponentTest.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/components/ModificationCounterComponentTest.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/components/ModificationCounterComponentTest.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -9,7 +9,7 @@

/**
* Test the Modification Counter Component, thread aware
- * @author Antoine Delvaux

+ * @author Antoine Delvaux - DANTE
*
*/
public class ModificationCounterComponentTest extends TestCase {

Deleted:
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/CommonExistDaoTests.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/CommonExistDaoTests.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/CommonExistDaoTests.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -1,86 +0,0 @@
-package org.perfsonar.tests.dao;
-
-import junit.framework.TestCase;
-
-import org.apache.log4j.Logger;
-import org.perfsonar.base2.service.exceptions.PerfSONARException;
-import org.perfsonar.base2.xml.nmwg.Data;
-import org.perfsonar.base2.xml.nmwg.Metadata;
-import org.perfsonar.service.lookupservice.storage.LookupServiceDAO;
-import org.perfsonar.service.lookupservice.storage.LookupServiceDAOFactory;
-
-public class CommonExistDaoTests extends TestCase {
-
- private static final Logger logger =
Logger.getLogger(CommonExistDaoTests.class);
-
- private final LookupServiceDAO lsDao;
-
- /**
- * @throws PerfSONARException
- *
- */
- public CommonExistDaoTests() throws PerfSONARException {
- super();
- lsDao = LookupServiceDAOFactory.getDAO();
- }
-
- private void insertExampleData(String metadataId)
- throws PerfSONARException {
- Metadata meta = new Metadata();
- meta.setId(metadataId);
-
- Data data1 = new Data();
- data1.setId("id2");
- data1.setMetadataIdRef(meta.getId());
-
- Data data2 = new Data();
- data2.setId("id1");
- data2.setMetadataIdRef(meta.getId());
-
- lsDao.putLookupInfo(meta, new Data[] { data1, data2 });
- }
-
- public void testDBConnectivity() throws PerfSONARException {
- logger.info("•••• Testing LS DAO connectivity");
- assertTrue(lsDao.isDBConnectivity());
- }
-
- public void testCollectionExists() throws PerfSONARException {
- logger.info("•••• Testing LS DAO collections exists");
- assertTrue(lsDao.isDBConnectivity());
- assertFalse(lsDao.isDBCollection("DUMMY_COLLECTION_NAME"));
- assertNotNull(lsDao);
- assertTrue(lsDao.isDBCollection("LSStore-control"));
- }
-
- // TODO: Does nothing!
- public void testRemoveFile() throws PerfSONARException {
- logger.info("•••• Testing LS DAO file removal (empty test for
now)");
- assertTrue(lsDao.isDBConnectivity());
- }
-
-
- public void testIsLookupInfo() throws PerfSONARException {
- logger.info("•••• Testing LS DAO insertion and deletion of LS
info");
- assertTrue(lsDao.isDBConnectivity());
- String metadataId="id"+System.currentTimeMillis();
-
- // Remove any existing element, to be safe
- logger.debug("Cleaning up for test data");
- lsDao.removeLookupInfo(metadataId);
-
- // Inserting some data
- logger.debug("Inserting test data");
- assertTrue(lsDao.isDBConnectivity());
- insertExampleData(metadataId);
- assertTrue(lsDao.isLookupInfo(metadataId));
- logger.debug("... done!");
-
- // Removing the data
- logger.debug("Removing test data");
- assertTrue(lsDao.isDBConnectivity());
- lsDao.removeLookupInfo(metadataId);
- assertFalse(lsDao.isLookupInfo(metadataId));
- }
-
-}

Copied:
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/MultipleFilesDAOImplTests.java
(from rev 5903,
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/CommonExistDaoTests.java)
===================================================================
---
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/MultipleFilesDAOImplTests.java
(rev 0)
+++
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/dao/MultipleFilesDAOImplTests.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -0,0 +1,181 @@
+package org.perfsonar.tests.dao;
+
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
+import org.perfsonar.base2.service.exceptions.PerfSONARException;
+import org.perfsonar.base2.service.storage.xml.XMLDBResult;
+import org.perfsonar.base2.xml.nmwg.Data;
+import org.perfsonar.base2.xml.nmwg.Metadata;
+import org.perfsonar.service.lookupservice.storage.LookupServiceDAO;
+import org.perfsonar.service.lookupservice.storage.LookupServiceDAOFactory;
+
+/**
+ * This Unit Test can check the behavior of the MultipleFilesDAOImpl class.
+ *
+ * @author Antoine Delvaux - DANTE
+ */
+public class MultipleFilesDAOImplTests extends TestCase {
+
+ private static final Logger logger =
Logger.getLogger(MultipleFilesDAOImplTests.class);
+
+ private final LookupServiceDAO lsDao;
+
+ /**
+ * @throws PerfSONARException
+ *
+ */
+ public MultipleFilesDAOImplTests() throws PerfSONARException {
+ super();
+ // TODO: are we checking SingleFileDAO or MultipleFileDAO?
+ lsDao = LookupServiceDAOFactory.getDAO();
+ }
+
+ /**
+ * Insert an empty element and sets its timestamp
+ * @param metadataId
+ * @throws PerfSONARException
+ */
+ private void insertExampleData(String metadataId)
+ throws PerfSONARException {
+ Metadata meta = new Metadata();
+ meta.setId(metadataId);
+
+ Data data1 = new Data();
+ data1.setId("id2");
+ data1.setMetadataIdRef(meta.getId());
+
+ Data data2 = new Data();
+ data2.setId("id1");
+ data2.setMetadataIdRef(meta.getId());
+
+ long now = System.currentTimeMillis() / 1000;
+ lsDao.setControlParameter(metadataId, "timestamp",
Long.toString(now));
+ lsDao.putLookupInfo(meta, new Data[] { data1, data2 });
+ }
+
+ public void testDBConnectivity() throws PerfSONARException {
+ logger.info("•••• Testing LS DAO connectivity");
+ assertTrue(lsDao.isDBConnectivity());
+ }
+
+ public void testCollectionExists() throws PerfSONARException {
+ logger.info("•••• Testing LS DAO collections exists");
+ assertTrue(lsDao.isDBConnectivity());
+ assertFalse(lsDao.isDBCollection("DUMMY_COLLECTION_NAME"));
+ assertNotNull(lsDao);
+ assertTrue(lsDao.isDBCollection("LSStore-control"));
+ }
+
+ /**
+ * Tests both the insertion and the removal of lookup information
+ *
+ * @throws PerfSONARException
+ */
+ public void testIsLookupInfo() throws PerfSONARException {
+ logger.info("•••• Testing LS DAO insertion and deletion of LS
info");
+ assertTrue(lsDao.isDBConnectivity());
+ String metadataId = "id" + System.currentTimeMillis();
+
+ // Remove any existing element, to be safe
+ logger.debug("Cleaning up for test data");
+ lsDao.removeLookupInfo(metadataId);
+
+ // Inserting some data
+ logger.debug("Inserting test data");
+ assertTrue(lsDao.isDBConnectivity());
+ insertExampleData(metadataId);
+ assertTrue(lsDao.isLookupInfo(metadataId));
+ logger.debug("... done!");
+
+ // Removing the data
+ logger.debug("Removing test data");
+ assertTrue(lsDao.isDBConnectivity());
+ lsDao.removeLookupInfo(metadataId);
+ assertFalse(lsDao.isLookupInfo(metadataId));
+ }
+
+ /**
+ * Test the cleanup of old LS info currently stored.<\br>
+ * This test does the following:<ul>
+ * <li> count the number of existing entries
+ * <li> insert 2 old entries (> TTL) and 2 new entries (< TTL)
+ * <li> run the cleanup process
+ * <li> check that at least 2 entries have been removed
+ * <li> check that the 2 old dated entries are removed and that the 2
new ones are left intact
+ * </ul>
+ * @throws PerfSONARException
+ * @throws InterruptedException
+ */
+ public void testCleanupLookupInfo() throws PerfSONARException,
InterruptedException {
+ logger.info("•••• Testing LS DAO cleanup of LS info");
+ assertTrue(lsDao.isDBConnectivity());
+
+ // Initial number of entries in the Store
+ int initialCount = countStoreEntries();
+
+ // Insert 2 new entries without lsTTL
+ String metadataId = "id" + System.currentTimeMillis();
+ insertExampleData(metadataId);
+ assertTrue(lsDao.isLookupInfo(metadataId));
+ metadataId = "id" + System.currentTimeMillis();
+ insertExampleData(metadataId);
+ assertTrue(lsDao.isLookupInfo(metadataId));
+
+ // Insert 2 new entries with a long lsTTL (10.000 sec)
+ metadataId = "id" + System.currentTimeMillis();
+ lsDao.setControlParameter(metadataId, "lsTTL", "10000");
+ insertExampleData(metadataId);
+ assertTrue(lsDao.isLookupInfo(metadataId));
+ String savedMetadataId1 = metadataId;
+ metadataId = "id" + System.currentTimeMillis();
+ lsDao.setControlParameter(metadataId, "lsTTL", "10000");
+ insertExampleData(metadataId);
+ assertTrue(lsDao.isLookupInfo(metadataId));
+ String savedMetadataId2 = metadataId;
+
+ // Check that we actually have 4 entries more
+ int afterInsertCount = countStoreEntries();
+ assertTrue(initialCount + 4 == afterInsertCount);
+
+ // Run the cleanup process and try to remove the 2 entries
without lsTTL defined
+ // Removes any entry expiring one second from now and not
having its own lsTTL defined
+ long now = (System.currentTimeMillis() / 1000) + 1;
+ int removedEntries = lsDao.cleanupLookupInfo(now, 0);
+ assertTrue(removedEntries >= 2);
+
+ // Check that we have at least 2 entries less but that at
least the 2 old entries are present
+ int afterCleanupCount = countStoreEntries();
+ assertTrue(afterCleanupCount <= afterInsertCount - 2);
+ assertTrue(afterCleanupCount >= 2);
+
+ // Check that the old entries are still present
+ assertTrue(lsDao.isLookupInfo(savedMetadataId1));
+ assertTrue(lsDao.isLookupInfo(savedMetadataId2));
+
+ // Removes the non expired entries
+ lsDao.removeLookupInfo(savedMetadataId1);
+ lsDao.removeLookupInfo(savedMetadataId2);
+ assertTrue("Maybe some older entries where present and have
been removed as well, run again to be sure", initialCount ==
countStoreEntries());
+
+ }
+
+ /**
+ * Count the number of entries in the XML-DB Store
+ * @return the total number of entries
+ */
+ private int countStoreEntries() {
+ int count = 0;
+ // TODO: build and XQuery counting the number of elements and
run it
+ try {
+ logger.debug("Counting documents in /db/ls/store/
with an XQuery");
+ XMLDBResult result =
lsDao.performXQuery("count(collection('/db/ls/store/'))");
+ count =
Integer.valueOf(result.getFirstElement().getText());
+ } catch (PerfSONARException ex) {
+ logger.error("Our count() XQuery returned an error: "
+ ex.getMessage());
+ }
+
+ logger.debug(count + " documents found!");
+ return count;
+ }
+}

Modified:
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/suite/LSTestSuite.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/suite/LSTestSuite.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/suite/LSTestSuite.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -47,8 +47,9 @@
suite.addTestSuite(XMLDBComponentTest.class);

// DAO - FIXME: threading issue
+ // TODO: add LookupServiceDAOFactory
suite.addTestSuite(XQueryExistDAOImplTests.class);
- suite.addTestSuite(CommonExistDaoTests.class);
+ suite.addTestSuite(MultipleFilesDAOImplTests.class);
suite.addTestSuite(SummarizationDaoTests.class);

// Actions
@@ -62,6 +63,8 @@
// Summarization
suite.addTestSuite(SummarizationTests.class);

+ // TODO: Scheduler
+
return suite;
}
}

Modified:
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/utilities/AsynchTester.java
===================================================================
---
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/utilities/AsynchTester.java
2012-03-12 20:31:39 UTC (rev 5903)
+++
trunk/perfsonar-java-xml-ls/src/test/java/org/perfsonar/tests/utilities/AsynchTester.java
2012-03-15 20:08:04 UTC (rev 5904)
@@ -5,7 +5,7 @@
*
* Reports the errors and exceptions up to the parent thread
*
- * @author Antoine Delvaux

+ * @author Antoine Delvaux - DANTE
*/
public class AsynchTester {
private Thread thread;


  • [pS-dev] [GEANT/SA2/ps-java-services] r5904 - in trunk/perfsonar-java-xml-ls: . log4j/development src/main/java/org/perfsonar/service/lookupservice src/main/java/org/perfsonar/service/lookupservice/storage src/main/java/org/perfsonar/service/lookupservice/storage/exist/dao src/test/java/org/perfsonar/tests/actions src/test/java/org/perfsonar/tests/components src/test/java/org/perfsonar/tests/dao src/test/java/org/perfsonar/tests/suite src/test/java/org/perfsonar/tests/utilities, svn-noreply, 03/15/2012

Archive powered by MHonArc 2.6.16.

Top of Page