perfsonar development work

[Stijn Melis <>]

Hi,

I created a class which will be handling all communication between the
MP and the metadata, which is contained in a eXist database. I based my
code loosely on the RRD MA. I use the same method to connect to the
xmldb, yet because I only need one class which creates the queries, this
is also handled in the same class.

I tried to test my implementation yesterday, and I am having problems
connecting to the db. At the moment I use the same principle as in the 
RRD MA, meaning that I create a getStoragemanager to get the 
XmlDbStorageManager, and then I use this to fetch the data. I attached 
the DbMetadataConfiguration class which is used to do this. All classes 
using it first call the constructor, and then immediately call the 
initStorage method.

When I run this and try to query the db, I get an error. I also attached 
the logs.

I also tried to run the ExistDbXmlrpcXmlStorageManager.java class as a 
standalone application to see if that one can connect to the xmldb, and 
that one doesn't work either. I also attached that one. I get a 
NullPointerException at line 317, being:

ExistDbXmlrpcXmlDataRepository dataRepository = new 
ExistDbXmlrpcXmlDataRepository();

The credentials I use to connect to the xmldb using the client GUI are 
the following:

username: sshtelnet
password: sshtelnet
url: xmldb:exist://localhost:8680/exist/xmlrpc

This works perfectly.

Does anyone has any clues as to what I am doing wrong?

Best regards and thanks in advance,

Stijn


package org.perfsonar.service.measurementPoint.lookingGlassType.metadata;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.perfsonar.base.auxiliary.AuxiliaryComponentManager;
import org.perfsonar.base.auxiliary.ComponentNames;
import 
org.perfsonar.base.auxiliary.components.configuration.ConfigurationComponent;
import org.perfsonar.base.auxiliary.components.logger.LoggerComponent;
import org.perfsonar.base.exceptions.PerfSONARException;
import org.perfsonar.service.base.storage.StorageManager;
import org.perfsonar.service.base.storage.xmldb.XmlDbDataRepository;
import org.perfsonar.service.base.storage.xmldb.XmlDbStorageManager;
import org.perfsonar.service.base.storage.xmldb.exist.ExistDbFactory;


public class DbMetadataConfiguration implements MetadataConfiguration {

        /**
         *  This object provides access to metadata configuration
         */

        protected StorageManager existDbXmlStorageManager = null;

        protected LoggerComponent logger = null;

        protected ConfigurationComponent configuration = null;

        protected String metadataFile = "";
        
        public DbMetadataConfiguration () throws PerfSONARException {
                try {

                        this.logger = (LoggerComponent) 
AuxiliaryComponentManager.
                        getInstance().getComponent(ComponentNames.LOGGER);

                } catch (PerfSONARException e) {

                        throw new PerfSONARException(
                                        "error.common.no_logger",
                                        
"MetadataConfigurationStorageManager.constructor: "
                                        + "Cannot get logger component");

                }

                try {

                        this.configuration = 
(ConfigurationComponent)AuxiliaryComponentManager.
                        getInstance().getComponent(ComponentNames.CONFIG);

                } catch (PerfSONARException e) {

                        throw new PerfSONARException(
                                        "error.common.no_configuration",
                                        
"MetadataConfigurationStorageManager.constructor: "
                                        + "Cannot get config component");

                }       
                
                metadataFile = 
configuration.getProperty("component.mp.xmldb.metadata_file_name"); 
        }

        public void initStorage() throws PerfSONARException {

                this.existDbXmlStorageManager = getStorageManager();
        }


        public String getProperty(String property)  throws PerfSONARException{
                logger.debug("[DbMetadataConfiguration] getProperty("
                                + property + ") called.");
                                
                return null;
        }

        public String getProperty(String device, String property)  throws 
PerfSONARException{
                logger.debug("[DbMetadataConfiguration] getProperty("
                                + device +", "+ property + ") called.");
                
                
                String query = "";
                String result = "";
                
                
                if (property.equalsIgnoreCase("url") || 
property.equalsIgnoreCase("class_name") 
                                || property.equalsIgnoreCase("port") || 
property.equalsIgnoreCase("prompt")
                                || property.equalsIgnoreCase("rate") || 
property.equalsIgnoreCase("username")
                                || property.equalsIgnoreCase("password") || 
property.equalsIgnoreCase("cache")
                                || property.equalsIgnoreCase("keyUrl") || 
property.equalsIgnoreCase("passphrase")){
                        query += "declare namespace 
nmwg='http://ggf.org/ns/nmwg/base/2.0/';\n";
                        query += "for $i in doc(\"" + metadataFile + 
"\")/nmwg:store/nmwg:metadata\n";
                        query += "where $i/nmwg:subject=\"" + device + "\"\n";
                        query += 
"fn:data($i/nmwg:parameters/nmwg:parameter[@name=\""
 + property + "\"]/@value)";
                }               
                
                logger.debug("[DbMetadataConfiguration] query: " + query);
                
                Object object = null;
        try {
            object = this.existDbXmlStorageManager.fetch(query);
        } catch (Exception ex) {
            logger.error("MetadataConfigurationStorageManager.fetch: "
                    + "Connection to xml database failed: "
                    + ex.toString());
            ex.printStackTrace();
                throw new PerfSONARException(
                    "error.common.storage.xmldb",
                    "DbMetadataConfiguration.getProperties: "
                    + "Connection to xml database failed");
        }
        String[] res = (String[]) object;
        if (res == null || res.length != 1)
                throw new PerfSONARException(
                    "error.mp",
                    "DbMetadataConfiguration.getProperty: "
                    + "Result array of metadata search is null or it contains 
no or too much elements"); 
        
        result = res[0];
        
        if (result == null)
            throw new PerfSONARException(
                    "error.mp",
                    "DbMetadataConfiguration.getProperty: "
                    + "Result array of metadata search in null"); 
        
                return result;
        }

        public String getProperty(String device, String command, String 
property)  throws PerfSONARException{

                logger.debug("[DbMetadataConfiguration] getProperty("
                                + device + ", " + command + ", " + property + 
") called.");
                
                
                String query = "";
                String result = "";
                
                
                if (property.equalsIgnoreCase("description") || 
property.equalsIgnoreCase("command")
                                || property.equalsIgnoreCase("minParamCount") 
|| property.equalsIgnoreCase("maxParamCoung")
                                || property.equalsIgnoreCase("caching") || 
property.equalsIgnoreCase("syntax") 
                                || property.equalsIgnoreCase("regex")){
                        query += "declare namespace 
nmwg=\"http://ggf.org/ns/nmwg/base/2.0/\";\n";;
                        query += "\n";
                        query += "let $file := doc(\"" + metadataFile + 
"\")\n";
                        query += "\n";
                        query += "let $ids :=\n";
                        query += "      for $j in 
$file/nmwg:store/nmwg:metadata\n";
                        query += "      where $j/nmwg:subject=\"" + device + 
"\"\n";
                        query += "      return fn:data($j/@id)\n";
                        query += "\n";
                        query += "for $i in $ids\n";
                        query += "return 
fn:data($file/nmwg:store/nmwg:data[@metadataIdRef=$i]/nmwg:key[nmwg:eventType='"
 + command + 
"']/nmwg:parameters/nmwg:parameter[@name='"
 + property + "']/@value)\n";
                }               
                
                logger.debug("[DbMetadataConfiguration] query: " + query);
                
                Object object = null;
        try {
            object = this.existDbXmlStorageManager.fetch(query);
        } catch (Exception ex) {
            logger.error("MetadataConfigurationStorageManager.fetch: "
                    + "Connection to xml database failed: "
                    + ex.toString());
            ex.printStackTrace();
                throw new PerfSONARException(
                    "error.common.storage.xmldb",
                    "DbMetadataConfiguration.getProperties: "
                    + "Connection to xml database failed");
        }
        
        String[] res = (String[]) object;
        if (res == null || res.length != 1)
                throw new PerfSONARException(
                    "error.mp",
                    "DbMetadataConfiguration.getProperty: "
                    + "Result array of metadata search is null or it contains 
no or too much elements"); 
        
        result = res[0];
        
        if (result == null)
            throw new PerfSONARException(
                    "error.mp",
                    "DbMetadataConfiguration.getProperty: "
                    + "Result array of metadata search is null"); 
    
        
                return result;
        }
        
        public List getProperties(String property) throws PerfSONARException{

                logger.debug("[DbMetadataConfiguration] getProperties("
                                + property + ") called.");
                
                
                String query = "";
                
                if (property.equals(MetadataConfiguration.DEVICE_NAMES)){
                        // search for all device names
                        // Query: 
                        query += "declare namespace 
nmwg='http://ggf.org/ns/nmwg/base/2.0/';\n";
                        query += "for $i in doc(\"" + metadataFile + 
"\")/nmwg:store/nmwg:metadata/nmwg:subject\n";
                        query += "return data($i)";
                }

                logger.debug("[DbMetadataConfiguration] query: " + query);
                
                Object object = null;
        try {
            object = this.existDbXmlStorageManager.fetch(query);
        } catch (Exception ex) {
            logger.error("DbMetadataConfiguration.getProperties: "
                    + "Connection to xml database failed: "
                    + ex.toString());
            ex.printStackTrace();
                throw new PerfSONARException(
                    "error.common.storage.xmldb",
                    "DbMetadataConfiguration.getProperties: "
                    + "Connection to xml database failed");
        }
        String[] xmlResults = (String[]) object;
        
        if (xmlResults == null)
            throw new PerfSONARException(
                    "error.mp",
                    "DbMetadataConfiguration.getProperties: "
                    + "Result array of metadata search in null"); 

        logger.debug("DbMetadataConfiguration.getProperties: "
            + "The number of elements returned as a result of xquery 
statement: " 
            + xmlResults.length);

        if (xmlResults.length == 0)
            throw new PerfSONARException(
                    "warning.mp.metadata_configuration",
                    "DbMetadataConfiguration.getProperties: "
                    + "No metadata in the metadata configuration");

        if (xmlResults.length == 1) 
            if (xmlResults[0].trim().equals(""))
                throw new PerfSONARException(
                        "warning.mp.metadata_configuration",
                        "DbMetadataConfiguration.getProperties: "
                        + "No metadata in the metadata configuration");
                     
        ArrayList result = new ArrayList();
        for (int i = 0; i < xmlResults.length; i++)
                result.add(xmlResults[i]);
                return result;
        }

        public List getProperties(String device, String property) throws 
PerfSONARException{

                logger.debug("[DbMetadataConfiguration] getProperty("
                                + device + ", " + property + ") called.");
                
                
                String query = "";
                
                if (property.equalsIgnoreCase("commands")){
                        // search for all command names for a certain device
                        // Query: 
                        query += "declare namespace 
nmwg=\"http://ggf.org/ns/nmwg/base/2.0/\";\n";;
                        query += "\n";
                        query += "let $file := doc(\"" + metadataFile + 
"\")\n";
                        query += "\n";
                        query += "let $ids :=\n";
                        query += "      for $j in 
$file/nmwg:store/nmwg:metadata\n";
                        query += "      where $j/nmwg:subject=\"" + device + 
"\"\n";
                        query += "      return fn:data($j/@id)\n";
                        query += "\n";
                        query += "for $i in $ids\n";
                        query += "return 
fn:data($file/nmwg:store/nmwg:data[@metadataIdRef=$i]/nmwg:key/nmwg:eventType)\n";
                }

                logger.debug("[DbMetadataConfiguration] query: " + query);
                
                Object object = null;
        try {
            object = this.existDbXmlStorageManager.fetch(query);
        } catch (Exception ex) {
            logger.error("DbMetadataConfiguration.getProperties: "
                    + "Connection to xml database failed: "
                    + ex.toString());
            ex.printStackTrace();
                throw new PerfSONARException(
                    "error.common.storage.xmldb",
                    "DbMetadataConfiguration.getProperties: "
                    + "Connection to xml database failed");
        }
        String[] xmlResults = (String[]) object;
        
        if (xmlResults == null)
            throw new PerfSONARException(
                    "error.mp",
                    "DbMetadataConfiguration.getProperties: "
                    + "Result array of metadata search in null"); 

        logger.debug("DbMetadataConfiguration.getProperties: "
            + "The number of elements returned as a result of xquery 
statement: " 
            + xmlResults.length);

        if (xmlResults.length == 0)
            throw new PerfSONARException(
                    "warning.mp.metadata_configuration",
                    "DbMetadataConfiguration.getProperties: "
                    + "No metadata in the metadata configuration");

        if (xmlResults.length == 1) 
            if (xmlResults[0].trim().equals(""))
                throw new PerfSONARException(
                        "warning.mp.metadata_configuration",
                        "DbMetadataConfiguration.getProperties: "
                        + "No metadata in the metadata configuration");
                     
        ArrayList result = new ArrayList();
        for (int i = 0; i < xmlResults.length; i++)
                result.add(xmlResults[i]);
                return result;
        }
        
        
        protected XmlDbStorageManager getStorageManager() 
        throws PerfSONARException {

                String xmldbAccess = "http";

                try {
                        xmldbAccess =
                                
configuration.getProperty("component.mp.xmldb.db_access");
                } catch (Exception ex) {
                        throw new PerfSONARException(
                                        "warning.mp.metadata_configuration",
                                        
"DbMetadataConfiguration.getStorageManager: "
                                        + "Something went wrong while reading 
the properties file");
                }

                if (xmldbAccess == null || xmldbAccess.equals(""))
                        xmldbAccess = "http";

                System.out.println("db_access: " + xmldbAccess);
                
                XmlDbDataRepository dataRepository =
                        ExistDbFactory.getDataRepositoryByProperties(
                                        xmldbAccess.toUpperCase(), 
                                        "component.mp.xmldb.db_uri",
                                        "component.mp.xmldb.db_username",
                        "component.mp.xmldb.db_password");

                System.out.println("XmlDataRepositoryUri     : "+ 
dataRepository.getDbUri());
                System.out.println("XmlDataRepositoryPassword: "+ 
dataRepository.getDbPassword());
                System.out.println("XmlDataRepositoryUsername: "+ 
dataRepository.getDbUsername());
                System.out.println("XmlDataRepositoryType    : "+ 
dataRepository.getType());
                
                XmlDbStorageManager storageManager = 
                        ExistDbFactory.getXMLStorageManager(
                                        xmldbAccess, dataRepository);
                storageManager.setResultType(
                                XmlDbStorageManager.STRING_ARRAY_TYPE);

                System.out.println("storageManager: " + storageManager);

                return storageManager;

        }
        
        public static void main (String[] args) {

        }
}
import java.util.Vector;

import org.perfsonar.base.auxiliary.AuxiliaryComponent;
import org.perfsonar.base.exceptions.PerfSONARException;
import org.perfsonar.service.base.storage.DataRepository;
import org.perfsonar.service.base.storage.StorageManager;
import org.perfsonar.service.base.storage.xmldb.XmlDbConstants;
import org.perfsonar.service.base.storage.xmldb.XmlDbResult;
import org.perfsonar.service.base.storage.xmldb.XmlDbStorageManager;
import org.perfsonar.service.base.storage.xmldb.exist.ExistDbFactory;
import 
org.perfsonar.service.base.storage.xmldb.exist.ExistDbGenericStorageManager;
import 
org.perfsonar.service.base.storage.xmldb.exist.xmlrpc.ExistDbXmlrpcXmlDataRepository;
import org.xmldb.api.base.Resource;
import org.xmldb.api.base.ResourceIterator;
import org.xmldb.api.base.ResourceSet;
import org.xmldb.api.base.XMLDBException;
import org.xmldb.api.modules.XPathQueryService;
import org.xmldb.api.modules.XUpdateQueryService;

/**
 *  This class is StorageManager for Exist DB XML
 *  (http://exist-db.org/)
 *  
 *  It provides fetch (XPath/XQuery expression as an input) and
 *  store (XUpdate XML expression as an input).
 *   
 *  @author Maciej Glowiak
 *
 */
public class ExistDbXmlrpcXmlStorageManager 
    extends ExistDbGenericStorageManager
    implements XmlDbStorageManager, AuxiliaryComponent {

    
    
    /**
     * Data repository object
     */
    private ExistDbXmlrpcXmlDataRepository dataRepository = null;
    
    
    /**
     * if false, the connection to DB will be established before each 
query/update and dropped 
     * after this action. If true, the connection will be kept open all the 
time
     */
    private boolean keepConnection = true;
    
    
    /**
     * Indicates if connection to DB is established
     */
    private boolean isConnected = false;
    
    
    // 
----------------------------------------------------------------------- 
Constructor
    
    
    public ExistDbXmlrpcXmlStorageManager() {
        
        isConnected = false;
        
    }
    
    // --------------------------------------------------------------------- 
Public methods 
    
    
    /**
     * Set data repository for this Storage Manager.
     * Important note: DataRepository MUST be ExistDbXmlrpcXmlDataRepository. 
Otherwise,
     * a PerfSONARException will be thrown!
     */
    public void initStorage(DataRepository dataRepository) throws 
PerfSONARException {
        
        getLogger().debug("ExistDbXmlrpcXmlStorageManager: initStorage with 
data repository");
        
        try {
        
            this.dataRepository = 
(ExistDbXmlrpcXmlDataRepository)dataRepository;
            
        } catch (ClassCastException ex) {
            
            getLogger().error("ExistDbXmlrpcXmlStorageManager: wrong data 
repository class ("+
                    dataRepository.getClass().getName()+
                    ")! Data repository must implement 
ExistDbXmlrpcXmlDataRepository class");
            
        }
        
    }

    
    public void initComponent() throws PerfSONARException {
        
        getLogger().debug("ExistDbXmlrpcXmlStorageManager: initComponent");
        
        //read properties
        getLogger().debug("ExistDbXmlrpcXmlStorageManager: read properties 
from configuration component");
        
        DataRepository dataRepository = 
            ExistDbFactory.getDataRepositoryByProperties(
                ExistDbFactory.EXIST_XMLRPC,     
                "component."+getComponentName()+".db_uri",
                "component."+getComponentName()+".db_username",
                "component."+getComponentName()+".db_password"
                );
        
        
        //Init Storage Manager with DataRepository
        initStorage(dataRepository);

    }    

    
    /**
     * Fetch data. dataQuery input parameter MUST be a String
     * @param dataQuery Must be String that contain XPath/XQuery expression
     * @return always StringArrayDataSet object
     */
    public Object fetch(Object dataQuery) throws PerfSONARException {

        getLogger().debug("ExistXMLDatabaseAccess: perform XQuery");
        
        String xqueryExpression = null;
        try {
            xqueryExpression = (String)dataQuery;
        } catch (ClassCastException ex) {
            throw new 
PerfSONARException("error.common.storage.xmldb.wrong_query",
                    "ExistDbXmlrpcXmlStorageManager: This method accepts " +
                    "StringData data format only");
        }
        
        if (xqueryExpression==null) {
            throw new 
PerfSONARException("error.common.storage.xmldb.empty_query",
                    "ExistDbXmlrpcXmlStorageManager: Input DataQuery " +
                    "contains no data");
        }

        try {

            connect();
            
            XPathQueryService service =
                (XPathQueryService) 
dataRepository.getExistCollection().getService(
                        "XPathQueryService", "1.0");
            //service.setProperty("indent", "yes");
                    
            ResourceSet result = service.query(xqueryExpression);
            ResourceIterator resIt = result.getIterator();
            Vector list = new Vector();
            
            int i=0;
            while(resIt.hasMoreResources()) {
                Resource r = resIt.nextResource();
                list.add(i,r.getContent());
                i++;
            }
            
            String[] xml = new String[list.size()];
            for (i=0; i < list.size(); i++) {
                xml[i] = (String)list.get(i);
            }
            
            disconnect();
            
            //determine return type
            
            if (getResultType() == XmlDbConstants.RETURN_STRING_ARRAY)
                return xml;
            else if (getResultType() == XmlDbConstants.RETURN_XML_DB_RESULT) {
                return new XmlDbResult(xml, xml.length, 1, xml.length);
            } else {
                //default
                return xml;
            }
            
        } catch (XMLDBException e) {            
            throw new PerfSONARException("error.common.storage.xmldb.query",
                    "Can't perform XQuery: "+e.getMessage());
        } 
        
    }
    
    
    /**
     * Store data. dataSet input parameter MUST be String
     * Yoo may use org.perfsonar.service.commons.storage.xmldb.StringDataQuery
     * @param dataSet Must be String, contains XUpdate string
     * @return null
     */
    public Object store(Object dataSet) throws PerfSONARException {

        
        getLogger().debug("ExistDbXmlrpcXmlStorageManager: perform XUpdate"); 
       
        
        String xupdateExpression = null;
        try {
        
            xupdateExpression = (String)dataSet;
            
        } catch (ClassCastException ex) {
            throw new 
PerfSONARException("error.common.storage.xmldb.wrong_query",
                    "ExistDbXmlrpcXmlStorageManager: This method accepts " +
                    "StringData data format only");
        }
        
        if (xupdateExpression==null) {
            throw new 
PerfSONARException("error.common.storage.xmldb.empty_query",
                    "ExistDbXmlrpcXmlStorageManager: Input DataQuery " +
                    "contains no data");
        }
        try {
            
            connect();
        
            XUpdateQueryService service = 
(XUpdateQueryService)dataRepository.getExistCollection().getService(
                        "XUpdateQueryService", "1.0");
            service.setCollection(dataRepository.getExistCollection());

            service.update(xupdateExpression);
            
            disconnect();
        
        } catch (XMLDBException e) {            
            throw new PerfSONARException("error.common.storage.xmldb.query",
                    "ExistDbXmlrpcXmlStorageManager: Can't perform XUpdate: 
"+e.getMessage());
        } 
        
        return null;
    }

    /**
     * Check if keepConnection flag is set.
     * @return true if the connection is persistent (common for all DB 
requests), 
     *         false if connection is established for each DB request and 
then 
     *         it's dropped 
     */
    public boolean isKeepConnection() {
        
        return keepConnection;
        
    }


    /**
     * Set keepConnection flag.
     * @param keepConnection - true if the connection is persistent (common 
for all DB requests), 
     *                         false if connection is established for each DB 
request and then 
     *                         it's dropped
     *                          
     */
    public void setKeepConnection(boolean keepConnection) {
        
        this.keepConnection = keepConnection;
        
    }

    
    // ------------------------------------------------------------------- 
Private methods
    
    /**
     * Connect if required
     */
    private void connect() throws PerfSONARException {
        
        if (!isConnected) {
            
            dataRepository.open();
            isConnected = true;
            
        }
        
    }
    
    /**
     * Disconnect if required
     * @throws PerfSONARException
     */
    private void disconnect() throws PerfSONARException {
        
        if ((isConnected) && (!keepConnection)) {

            dataRepository.close();
            isConnected = false;
            
        }
    }

    
    // 
------------------------------------------------------------------------- 
test case

    
    /*
     * Run query test (XPath/XQuery)
     */
    private static void testQuery(StorageManager storageManager, String 
xquery) throws PerfSONARException {

        String[] result = (String[])storageManager.fetch(
                xquery);
        
        for (int i=0; i<result.length; i++) {
            System.out.println("Result["+i+"]: "+result[i]);
        }
        
    }    
    
    
    
    // ---------------------------------------------------------- test
    
    
    /*
     * Simple test case. Just for testing
     */
    public static void main(String[] argv) throws Exception {

        //create Data Repository for Exist DB
        
        ExistDbXmlrpcXmlDataRepository dataRepository = new 
ExistDbXmlrpcXmlDataRepository();
        
        // or set all parameters manually
        dataRepository.setDbPassword("sshtelnet");
        dataRepository.setDbUsername("sshtelnet");
        dataRepository.setDbUri("xmldb:exist://localhost:8680/exist/xmlrpc");
        
        //create and init Storage Manager
        ExistDbXmlrpcXmlStorageManager storageManager = new 
ExistDbXmlrpcXmlStorageManager();
        
    
        //keepConnection=true means that the connection will be persistent
        storageManager.setKeepConnection(true);
        
        storageManager.initStorage(dataRepository);

        //run test cases
        testQuery(storageManager,"//nmwgt:subject");
        testQuery(storageManager,"//nmwgt:parameters");
        testQuery(storageManager,"//nmwgt:data");
        
    }
    
   
    // ------------------------------------------------------------------- 
end of test case
    
    
} //ExistDbXmlrpcXmlStorageManager

Attachment: logs.rar
Description: Binary data