perfsonar-dev - perfsonar: r5304 - in branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice: dao summarization
Subject: perfsonar development work
List archive
perfsonar: r5304 - in branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice: dao summarization
Chronological Thread
- From:
- To:
- Subject: perfsonar: r5304 - in branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice: dao summarization
- Date: Mon, 12 Oct 2009 05:57:51 -0400
Author: trzaszcz
Date: 2009-10-12 05:57:50 -0400 (Mon, 12 Oct 2009)
New Revision: 5304
Modified:
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/ISummarizationDAO.java
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/SummarizationDAO.java
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/summarization/LSSummarySynchronizationAction.java
Log:
performance enchancement
Modified:
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/ISummarizationDAO.java
===================================================================
---
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/ISummarizationDAO.java
2009-10-09 10:56:01 UTC (rev 5303)
+++
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/ISummarizationDAO.java
2009-10-12 09:57:50 UTC (rev 5304)
@@ -1,5 +1,7 @@
package org.perfsonar.service.lookupservice.dao;
+import java.util.List;
+
import org.perfsonar.base2.service.exceptions.PerfSONARException;
import org.perfsonar.base2.service.storage.xml.XMLDBResult;
import org.perfsonar.base2.xml.Element;
@@ -28,7 +30,26 @@
*/
void putSummaryData(Element m) throws PerfSONARException;
+
+
/**
+ * put batch data in few parts, where one part has maximal batchSize
number of elements
+ *
+ * @param data - list of data
+ * @param batchSize - size of the saved batch data
+ * @throws PerfSONARException
+ */
+ void putBatchData(List<Element> data,int batchSize)throws
PerfSONARException;
+
+ /**
+ * put batch of summarized data
+ *
+ * @param data
+ * @throws PerfSONARException
+ */
+ void putBatchData(List<Element> data)throws PerfSONARException;
+
+ /**
* Return set of IP addresses used by services
*
* @return
Modified:
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/SummarizationDAO.java
===================================================================
---
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/SummarizationDAO.java
2009-10-09 10:56:01 UTC (rev 5303)
+++
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/dao/SummarizationDAO.java
2009-10-12 09:57:50 UTC (rev 5304)
@@ -1,5 +1,8 @@
package org.perfsonar.service.lookupservice.dao;
+import java.util.ArrayList;
+import java.util.List;
+
import org.perfsonar.base2.service.exceptions.PerfSONARException;
import org.perfsonar.base2.service.storage.xml.XMLDBResult;
import org.perfsonar.base2.xml.Element;
@@ -22,10 +25,8 @@
StringBuffer sb = new StringBuffer();
-// sb.append(XQUERY_DECLARE_NAMESPACES);
+ // performance improvement
sb.append("xmldb:copy('/db/copy/', '/db/ls/',
'LSStore-summary.xml')");
-
-// .append("update delete
/nmwg:store[@type='LSStore-summary']/*");
getDataStorage().performXQuery(sb.toString(),
getDataCollection(),
getDataCredential());
@@ -124,4 +125,64 @@
return performXQuery(sb.toString());
}
+ @Override
+ public void putBatchData(List<Element> data) throws
PerfSONARException {
+ ElementSerializer ser = new ElementSerializer();
+ ser.setStartingDocumentDeclaration(false);
+ StringBuffer xmls = new StringBuffer();
+ StringBufferOutputStream os = new
StringBufferOutputStream(xmls);
+
+ // add xquery namespaces and UPDATE instruction
+ xmls.append(XQUERY_DECLARE_NAMESPACES);
+ xmls.append(UPDATE_INSERT);
+ xmls.append(" (\n");
+
+ for (int c = 0; c < data.size(); c++) {
+ Element element = data.get(c);
+ try {
+
+ ser.write(os, element);
+ if (c != data.size() - 1) {
+ xmls.append(", \n");
+ }
+
+ } catch (Exception e) {
+ throw new PerfSONARException("error/parse",
+ "Cannot serialize elements in
XQueryExistDAOImpl");
+ }
+ }
+ // send XQuery
+ xmls.append(" )\n");
+
+ xmls.append(UPDATE_INTO);
+ xmls.append(UPDATE_LSSTORE_SUMMARY);
+
+ String query = xmls.toString();
+ getDataStorage().performXQuery(query, getDataCollection(),
+ getDataCredential());
+
+ }
+
+ @Override
+ public void putBatchData(List<Element> data, int batchSize)
+ throws PerfSONARException {
+ List<Element> elements = new ArrayList<Element>();
+ int nrOfPart = 1;
+ for (int c = 0; c < data.size(); c++) {
+ if (nrOfPart * batchSize >= c) {
+ elements.add(data.get(c));
+ } else {
+ putBatchData(elements);
+ nrOfPart++;
+ elements = new ArrayList<Element>();
+ elements.add(data.get(c));
+ }
+ }
+
+ if (!elements.isEmpty()) {
+ putBatchData(elements);
+ }
+
+ }
+
}
Modified:
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/summarization/LSSummarySynchronizationAction.java
===================================================================
---
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/summarization/LSSummarySynchronizationAction.java
2009-10-09 10:56:01 UTC (rev 5303)
+++
branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice/summarization/LSSummarySynchronizationAction.java
2009-10-12 09:57:50 UTC (rev 5304)
@@ -2,7 +2,6 @@
import java.util.List;
-import org.apache.log4j.Logger;
import org.perfsonar.base2.service.configuration.Action;
import org.perfsonar.base2.service.configuration.ConfigurationManager;
import org.perfsonar.base2.service.exceptions.PerfSONARException;
@@ -32,8 +31,6 @@
private boolean firstIteration=true;
private ISummarizationDAO summarizationDao;
private SummarizationBuilder builder;
- private Logger logger = Logger
- .getLogger(LSSummarySynchronizationAction.class);
public LSSummarySynchronizationAction() throws PerfSONARException {
super();
@@ -62,12 +59,10 @@
NR_OF_MODIFICATIONS =
getOption("nrOfModifications").getValueAsInteger();
NR_OF_ITERATIONS =
getOption("nrOfIterations").getValueAsInteger();
-
logger.debug("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
try {
if (firstIteration) {
- // in first iteration summary data is build
- logger.debug(" ----------------FIRST
TIME------------------------");
+ // in first iteration summary data is built
synchronizeSummarizationCollection();
iteration++;
firstIteration=false;
@@ -76,7 +71,6 @@
if (nrOfChanges >= NR_OF_MODIFICATIONS) {
// if nr of changes is greater then
const NR_OF_MODIFICATIONS -> summary data must be refreshed
- logger.debug("------------ 1
-----------------");
synchronizeSummarizationCollection();
component.resetCounter();
}
@@ -86,7 +80,6 @@
iteration = 0;
//summarization is obligatory when there are
any differences
if (nrOfChanges != 0) {
- logger.debug("----------------------
2 ----------------");
synchronizeSummarizationCollection();
component.resetCounter();
}
@@ -97,22 +90,12 @@
} catch (PerfSONARException e1) {
e1.printStackTrace();
}
-
logger.debug("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
}
private void synchronizeSummarizationCollection() throws
PerfSONARException {
- logger.debug("building summarization data");
- long t1=System.currentTimeMillis();
summarizationDao.removeSummaryData();
-
System.out.println("1.+++++++++++++"+(System.currentTimeMillis()-t1));
- t1=System.currentTimeMillis();
List<Element> datas = builder.build();
-
System.out.println("2.+++++++++++++"+(System.currentTimeMillis()-t1));
- t1=System.currentTimeMillis();
- for (Element element : datas) {
- summarizationDao.putSummaryData(element);
- }
-
System.out.println("3.+++++++++++++"+(System.currentTimeMillis()-t1));
+ summarizationDao.putBatchData(datas, 100);
}
- perfsonar: r5304 - in branches/new-structure-with-base2/ps-mdm-ls/src/main/java/org/perfsonar/service/lookupservice: dao summarization, svnlog, 10/12/2009
Archive powered by MHonArc 2.6.16.