Ticket #1500: 0001-ticket-1500-Petascope-using-one-RasqlImplementation.patch

File 0001-ticket-1500-Petascope-using-one-RasqlImplementation.patch, 21.0 KB (added by Bang Pham Huu, 7 years ago)

patch which tried to test

  • applications/petascope/src/main/java/petascope/PetascopeInterface.java 

    From 410f959108eed645c995aa42010ff4d9c2e08362 Mon Sep 17 00:00:00 2001
From: Bang Pham Huu <b.phamhuu@jacobs-university.de>
Date: Thu, 2 Mar 2017 10:02:35 +0100
Subject: [PATCH] ticket:1500 - Petascope using one RasqlImplementation

---
 .../main/java/petascope/PetascopeInterface.java    |  11 +++
 .../main/java/petascope/rasdaman/RasqlServlet.java |  38 +++-----
 .../src/main/java/petascope/util/ras/RasUtil.java  | 106 +++++++++++++--------
 .../main/java/petascope/util/ras/TypeRegistry.java |   8 +-
 .../wcst/helpers/update/RasdamanValuesUpdater.java |   2 +-
 .../petascope/wms2/rasdaman/RasdamanService.java   |   2 +-
 6 files changed, 94 insertions(+), 73 deletions(-)

diff --git a/applications/petascope/src/main/java/petascope/PetascopeInterface.java b/applications/petascope/src/main/java/petascope/PetascopeInterface.java
index dd0c1a0..3f97b89 100644
    a b import java.util.HashMap;  
    3535import java.util.Map;
    3636import java.util.Map.Entry;
    3737import java.util.Set;
     38import java.util.logging.Level;
    3839import java.util.regex.Matcher;
    3940import javax.servlet.ServletException;
    4041import javax.servlet.http.HttpServletRequest;
    … … import petascope.util.WcpsConstants;  
    6667import petascope.util.XMLSymbols;
    6768import petascope.util.XMLUtil;
    6869import petascope.util.ras.RasQueryResult;
     70import petascope.util.ras.RasUtil;
    6971import petascope.util.response.MultipartResponse;
    7072import petascope.wcps.server.core.ProcessCoveragesRequest;
    7173import petascope.wcps.server.core.Wcps;
    … … public class PetascopeInterface extends CORSHttpServlet {  
    116118        } catch (RasdamanException ex) {
    117119            throw new ServletException(ex);
    118120        }
     121       
     122        try {
     123            // Initialize the singleton for Rasql Authentification (only one time)
     124            log.debug("Initialize Rasql Implementation");
     125            RasUtil.getRasImplementation();
     126        } catch (RasdamanException ex) {
     127            log.error("Could not initialize RasImplementation from configuration", ex);
     128            throw new ServletException(ex);           
     129        }
    119130
    120131        // Initialize WSC schema (NOTE: it will take 1 - 2 minutes, so only run when xml_validation=true)
    121132        if (ConfigManager.XML_VALIDATION) {

  • applications/petascope/src/main/java/petascope/rasdaman/RasqlServlet.java 

    diff --git a/applications/petascope/src/main/java/petascope/rasdaman/RasqlServlet.java b/applications/petascope/src/main/java/petascope/rasdaman/RasqlServlet.java
index 345dd2f..84ba92e 100644
    a b public class RasqlServlet extends CORSHttpServlet {  
    120120                return;
    121121            }
    122122
    123             Map<String, String> kvp = RequestUtil.parseKVPRequestParams(wrapperRequest.getQueryString());
    124             String username = kvp.get(KVPSymbols.KEY_USERNAME);
    125             String password = kvp.get(KVPSymbols.KEY_PASSWORD);
     123            Map<String, String> kvp = RequestUtil.parseKVPRequestParams(wrapperRequest.getQueryString());           
    126124            String query = kvp.get(KVPSymbols.KEY_QUERY);
    127125
    128126            // validate user before running the query
    129             request validRet = validateRequest(outStream, res, username, password, query);
     127            request validRet = validateRequest(outStream, res, query);
    130128            if (validRet == request.INVALID) {
    131129                // validation is error, return
    132130                return;
    … … public class RasqlServlet extends CORSHttpServlet {  
    149147            // then run the rasql query to Rasdaman
    150148            try {
    151149                // select, delete, update without decode()
    152                 executeQuery(outStream, res, username, password, query, filePath);
     150                executeQuery(outStream, res, query, filePath);
    153151            } catch (PetascopeException ex) {
    154152                printError(outStream, res, "Failed evaluating query: " + query, ex, USE_DEFAULT_STATUS);
    155153                log.error("Failed evaluating query: " + query, ex);
    … … public class RasqlServlet extends CORSHttpServlet {  
    221219
    222220    /**
    223221     * Execute rasql query.
    224      *
    225      * @param username rasdaman username
    226      * @param password rasdaman password
     222     *     
    227223     * @param query rasql query to execute
    228224     * @throws PetascopeException in case of error in query evaluation
    229225     */
    230     private void executeQuery(OutputStream os, HttpServletResponse res, String username,
    231                               String password, String query, String filePath) throws PetascopeException {
     226    private void executeQuery(OutputStream os, HttpServletResponse res, String query, String filePath) throws PetascopeException {
    232227        try {
    233228            Object tmpResult = null;
    234229
    … … public class RasqlServlet extends CORSHttpServlet {  
    236231                // no decode() or inv_*() in rasql query
    237232                if (RasUtil.isSelectQuery(query)) {
    238233                    // no need to open transcaction with "select" query
    239                     tmpResult = RasUtil.executeRasqlQuery(query, username, password, false);
     234                    tmpResult = RasUtil.executeRasqlQuery(query, false);
    240235                } else {
    241236                    // drop, delete need to open transacation
    242                     tmpResult = RasUtil.executeRasqlQuery(query, username, password, true);
     237                    tmpResult = RasUtil.executeRasqlQuery(query, true);
    243238                }
    244239            } else {
    245240                // decode() or inv_*() in rasql query
    246                 RasUtil.executeInsertUpdateFileStatement(query, filePath, username, password);
     241                RasUtil.executeInsertUpdateFileStatement(query, filePath,
     242                        ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS);
    247243            }
    248244            RasQueryResult queryResult = new RasQueryResult(tmpResult);
    249245
    … … public class RasqlServlet extends CORSHttpServlet {  
    278274    }
    279275
    280276    /**
    281      * Check if a request is valid (i.e: has username, password and query parameters)
    282      * @param username
    283      * @param password
     277     * Check if a request is valid (query parameters)     
    284278     * @param query
    285279     */
    286     private request validateRequest(OutputStream os, HttpServletResponse res, String username, String password, String query) {
    287         if (username == null) {
    288             printError(os, res, "required KVP parameter " +
    289                        KVPSymbols.KEY_USERNAME + " missing from request.", null, 400);
    290             return request.INVALID;
    291         }
    292         if (password == null) {
    293             printError(os, res, "required KVP parameter " +
    294                        KVPSymbols.KEY_PASSWORD + " missing from request.", null, 400);
    295             return request.INVALID;
    296         }
     280    private request validateRequest(OutputStream os, HttpServletResponse res, String query) {       
    297281        if (query == null) {
    298282            printError(os, res, "required KVP parameter " +
    299283                       KVPSymbols.KEY_QUERY + " missing from request.", null, 400);

  • applications/petascope/src/main/java/petascope/util/ras/RasUtil.java 

    diff --git a/applications/petascope/src/main/java/petascope/util/ras/RasUtil.java b/applications/petascope/src/main/java/petascope/util/ras/RasUtil.java
index 2e5352c..53b1513 100644
    a b import java.io.InputStreamReader;  
    2727import java.math.BigDecimal;
    2828import java.math.BigInteger;
    2929import java.util.Iterator;
     30import java.util.logging.Level;
    3031import java.util.regex.Matcher;
    3132import java.util.regex.Pattern;
    3233import org.antlr.runtime.ANTLRStringStream;
    … … public class RasUtil {  
    7677
    7778    // Useful patterns to extract data from the ``--out string'' RasQL output
    7879    private static final String VERSION_PATTERN = "rasdaman (\\S+)-\\S+ .*$"; // group _1_ is version
    79 
     80   
     81    private static RasImplementation rasqlImplementation = null;
     82   
     83    private static Database db;
     84   
     85    private static int HAVE_TO_INITIALIZE = 0;
     86   
     87   
    8088    /**
    81      * Execute a RasQL query with configured credentials.
     89     * Returns the instance of the ConfigManager. If no such instance exists,
     90     * it creates one with the specified settings file.
    8291     *
    83      * @param query
     92     * @param confDir Path to the settings file
     93     * @return instance of the ConfigManager class
    8494     * @throws RasdamanException
    8595     */
    86     public static Object executeRasqlQuery(String query) throws RasdamanException {
    87         return executeRasqlQuery(query, ConfigManager.RASDAMAN_USER, ConfigManager.RASDAMAN_PASS);
     96    public static RasImplementation getRasImplementation() throws RasdamanException {
     97        if (rasqlImplementation == null) {
     98            rasqlImplementation = new RasImplementation(ConfigManager.RASDAMAN_URL);
     99            rasqlImplementation.setUserIdentification(ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS);
     100           
     101            db = rasqlImplementation.newDatabase();
     102        }
     103        return rasqlImplementation;
     104    }   
     105   
     106    /**
     107     * If the client id is not fit with current RasImplementation, then must reinitialize them
     108     * @throws RasdamanException
     109     */
     110    public static void setRasqlImplementation() throws RasdamanException {
     111        //rasqlImplementation = new RasImplementation(ConfigManager.RASDAMAN_URL);
     112        rasqlImplementation.setUserIdentification(ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS);
    88113    }
    89 
     114 
    90115    /**
    91116     * Executes a RasQL query, allowing write transactions by setting the flag.
    92117     *
    93      * @param query
    94      * @param username
    95      * @param password
     118     * @param query     
    96119     * @param isWriteTransaction
    97120     * @return
    98121     * @throws RasdamanException
    99122     */
    100     public static Object executeRasqlQuery(String query, String username, String password, Boolean isWriteTransaction) throws RasdamanException {
    101         RasImplementation impl = new RasImplementation(ConfigManager.RASDAMAN_URL);
    102         impl.setUserIdentification(username, password);
    103         Database db = impl.newDatabase();
     123    public static Object executeRasqlQuery(String query, Boolean isWriteTransaction) throws RasdamanException {                   
     124        int openFlag = isWriteTransaction ? Database.OPEN_READ_WRITE : Database.OPEN_READ_ONLY;
     125        try {
     126            db.open(ConfigManager.RASDAMAN_DATABASE, openFlag);
     127        } catch (RuntimeException ex) {           
     128            log.debug("INITIALIZE NEW RASQL IMPLEMENTATION: " + HAVE_TO_INITIALIZE);
     129            HAVE_TO_INITIALIZE++;
     130            // Error when could not open new connection from RasqlImplementation, initialize new one
     131            setRasqlImplementation();
     132            // and open the database again
     133            //db = rasqlImplementation.newDatabase();           
     134            try {
     135                db.open(ConfigManager.RASDAMAN_DATABASE, openFlag);
     136            } catch (ODMGException ex1) {
     137                throw new RasdamanException(ExceptionCode.RasdamanUnavailable,
     138                                            "Unable to get a free rasdaman server.", ex1);
     139            }
     140           
     141        } catch (ODMGException ex) {
     142            throw new RasdamanException(ExceptionCode.RasdamanUnavailable,
     143                                            "Unable to get a free rasdaman server.", ex);
     144        }
     145       
    104146        int maxAttempts, timeout, attempts = 0;
    105147
    106148        //The result of the query will be assigned to ret
    … … public class RasUtil {  
    134176        while (!queryCompleted) {
    135177
    136178            //Try to obtain a free rasdaman server
    137             try {
    138                 int openFlag = isWriteTransaction ? Database.OPEN_READ_WRITE : Database.OPEN_READ_ONLY;
    139                 db.open(ConfigManager.RASDAMAN_DATABASE, openFlag);
     179            try {               
    140180                dbOpened = true;
    141                 tr = impl.newTransaction();
     181                tr = rasqlImplementation.newTransaction();
    142182                tr.begin();
    143                 OQLQuery q = impl.newOQLQuery();
     183                OQLQuery q = rasqlImplementation.newOQLQuery();
    144184
    145185                //A free rasdaman server was obtain, executing query
    146186                try {
    … … public class RasUtil {  
    205245                    throw new RasdamanException(ExceptionCode.RasdamanUnavailable,
    206246                                                "Unable to get a free rasdaman server.");
    207247                }
    208             } catch (ODMGException ex) {
    209 
    210                 //The maximum ammount of connection attempts was exceded
    211                 //and a connection could not be established. Return
    212                 //an exception indicating Rasdaman is unavailable.
    213                 log.error("A Rasdaman request could not be fullfilled since no "
    214                           + "free Rasdaman server were available. Consider adjusting "
    215                           + "the values of rasdaman_retry_attempts and rasdaman_retry_timeout "
    216                           + "or adding more Rasdaman servers.", ex);
    217 
    218                 throw new RasdamanException(ExceptionCode.RasdamanUnavailable,
    219                                             "Unable to get a free rasdaman server.");
    220248            } catch (RasClientInternalException ex) {
    221249                //when no rasdaman servers are started, rasj throws this type of exception
    222250                throw new RasdamanException(ExceptionCode.RasdamanUnavailable,
    … … public class RasUtil {  
    228256    }
    229257
    230258    /**
    231      * Execute a RasQL query with specified credentials, allowing only read
     259     * Execute a RasQL query allowing only read
    232260     * transactions.
    233261     *
    234      * @param query
    235      * @param username
    236      * @param password
     262     * @param query     
    237263     * @throws RasdamanException
    238264     */
    239265    // FIXME - should return just String?
    240     public static Object executeRasqlQuery(String query, String username, String password) throws RasdamanException {
    241         return executeRasqlQuery(query, username, password, false);
     266    public static Object executeRasqlQuery(String query) throws RasdamanException {
     267        return executeRasqlQuery(query, false);
    242268    }
    243269
    244270    /**
    … … public class RasUtil {  
    434460     */
    435461    public static void deleteFromRasdaman(BigInteger oid, String collectionName) throws RasdamanException {
    436462        String query = TEMPLATE_DELETE.replaceAll(TOKEN_COLLECTION_NAME, collectionName).replace(TOKEN_OID, oid.toString());
    437         executeRasqlQuery(query, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     463        executeRasqlQuery(query, true);
    438464        //check if there are other objects left in the collection
    439465        log.info("Checking the number of objects left in collection " + collectionName);
    440466        RasBag result = (RasBag) executeRasqlQuery(TEMPLATE_SDOM.replace(TOKEN_COLLECTION_NAME, collectionName));
    … … public class RasUtil {  
    442468        if (result.isEmpty()) {
    443469            //no object left, delete the collection so that the name can be reused in the future
    444470            log.info("No objects left in the collection, dropping the collection so the name can be reused in the future.");
    445             executeRasqlQuery(TEMPLATE_DROP_COLLECTION.replace(TOKEN_COLLECTION_NAME, collectionName), ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     471            executeRasqlQuery(TEMPLATE_DROP_COLLECTION.replace(TOKEN_COLLECTION_NAME, collectionName), true);
    446472        }
    447473    }
    448474
    … … public class RasUtil {  
    456482    public static void createRasdamanCollection(String collectionName, String collectionType) throws RasdamanException {
    457483        String query = TEMPLATE_CREATE_COLLECTION.replace(TOKEN_COLLECTION_NAME, collectionName)
    458484                       .replace(TOKEN_COLLECTION_TYPE, collectionType);
    459         executeRasqlQuery(query, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     485        executeRasqlQuery(query, true);
    460486    }
    461487
    462488    /**
    … … public class RasUtil {  
    472498        String tilingClause = (tiling == null || tiling.isEmpty()) ? "" : TILING_KEYWORD + " " + tiling;
    473499        String query = TEMPLATE_INSERT_VALUES.replace(TOKEN_COLLECTION_NAME, collectionName)
    474500                       .replace(TOKEN_VALUES, values).replace(TOKEN_TILING, tilingClause);
    475         executeRasqlQuery(query, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     501        executeRasqlQuery(query, true);
    476502        //get the collection oid
    477503        String oidQuery = TEMPLATE_SELECT_OID.replaceAll(TOKEN_COLLECTION_NAME, collectionName);
    478504        RasBag result = (RasBag) executeRasqlQuery(oidQuery);

  • applications/petascope/src/main/java/petascope/util/ras/TypeRegistry.java 

    diff --git a/applications/petascope/src/main/java/petascope/util/ras/TypeRegistry.java b/applications/petascope/src/main/java/petascope/util/ras/TypeRegistry.java
index d381921..36e1e1e 100644
    a b public class TypeRegistry {  
    145145                                 .replace("$typeStructure", bandBaseTypes.get(0))
    146146                                 .replace("$dimensions", expandDimensions(numberOfDimensions));
    147147            //create the marray type
    148             RasUtil.executeRasqlQuery(queryMarray, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     148            RasUtil.executeRasqlQuery(queryMarray, true);
    149149        } else {
    150150            //struct types
    151151            String structName = getRandomTypeName();
    152152            String queryStruct = QUERY_CREATE_STRUCT_TYPE.replace("$structTypeName", structName)
    153153                                 .replace("$structStructure", generateStructStructure(bandBaseTypes));
    154154            //create the struct type
    155             RasUtil.executeRasqlQuery(queryStruct, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     155            RasUtil.executeRasqlQuery(queryStruct, true);
    156156            //marray type
    157157            String queryMarray = QUERY_CREATE_MARRAY_TYPE.replace("$typeName", marrayName)
    158158                                 .replace("$typeStructure", structName)
    159159                                 .replace("$dimensions", expandDimensions(numberOfDimensions));
    160160            //create it
    161             RasUtil.executeRasqlQuery(queryMarray, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     161            RasUtil.executeRasqlQuery(queryMarray, true);
    162162        }
    163163
    164164        String querySet = QUERY_CREATE_SET_TYPE.replace("$typeName", setName)
    165165                          .replace("$marrayTypeName", marrayName)
    166166                          .replace("$nullValues", generateNullValuesRepresentation(nullValues));
    167167        //create it
    168         RasUtil.executeRasqlQuery(querySet, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     168        RasUtil.executeRasqlQuery(querySet, true);
    169169        this.reinitialize();
    170170        return setName;
    171171    }

  • applications/petascope/src/main/java/petascope/wcs2/handlers/wcst/helpers/update/RasdamanValuesUpdater.java 

    diff --git a/applications/petascope/src/main/java/petascope/wcs2/handlers/wcst/helpers/update/RasdamanValuesUpdater.java b/applications/petascope/src/main/java/petascope/wcs2/handlers/wcst/helpers/update/RasdamanValuesUpdater.java
index 539fcc7..4e86ffd 100644
    a b public class RasdamanValuesUpdater implements RasdamanUpdater {  
    6161                             .replace("$oid", affectedCollectionOid)
    6262                             .replace("$values", values)
    6363                             .replace("$shiftDomain", shiftDomain);
    64         RasUtil.executeRasqlQuery(queryString, ConfigManager.RASDAMAN_ADMIN_USER, ConfigManager.RASDAMAN_ADMIN_PASS, true);
     64        RasUtil.executeRasqlQuery(queryString, true);
    6565    }
    6666
    6767    private static final String UPDATE_TEMPLATE_VALUES = "UPDATE $collection SET $collection$domain ASSIGN shift($values, $shiftDomain) WHERE oid($collection) = $oid";

  • applications/petascope/src/main/java/petascope/wms2/rasdaman/RasdamanService.java 

    diff --git a/applications/petascope/src/main/java/petascope/wms2/rasdaman/RasdamanService.java b/applications/petascope/src/main/java/petascope/wms2/rasdaman/RasdamanService.java
index 0adeff0..4763fb7 100644
    a b public class RasdamanService {  
    5959        //as it reconnects to rasdaman each time a query is executed. Ideally we should establish a number of
    6060        //connections and use them so that we eliminate any overhead of the connection
    6161        try {
    62             RasQueryResult result = new RasQueryResult(RasUtil.executeRasqlQuery(query, config.getRasdamanUser(), config.getRasdamanPassword()));
     62            RasQueryResult result = new RasQueryResult(RasUtil.executeRasqlQuery(query));
    6363
    6464            if (result.getMdds() == null || result.getMdds().size() == 0) {
    6565                throw new WMSDataStoreException();