// CentralDigestCachedImpl.java
//
// Created: September 2004
//
// This file is a component of the BioMoby project.
// Copyright Martin Senger (martin.senger@gmail.com).
//

package org.biomoby.client;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.Vector;

import org.biomoby.registry.meta.Registry;
import org.biomoby.shared.Central;
import org.biomoby.shared.MobyDataType;
import org.biomoby.shared.MobyException;
import org.biomoby.shared.MobyNamespace;
import org.biomoby.shared.MobyPrimaryData;
import org.biomoby.shared.MobyPrimaryDataSet;
import org.biomoby.shared.MobyPrimaryDataSimple;
import org.biomoby.shared.MobyRelationship;
import org.biomoby.shared.MobyResourceRef;
import org.biomoby.shared.MobySecondaryData;
import org.biomoby.shared.MobyService;
import org.biomoby.shared.MobyServiceType;
import org.biomoby.shared.NoSuccessException;
import org.biomoby.shared.extended.DataTypeParser;
import org.biomoby.shared.extended.ServiceInstanceParser;
import org.biomoby.shared.extended.ServiceTypeParser;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

/**
 * An implementation of {@link org.biomoby.shared.CentralAll}, allowing to
 * cache locally results of the cumulative methods so it does not need to access
 * Moby registry all the time. The other methods of the Central interface do not
 * use the results of the cached cumulative results (their implementation is
 * just passed to the parent class).
 * <p>
 * 
 * The caching is done in the file system, not in memory, so the results are
 * permanent (until someone removes the caching directory, or calls
 * {@link #removeFromCache}).
 * <p>
 * 
 * This class can be used also without caching - just instantiate it with
 * 'cacheDir' set to null in the constructor.
 * <p>
 * 
 * @author <A HREF="mailto:martin.senger@gmail.com">Martin Senger</A>
 * @version $Id: CentralDigestCachedImpl.java,v 1.33 2008/07/16 20:25:24 gordonp Exp $
 */
public class CentralDigestCachedImpl
    extends CentralDigestCachedSimpleImpl {

    private static org.apache.commons.logging.Log log =
	org.apache.commons.logging.LogFactory.getLog(CentralDigestCachedImpl.class);

    private int datatype_threshold ;
    private int service_threshold ;
    private int service_type_threshold ;

    /**
     * The same as calling CentralDigestCachedImpl(null)
     */
    public CentralDigestCachedImpl () throws MobyException{
	this(null);
    }
    
    /***************************************************************************
     * Create an instance that will access a default Moby registry and
     * will cache results in the 'cacheDir' directory.  <p>
     **************************************************************************/
    public CentralDigestCachedImpl (String cacheDir)
	throws MobyException {
	this (null, null, cacheDir);
    }

    /***************************************************************************
     * Create an instance that will access a specific Moby registry and
     * will cache results in the default cache directory.  <p>
     **************************************************************************/
    public CentralDigestCachedImpl (String endpoint, String namespace)
	throws MobyException{
	this (endpoint, namespace, null);
    }

    /***************************************************************************
     * Create an instance that will access a Moby registry defined by
     * its 'endpoint' and 'namespace', and will cache results in the
     * 'cacheDir' directory. Note that the same 'cacheDir' can be
     * safely used for more Moby registries.  <p>
     **************************************************************************/
    public CentralDigestCachedImpl (String endpoint, String namespace, String cacheDir)
	throws MobyException {
	super (endpoint, namespace, cacheDir);
	
	datatype_threshold = Integer.getInteger("cache.threshold.datatypes", 7).intValue();
	if (datatype_threshold < 0 || datatype_threshold > 100)
	    datatype_threshold = 7;
	service_threshold = Integer.getInteger("cache.threshold.services", 30).intValue();
	if (service_threshold < 0 || service_threshold > 100)
	    service_threshold = 30;
	service_type_threshold = Integer.getInteger("cache.threshold.servicetypes", 5).intValue();
	if (service_type_threshold < 0 || service_type_threshold > 100)
	    service_type_threshold = 5;
	

    }

    /***************************************************************************
     * Update data types from a moby registry: - get a new LIST_FILE (but do not
     * put it into the cache yet) if failed do nothing (except reporting it) -
     * remove LIST_FILE - compare contents of new LIST_FILE with file names in
     * the cache and remove them, or fetched missing ones if success add there
     * new LIST_FILE
     **************************************************************************/
    protected boolean fillDataTypesCache()
	throws MobyException {
	try {
	    
	    if (isCacheEmpty(dataTypesCache)) {
		fireEvent (DATA_TYPES_START);
		// download RDF and parse it into individual datatypes
		DataTypeParser sip = new DataTypeParser(getResourceURL(DATA_TYPES_RESOURCE_NAME));
                MobyDataType[] datatypes = sip.getMobyDataTypesFromRDF();
                fireEvent (DATA_TYPES_COUNT, new Integer (datatypes.length));
                // create a map of name => to String representation as is with retrieveObjectDefinition
                Map<String,String> map = createRetrieveObjectXML(datatypes);
                for (String datatype : map.keySet()) {
                    fireEvent (DATA_TYPE_LOADING, datatype);
                    String filecontents = map.get(datatype);
                    store(dataTypesCache, datatype, filecontents);
                    fireEvent (DATA_TYPE_LOADED, datatype);
                }
                // finally, put there the new LIST_FILE
                store(dataTypesCache, LIST_FILE, getDataTypeNamesAsXML());
                // done
                return true;
	    }
	    
	    fireEvent (DATA_TYPES_START);
	    String typesAsXML = getDataTypeNamesAsXML();

	    // get a list file with all data type names currently in
	    // the cache...
	    Map cachedTypes = new HashMap();
	    String xmlList = getListFile (dataTypesCache);
	    if (xmlList != null)
		cachedTypes = createDataTypeNamesFromXML (xmlList, false);

	    // ...and remove it
	    remove (dataTypesCache, LIST_FILE);

	    // get a list file with all data types from the registry
	    Map types = createDataTypeNamesFromXML (typesAsXML, false);
	    fireEvent (DATA_TYPES_COUNT, new Integer (types.size()));

	    // list of current files in this cache
	    HashSet currentFiles = new HashSet();
	    File[] list = dataTypesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (dataTypesCache));
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i]))
		    currentFiles.add (list[i].getName());
	    }
	    // a list of datatypes needed to fetch from the registry
	    ArrayList<String> datatypesToFetch = new ArrayList<String>();
	    
	    // iterate over LIST_FILE and fetch missing files
	    for (Iterator it = types.entrySet().iterator(); it.hasNext(); ) {
		Map.Entry entry = (Map.Entry)it.next();
		boolean needToFetch = false;
		String name = (String)entry.getKey();
		if ( ! currentFiles.contains (name)) {
		    // missing file
		    needToFetch = true;
		} else {
		    // check by comparing LSIDs
		    MobyDataType dt = (MobyDataType)entry.getValue();
		    String lsid = dt.getLSID();
		    if (cachedTypes.containsKey (name)) {
			// should always go here - or we have a broken cache, anyway
			String cachedLSID =
			    ( (MobyDataType)cachedTypes.get (name) ).getLSID();
			if (! lsid.equals (cachedLSID)) {
			    needToFetch = true;
			}
		    } else {
			needToFetch = true;
		    }
		}
		if (needToFetch) {
		    // missing file: add to fetch it from a registry list 
		    datatypesToFetch.add(name);
		}
		currentFiles.remove (name);
	    }
	    
	    if (((datatypesToFetch.size()*100) / types.size()) > datatype_threshold) {
		// download rdf instead of making individual calls to central
		DataTypeParser sip = new DataTypeParser(getResourceURL(DATA_TYPES_RESOURCE_NAME));
                MobyDataType[] datatypes = sip.getMobyDataTypesFromRDF();
                // create a map of name => to String representation as is with retrieveObjectDefinition
                Map<String,String> map = createRetrieveObjectXML(datatypes);
                for (String datatype : datatypesToFetch) {
                    if (!map.containsKey(datatype)) {
                	log.warn("'" + datatype + "' was not found in the RDF ...");
                	continue;
                    }
                    fireEvent (DATA_TYPE_LOADING, datatype);
                    String filecontents = map.get(datatype);
                    store(dataTypesCache, datatype, filecontents);
                    fireEvent (DATA_TYPE_LOADED, datatype);
                    if (stopDT) {
			return false;
		    }
                }
	    } else {
		// use api to get datatypes
		for (String name : datatypesToFetch) {
		    fireEvent (DATA_TYPE_LOADING, name);
		    String xml = getDataTypeAsXML (name);
		    store (dataTypesCache, name, xml);
		    fireEvent (DATA_TYPE_LOADED, name);
		    if (stopDT) {
			return false;
		    }
		}
	    }
	    
	    // remove files that are not any more needed
	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
		remove (dataTypesCache, (String)it.next());

	    // finally, put there the new LIST_FILE
	    store (dataTypesCache, LIST_FILE, typesAsXML);
	    return true;

	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (stopDT ? DATA_TYPES_CANCELLED : DATA_TYPES_END);
	    stopDT = false;
	}
}

    /***************************************************************************
     * Update services from a moby registry: - get a new LIST_FILE (but do not
     * put it into the cache yet) if failed do nothing (except reporting it) -
     * remove LIST_FILE - compare contents of new LIST_FILE with file names in
     * the cache and remove them, or fetched missing ones; in order to compare
     * properly you need to read individual files and look if they really
     * contain all services mentioned in the LIST_FILE if success add there new
     * LIST_FILE
     **************************************************************************/
    protected boolean fillServicesCache()
	throws MobyException {
	try {
	    if (isCacheEmpty(servicesCache)) {
		fireEvent (AUTHORITIES_START);
		// get a list file
                String byAuthorityAsXML = getServiceNamesByAuthorityAsXML();
                
		// download RDF and fill it up
		ServiceInstanceParser sip = new ServiceInstanceParser(getResourceURL(SERVICE_INSTANCES_RESOURCE_NAME));
                MobyService[] services = sip.getMobyServicesFromRDF();
                // sort the services
                Map<String, ArrayList<MobyService>> sorted = new HashMap<String, ArrayList<MobyService>>();
                for (MobyService s : services) {
                    ArrayList<MobyService> list = (sorted.containsKey(s
                            .getAuthority()) ? sorted.remove(s
                            .getAuthority()) : new ArrayList<MobyService>());
                    list.add(s);
                    sorted.put(s.getAuthority(), list);
                }
                fireEvent (AUTHORITIES_COUNT, new Integer (sorted.size()));
                // free memory
                services = null;
                for (String authURI : sorted.keySet()) {
                    fireEvent (AUTHORITY_LOADING, authURI);
                    String s = createServiceXML(sorted.get(authURI).toArray(
                                    new MobyService[] {}));
                    store(servicesCache, authURI, s);
                    fireEvent (AUTHORITY_LOADED, authURI);
                }
                // free more memory
                sorted.clear();
                
                // finally, store the new LIST_FILE
                store(servicesCache, LIST_FILE, byAuthorityAsXML);
                // done
                return true;
	    }
	    
	    fireEvent (AUTHORITIES_START);
	    String byAuthorityAsXML = getServiceNamesByAuthorityAsXML();
	    remove (servicesCache, LIST_FILE);
	    Map authorities = createServicesByAuthorityFromXML (byAuthorityAsXML,
								false);
	    // list of current files in this cache
	    HashSet currentFiles = new HashSet();
	    File[] list = servicesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (servicesCache));
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i]))
		    currentFiles.add (list[i].getName());
	    }
	    // a list of authorities with services needed to fetch from the registry
	    ArrayList<String> servicesToFetch = new ArrayList<String>();

	    // iterate over LIST_FILE and fetch missing files
	    fireEvent (AUTHORITIES_COUNT, new Integer (authorities.size()));
	    for (Iterator it = authorities.entrySet().iterator(); it.hasNext(); ) {
		Map.Entry entry = (Map.Entry)it.next();
		String authority = (String)entry.getKey();
		if (currentFiles.contains (authority)) {
		    MobyService[] servs =
			extractServices (load (new File (servicesCache, authority)));
		    // compare names in 'servs' (those are services we have in cache)
		    // with names in 'entry' (those are the ones we should have)
		    boolean theyAreEqual = true;
		    HashMap currentServices = new HashMap (servs.length);
		    for (int i = 0; i < servs.length; i++)
			currentServices.put (servs[i].getName(), servs[i]);
		    MobyService[] newServices = (MobyService[])entry.getValue();
		    for (int i = 0; i < newServices.length; i++) {
			String currName = newServices[i].getName();
			if (currentServices.containsKey (currName)) {
			    // check whether the old and new ones have the same LSID
			    MobyService current = (MobyService)currentServices.get (currName);
			    if (newServices[i].getLSID().equals (current.getLSID())) {
				currentServices.remove (currName);
			    } else {
				theyAreEqual = false;
			    }
			} else {
			    theyAreEqual = false;
			    break;
			}
		    }
		    if (currentServices.size() > 0)
			theyAreEqual = false;
		    if (! theyAreEqual)
			currentFiles.remove (authority);
		}

		if (! currentFiles.contains (authority)) {
		    // missing file: fetch it from a registry
		    servicesToFetch.add(authority);
		} else {
		    currentFiles.remove (authority);
		}
	    }
	    
	    if (((servicesToFetch.size() *100) / authorities.size()) > service_threshold) {
		// download and process rdf
		ServiceInstanceParser sip = new ServiceInstanceParser(getResourceURL(SERVICE_INSTANCES_RESOURCE_NAME));
                MobyService[] services = sip.getMobyServicesFromRDF();
                // sort the services
                Map<String, ArrayList<MobyService>> sorted = new HashMap<String, ArrayList<MobyService>>();
                for (MobyService s : services) {
                    ArrayList<MobyService> al = (sorted.containsKey(s
                            .getAuthority()) ? sorted.remove(s
                            .getAuthority()) : new ArrayList<MobyService>());
                    al.add(s);
                    sorted.put(s.getAuthority(), al);
                }
                // free memory
                services = null;
                for (String authURI : servicesToFetch) {
                    if (!sorted.containsKey(authURI)) {
                	log.warn("'" + authURI + "' was not found in the RDF oddly enough ...");
                	continue;
                    }
                    fireEvent (AUTHORITY_LOADING, authURI);
                    String s = createServiceXML(sorted.get(authURI).toArray(new MobyService[] {}));
                    store(servicesCache, authURI, s);
                    fireEvent (AUTHORITY_LOADED, authURI);
                    if (stopS) {
			return false;
		    }
                }
	    } else {
		// process authorities one by one
		for (String authority : servicesToFetch) {
		    fireEvent (AUTHORITY_LOADING, authority);
		    MobyService pattern = new MobyService (MobyService.DUMMY_NAME, authority);
		    pattern.setCategory ("");
		    String xml = getServicesAsXML (pattern, null, true, true);
		    store (servicesCache, authority, xml);
		    fireEvent (AUTHORITY_LOADED, authority);
		    if (stopS) {
			return false;
		    }
		}
	    }
	    

	    // remove files that are not any more needed
	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
		remove (servicesCache, (String)it.next());

	    // finally, put there the new LIST_FILE
	    store (servicesCache, LIST_FILE, byAuthorityAsXML);
	    return true;

	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (stopS ? AUTHORITIES_CANCELLED : AUTHORITIES_END);
	    stopS = false;
	}
    }
 
    /*
     * Given a resource name, go to registry and get the URL for it
     */
    private URL getResourceURL (String resourceName)
	throws MobyException {
	MobyResourceRef[] resourceRefs = getResourceRefs();
	for (int i = 0; i < resourceRefs.length; i++) {
	    if (resourceName.equalsIgnoreCase (resourceRefs[i].getResourceName())) {
		URL url = resourceRefs[i].getResourceLocation();
		    return url;
	    }
	}
	throw new MobyException ("No resource found for '" + resourceName + "'.");
    }
    private String createServiceXML(MobyService[] services) {
	StringBuffer sb = new StringBuffer();
	sb.append("<Services>\n");
	for (MobyService s : services) {
	    sb.append("<Service authURI=\""+s.getAuthority()+"\" serviceName=\""+s.getName()+"\" lsid=\""+s.getLSID()+"\">\n");
	    sb.append("<serviceType lsid=\""+s.getServiceType().getLSID()+"\">"+ s.getServiceType().getName()+"</serviceType>\n");
	    sb.append("<authoritative>"+(s.isAuthoritative() ? "1" : "0")+"</authoritative>\n" + 
	    		"<Category>"+s.getCategory()+"</Category>\n" + 
	    		"<Description><![CDATA["+s.getDescription()+"]]></Description>\n" + 
	    		"<contactEmail>"+s.getEmailContact()+"</contactEmail>\n" + 
	    		"<signatureURL>"+s.getSignatureURL().replaceAll("&","&amp;")+"</signatureURL>\n" + 
	    		"<URL>"+s.getURL().replaceAll("&","&amp;")+"</URL>\n");
	    // process inputs
	    sb.append("<Input>\n");
	    for (MobyPrimaryData in : s.getPrimaryInputs()) {
		if (in instanceof MobyPrimaryDataSimple) {
		    sb.append("<Simple articleName=\""+in.getName()+"\">\n");
		    sb.append("<objectType lsid=\"" + (in.getDataType().getLSID() == null ? "" : in.getDataType().getLSID()) + "\">" + in.getDataType().getName() + "</objectType>\n");
		    for (MobyNamespace namespace : in.getNamespaces()) {
			sb.append("<Namespace lsid=\""+(namespace.getLSID() == null ? "" : namespace.getLSID())+"\">"+namespace.getName()+"</Namespace>\n");
		    }
		    sb.append("</Simple>\n");
		} else {
		    sb.append("<Collection articleName=\""+ in.getName() +"\">\n");
		    for (MobyPrimaryDataSimple sim : ((MobyPrimaryDataSet)in).getElements()) {
			sb.append("<Simple articleName=\"\">\n");
			sb.append("<objectType lsid=\"" + (sim.getDataType().getLSID() == null ? "" : sim.getDataType().getLSID()) + "\">" + sim.getDataType().getName() + "</objectType>\n");
			for (MobyNamespace namespace : sim.getNamespaces()) {
			    sb.append("<Namespace lsid=\""+(namespace.getLSID() == null ? "" : namespace.getLSID())+"\">"+namespace.getName()+"</Namespace>\n");
			}
			sb.append("</Simple>\n");
		    }
		    sb.append("</Collection>\n");
		}
	    }
	    sb.append("</Input>\n");
	    // process outputs
	    sb.append("<Output>\n");
	    for (MobyPrimaryData out : s.getPrimaryOutputs()) {
		if (out instanceof MobyPrimaryDataSimple) {
		    sb.append("<Simple articleName=\""+out.getName()+"\">\n");
		    sb.append("<objectType lsid=\"" + (out.getDataType().getLSID() == null ? "" : out.getDataType().getLSID()) + "\">" + out.getDataType().getName() + "</objectType>\n");
		    for (MobyNamespace namespace : out.getNamespaces()) {
			sb.append("<Namespace lsid=\""+(namespace.getLSID() == null ? "" : namespace.getLSID())+"\">"+namespace.getName()+"</Namespace>\n");
		    }
		    sb.append("</Simple>\n");
		} else {
		    sb.append("<Collection articleName=\""+ out.getName() +"\">\n");
		    for (MobyPrimaryDataSimple sim : ((MobyPrimaryDataSet)out).getElements()) {
			sb.append("<Simple articleName=\"\">\n");
			sb.append("<objectType lsid=\"" + (sim.getDataType().getLSID() == null ? "" : sim.getDataType().getLSID()) + "\">" + sim.getDataType().getName() + "</objectType>\n");
			for (MobyNamespace namespace : sim.getNamespaces()) {
			    sb.append("<Namespace lsid=\""+(namespace.getLSID() == null ? "" : namespace.getLSID())+"\">"+namespace.getName()+"</Namespace>\n");
			}
			sb.append("</Simple>\n");
		    }
		    sb.append("</Collection>\n");
		}
	    }
	    sb.append("</Output>\n");
	    // process secondaries
	    sb.append("<secondaryArticles>\n");
	    for (MobySecondaryData sec : s.getSecondaryInputs()) {
		sb.append(sec.toXML());
	    }
	    sb.append("</secondaryArticles>\n");
	    
	    // finished with the service
	    sb.append("</Service>\n");
	}
	sb.append("</Services>");
	return sb.toString();
    }
    
    private Map<String, String> createRetrieveObjectXML(MobyDataType[] datatypes) {
	HashMap<String, String> map = new HashMap<String,String>();
	
	
	for (MobyDataType d : datatypes) {
	    StringBuffer sb = new StringBuffer();
	    sb.append("<retrieveObjectDefinition>\n");
	    sb.append("<objectType lsid=\'"+d.getLSID()+"\'>"+d.getName()+"</objectType>\n" + 
	    		"<Description><![CDATA[" + d.getDescription() + "]]></Description>\n" + 
	    		"<authURI>"+d.getAuthority()+"</authURI>\n" + 
	    		"<contactEmail>"+d.getEmailContact()+"</contactEmail>\n");
	    // process HAS/HASA
	    ArrayList<String> has = new ArrayList<String>();
	    ArrayList<String> hasa = new ArrayList<String>();
	    for (MobyRelationship relationship : d.getChildren()) {
		if (relationship.getRelationshipType() == Central.iHAS) {
		    has.add("<objectType articleName=\'"+relationship.getName()+"\' lsid=\'\'>"+relationship.getDataTypeName()+"</objectType>\n");
		} else {
		    hasa.add("<objectType articleName=\'"+relationship.getName()+"\' lsid=\'\'>"+relationship.getDataTypeName()+"</objectType>\n");
		}
	    }
	    if (has.size() > 0) {
		sb.append("<Relationship relationshipType=\'urn:lsid:biomoby.org:objectrelation:has\'>\n");
		for (String s : has) {
		    sb.append(s);
		}
		sb.append("</Relationship>\n");
	    }
	    has = null;
	    if (hasa.size() > 0) {
		sb.append("<Relationship relationshipType=\'urn:lsid:biomoby.org:objectrelation:hasa\'>\n");
		for (String s : hasa) {
		    sb.append(s);
		}
		sb.append("</Relationship>\n");
	    }
	    hasa = null;
	    // set the isa - if it exists
	    if (!d.getParentName().equals(""))
		sb.append("<Relationship relationshipType=\'urn:lsid:biomoby.org:objectrelation:isa\'>\n" + 
			  "<objectType articleName=\'\' lsid=\'\'>" + d.getParentName() + "</objectType>\n" + 
	    		  "</Relationship>\n");
	    sb.append("</retrieveObjectDefinition>\n");
	    map.put(d.getName(), sb.toString());
	}
	return map;
    }
    
    /***************************************************************************
     * Update service types from a moby registry: - get a new LIST_FILE (but do
     * not put it into the cache yet) if failed do nothing (except reporting it) -
     * remove LIST_FILE - compare contents of new LIST_FILE with file names in
     * the cache and remove them, or fetched missing ones if success add there
     * new LIST_FILE
     **************************************************************************/
    protected boolean fillServiceTypesCache()
	throws MobyException {
	try {
	    
	    if (isCacheEmpty(serviceTypesCache)) {
		fireEvent (SERVICE_TYPES_START);
		// get a list file
                String byAuthorityAsXML = getServiceTypesAsXML();
                
		// download RDF and fill it up
		ServiceTypeParser sip = new ServiceTypeParser(getResourceURL(SERVICE_TYPES_RESOURCE_NAME));
                MobyServiceType[] services = sip.getMobyServiceTypesFromRDF();
                
                fireEvent (SERVICE_TYPES_COUNT, new Integer (services.length));
                for (MobyServiceType st : services) {
                    fireEvent (SERVICE_TYPE_LOADING, st.getName());
                    String s = createServiceTypeXML(st);
                    store(serviceTypesCache, st.getName(), s);
                    fireEvent (SERVICE_TYPE_LOADED, st.getName());
                }
                // process the root node
                fireEvent (SERVICE_TYPE_LOADING, "Service");
                String s = createServiceTypeXML(new MobyServiceType("Service"));
                store(serviceTypesCache, "Service", s);
                fireEvent (SERVICE_TYPE_LOADED, "Service");
                
                // finally, store the new LIST_FILE
                store(serviceTypesCache, LIST_FILE, byAuthorityAsXML);
                // done
                return true;
	    }
	    
	    fireEvent (SERVICE_TYPES_START);
	    String typesAsXML = getServiceTypesAsXML();

	    // get a list file with all service type names currently
	    // in the cache...
	    MobyServiceType[] cachedList = new MobyServiceType[] {};
	    String xmlList = getListFile (serviceTypesCache);
	    if (xmlList != null)
		cachedList = createServiceTypesFromXML (xmlList);

	    HashMap cachedTypes = new HashMap();
	    for (int i = 0; i < cachedList.length; i++) {
		cachedTypes.put (cachedList[i].getName(), cachedList[i]);
	    }

	    // ...and remove it
	    remove (serviceTypesCache, LIST_FILE);

	    // get a list file with all service types from the
	    // registry
	    MobyServiceType[] types = createServiceTypesFromXML (typesAsXML);
	    fireEvent (SERVICE_TYPES_COUNT, new Integer (types.length));

	    // list of current files in this cache
	    HashSet currentFiles = new HashSet();
	    File[] list = serviceTypesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (serviceTypesCache));
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i]))
		    currentFiles.add (list[i].getName());
	    }
	    // a list of service types needed to fetch from the registry
	    ArrayList<String> serviceTypesToFetch = new ArrayList<String>();
	    // iterate over LIST_FILE and fetch missing files
	    for (int i = 0 ; i < types.length; i++) {
		boolean needToFetch = false;
		String name = types[i].getName();
		if ( ! currentFiles.contains (name)) {
		    // missing file
		    needToFetch = true;
		} else {
		    // check by comparing LSIDs
		    String lsid = types[i].getLSID();
		    if (cachedTypes.containsKey (name)) {
			// should always go here - or we have a broken cache, anyway
			String cachedLSID =
			    ( (MobyServiceType)cachedTypes.get (name) ).getLSID();
			if (! lsid.equals (cachedLSID)) {
			    needToFetch = true;
			}
		    } else {
			needToFetch = true;
		    }
		}
		if (needToFetch) {
		    serviceTypesToFetch.add(name);
		}
		currentFiles.remove (name);
	    }

	    if (((serviceTypesToFetch.size() *100)/types.length) > service_type_threshold) {
		// download RDF and fill it up
		ServiceTypeParser sip = new ServiceTypeParser(getResourceURL(SERVICE_TYPES_RESOURCE_NAME));
                MobyServiceType[] services = sip.getMobyServiceTypesFromRDF();
		fireEvent (SERVICE_TYPES_COUNT, new Integer (services.length));
	        for (MobyServiceType st : services) {
	            fireEvent(SERVICE_TYPE_LOADING, st.getName());
		    // only process new ones
	            if (serviceTypesToFetch.contains(st.getName())) {
			String s = createServiceTypeXML(st);
		    	store(serviceTypesCache, st.getName(), s);
		    }
		    fireEvent(SERVICE_TYPE_LOADED, st.getName());
	        }
		
	    } else {
		// use multiple api calls
		for (String name : serviceTypesToFetch) {
		    fireEvent (SERVICE_TYPE_LOADING, name);
		    String xml = getServiceTypeRelationshipsAsXML(name, false);
		    store(serviceTypesCache, name, xml);
		    fireEvent(SERVICE_TYPE_LOADED, name);
    		    if (stopST) {
    			log.warn("Service types cache not fully updated");
    			return false;
    		    }
		}
	    }
	    
	    // remove files that are not any more needed
	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
		remove (serviceTypesCache, (String)it.next());

	    // finally, put there the new LIST_FILE
	    store (serviceTypesCache, LIST_FILE, typesAsXML);
	    return true;

	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (stopST ? SERVICE_TYPES_CANCELLED :SERVICE_TYPES_END);
	    stopST = false;
	}
}

    private String createServiceTypeXML(MobyServiceType st) {
	return "<Relationships>\n" + 
		(st.getName().equals("Service") ? 
			"" 
			: 
			"<Relationship relationshipType=\'ISA\' lsid=\'urn:lsid:biomoby.org:servicerelation:isa\'>\n" + 
			"<serviceType lsid=\'\' >"+ st.getParentName() +"</serviceType>\n" + 
			"</Relationship>\n" ) + 
		"</Relationships>\n";
    }

    /***************************************************************************
     * Update namespaces from a moby registry - this is easier than with other
     * entities: just get a new LIST_FILE.
     **************************************************************************/
    protected boolean fillNamespacesCache()
	throws MobyException {
	try {
	    fireEvent (NAMESPACES_START);
	    String xml = getNamespacesAsXML();
	    store (namespacesCache, LIST_FILE, xml);
	    return true;
	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (NAMESPACES_END);
	}
}
    
    /*************************************************************************
    *
    *************************************************************************/
   public Map getDataTypeNames()
	throws MobyException {
	if (dataTypesCache == null)
	    return super.getDataTypeNames();
	synchronized (dataTypesCache) {
	    if (isCacheEmpty (dataTypesCache)) {
		initCache();
		if (! fillDataTypesCache())
		    // callback stopped filling
		    return new TreeMap();
	    }

	    // get a list file (with all data type names)
	    String xmlList = getListFile (dataTypesCache);
	    if (xmlList == null) {
		initCache();
		if (! fillDataTypesCache())
		    // callback stopped filling
		    return new TreeMap();
		else {
		    xmlList = getListFile (dataTypesCache);
		    if (xmlList == null)
			return new TreeMap();
		}
	    }
	    return createDataTypeNamesFromXML (xmlList, true);
	}
   }
    /***************************************************************************
     * 
     **************************************************************************/
    public MobyDataType[] getDataTypes()
	throws MobyException {
	if (dataTypesCache == null)
	    return super.getDataTypes();
	synchronized (dataTypesCache) {
	    Vector v = new Vector();
	    if (isCacheEmpty (dataTypesCache)) {
		initCache();
		if (! fillDataTypesCache())
		    // callback stopped filling
		    return new MobyDataType[] {};
	    }
	    File[] list = dataTypesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (dataTypesCache));
	    Arrays.sort (list, getFileComparator());

	    for (int i = 0; i < list.length; i++) {
		try {
		    if (ignored (list[i])) continue;
		    v.addElement (createDataTypeFromXML (load (list[i]), "-dummy-"));
		} catch (NoSuccessException e) {
			log.error (MSG_CACHE_BAD_FILE (list[i], e));
			//System.err.println (MSG_CACHE_BAD_FILE (list[i], e));
		}
	    }
	    MobyDataType[] result = new MobyDataType [v.size()];
	    v.copyInto (result);
	    return result;
	}
}

    /***************************************************************************
     * 
     **************************************************************************/
    public Map getServiceNamesByAuthority() throws MobyException {
	if (servicesCache == null)
	    return super.getServiceNamesByAuthority();
	synchronized (servicesCache) {
	    if (isCacheEmpty(servicesCache)) {
		initCache();
		if (!fillServicesCache())
		    // callback stopped filling
		    return new TreeMap();
	    }

	    // get a list file (with all service names)
	    String xmlList = getListFile(servicesCache);
	    if (xmlList == null) {
		initCache();
		if (!fillServicesCache())
		    // callback stopped filling
		    return new TreeMap();
		else {
		    xmlList = getListFile(servicesCache);
		    if (xmlList == null)
			return new TreeMap();
		}
	    }
	    return createServicesByAuthorityFromXML(xmlList, true);
	}
    }

    /***************************************************************************
     * 
     **************************************************************************/
    public MobyService[] getServices()
	throws MobyException {
	if (servicesCache == null)
	    return super.getServices();
	synchronized (servicesCache) {
	    Vector v = new Vector();
	    if (isCacheEmpty (servicesCache)) {
		initCache();
		if (! fillServicesCache())
		    // callback stopped filling
		    return new MobyService[] {};
	    }
	    File[] list = servicesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (servicesCache));
	    Arrays.sort (list, getFileComparator());
	    for (int i = 0; i < list.length; i++) {
		try {
		    if (ignored (list[i])) continue;
		    MobyService[] servs = extractServices (load (list[i]));
		    for (int j = 0; j < servs.length; j++) {
			v.addElement (servs[j]);
		    }
		} catch (MobyException e) {
		    log.error (MSG_CACHE_BAD_FILE (list[i], e));
		}
	    }
	    MobyService[] result = new MobyService [v.size()];
	    v.copyInto (result);
	    return result;
	}
}

    /***************************************************************************
     * 
     **************************************************************************/
    public MobyNamespace[] getFullNamespaces()
	throws MobyException {
	if (namespacesCache == null)
	    return super.getFullNamespaces();
	synchronized (namespacesCache) {
	    if (isCacheEmpty (namespacesCache)) {
		initCache();
		fillNamespacesCache();
	    }

	    // get a list file (with all namespaces)
	    String xmlList = getListFile (namespacesCache);
	    if (xmlList == null) {
		initCache();
		fillNamespacesCache();
		xmlList = getListFile (namespacesCache);
		if (xmlList == null)
		    return new MobyNamespace[] {};
	    }
	    return createNamespacesFromXML (xmlList);
	}
}

    /***************************************************************************
     * 
     **************************************************************************/
    protected MobyServiceType[] readServiceTypes()
	throws MobyException {
	if (serviceTypesCache == null)
	    return super.readServiceTypes();
	synchronized (serviceTypesCache) {
	    if (isCacheEmpty (serviceTypesCache)) {
		initCache();
		if (! fillServiceTypesCache())
		    // a callback stopped filling
		    return new MobyServiceType[] {};
	    }

	    // get a list file (with all service type names)
	    String xmlList = getListFile (serviceTypesCache);
	    if (xmlList == null) {
		if (! fillServiceTypesCache())
		    // a callback stopped filling
		    return new MobyServiceType[] {};
		else {
		    xmlList = getListFile (serviceTypesCache);
		    if (xmlList == null)
			return new MobyServiceType[] {};
		}
	    }
	    MobyServiceType[] types = createServiceTypesFromXML (xmlList);

	    // add details about relationship to get full service types
	    for (int i = 0; i < types.length; i++) {
		String name = types[i].getName();
		File file = new File (serviceTypesCache, name);
		try {
		    types[i].setParentNames (createServiceTypeRelationshipsFromXML (load (file)));
		} catch (MobyException e) {
		    log.error (MSG_CACHE_BAD_FILE (file, e));
		}
	    }
	    return types;
	}
}

    /***************************************************************************
     * parse list file for just the names of the Namespaces
     **************************************************************************/
    protected String[] extractNamespacesFromXML(String result)
	    throws MobyException {

	// parse returned XML
	Document document = loadDocument(new ByteArrayInputStream(result
		.getBytes()));
	NodeList list = document.getElementsByTagName("Namespace");
	if (list == null || list.getLength() == 0)
	    return new String[] {};
	String[] results = new String[list.getLength()];
	for (int i = 0; i < list.getLength(); i++) {
	    Element elem = (Element) list.item(i);
	    results[i] = elem.getAttribute("name");
	}
	java.util.Arrays.sort(results);
	return results;
    }

    /***************************************************************************
     * Some file (when a cache is being tested for emptyness) are ignored.
     **************************************************************************/
    protected static boolean ignoredForEmptiness (File file) {
	String path = file.getPath();
	return path.endsWith("~");
    }
}