// CentralDigestCachedImpl.java
//
// Created: September 2004
//
// This file is a component of the BioMoby project.
// Copyright Martin Senger (martin.senger@gmail.com).
//

package org.biomoby.client;

import org.biomoby.shared.CentralCached;
import org.biomoby.shared.MobyDataType;
import org.biomoby.shared.MobyNamespace;
import org.biomoby.shared.MobyException;
import org.biomoby.shared.MobyService;
import org.biomoby.shared.MobyServiceType;
import org.biomoby.shared.NoSuccessException;
import org.biomoby.shared.Utils;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.Map;
import java.util.HashSet;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.Vector;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Properties;
import java.util.Date;

/**
 * An implementation of {@link org.biomoby.shared.CentralAll},
 * allowing to cache locally results of the cumulative methods so it
 * does not need to access Moby registry all the time. The other
 * methods of the Central interface do not use the results of the
 * cached cumulative results (their implementation is just passed to
 * the parent class). <p>
 *
 * The caching is done in the file system, not in memory, so the
 * results are permanent (until someone removes the caching
 * directory, or calls {@link #removeFromCache}). <p>

 * This class can be used also without caching - just instantiate it
 * with 'cacheDir' set to null in the constructor. <p>
 *
 * @author <A HREF="mailto:martin.senger@gmail.com">Martin Senger</A>
 * @version $Id: CentralDigestCachedSimpleImpl.java,v 1.5 2009/08/13 21:26:26 kawas Exp $
 */

public class CentralDigestCachedSimpleImpl
    extends CentralDigestImpl
    implements CentralCached {

    private static org.apache.commons.logging.Log log =
       org.apache.commons.logging.LogFactory.getLog (CentralDigestCachedSimpleImpl.class);

    // cache location
    protected String cacheDir;        // as defined in the constructor
    protected File dataTypesCache;
    protected File servicesCache;
    protected File namespacesCache;
    protected File serviceTypesCache;

    /*************************************************************************
     * Create an instance that will access a default Moby registry and
     * will cache results in the 'cacheDir' directory. <p>
     *************************************************************************/
    public CentralDigestCachedSimpleImpl (String cacheDir)
	throws MobyException {
	this (null, null, cacheDir);
    }

    /*************************************************************************
     * Create an instance that will access a Moby registry defined by
     * its 'endpoint' and 'namespace', and will cache results in the
     * 'cacheDir' directory. Note that the same 'cacheDir' can be
     * safely used for more Moby registries. <p>
     *************************************************************************/
    public CentralDigestCachedSimpleImpl (String endpoint, String namespace, String cacheDir)
	throws MobyException {
	super (endpoint, namespace);

	if(cacheDir == null){
	    cacheDir = System.getProperty("java.io.tmpdir");
	}
	this.cacheDir = cacheDir;
	initCache();
    }

    // it makes all necessary directories for cache given in the
    // constructor (which is now in global 'cacheDir'); it is
    // separated here because it can be called either from the
    // constructor, or everytime a cache is going to be used but it is
    // not there (somebody removed it)
    protected void initCache()
	throws MobyException {
	if (cacheDir != null) {
	    File cache = createCacheDir (cacheDir, getRegistryEndpoint());
	    dataTypesCache = createSubCacheDir (cache, "dataTypes");
	    servicesCache  = createSubCacheDir (cache, "services");
	    namespacesCache = createSubCacheDir (cache, "namespaces");
	    serviceTypesCache = createSubCacheDir (cache, "serviceTypes");
	}
    }

    /**************************************************************************
     * Return a directory name representing the current cache. This is
     * the same name as given in constructors. <p>
     *
     * @return current cache directory name
     **************************************************************************/
    public String getCacheDir() {
	return cacheDir;
    }

    /***************************************************************************
     * Indicate whether the implementtaion really is using a local cache. <p>
     * 
     * @return true if a local cache is used
     **************************************************************************/
    public boolean isUsingCache() {
	return getCacheDir() != null;
    }

    /**************************************************************************
     * Removes object groups from the cache. If 'id' is null it
     * removes the whole cache (for that Moby registry this instance
     * was initiated for). Otherwise 'id' indicates which part of the
     * cache that will be removed. <p>
     *
     * @param id should be either null, or one of the following:
     * {@link #CACHE_PART_DATATYPES}, {@link #CACHE_PART_SERVICES},
     * {@link #CACHE_PART_SERVICETYPES}, and {@link
     * #CACHE_PART_NAMESPACES}.
     **************************************************************************/
    public void removeFromCache (String id) {
	try {
	    if (cacheDir != null) {
		String[] parts = null;
		if (id == null)
		    parts = new String[] { "dataTypes", "services", "serviceTypes", "namespaces" };
		else if (id.equals (CACHE_PART_SERVICES))
		    parts = new String[] { "services" };
		else if (id.equals (CACHE_PART_DATATYPES))
		    parts = new String[] { "dataTypes" };
		else if (id.equals (CACHE_PART_SERVICETYPES))
		    parts = new String[] { "serviceTypes" };
		else if (id.equals (CACHE_PART_NAMESPACES))
		    parts = new String[] { "namespaces" };
		if (parts != null) {
		    removeCacheDir (cacheDir, getRegistryEndpoint(), parts);
		}
	    }
	} catch (MobyException e) {
	    log.error ("Removing cache failed: " + e.getMessage());
	}
    }

    /*************************************************************************
     * Update the indicated part of the cache. If 'id' is null it
     * updates the whole cache (for that Moby registry this instance
     * was initiated for). <p>
     *
     * Updates means to fetch a new list of entities, compare it with
     * existing entities in the cache, fetch the missing ones and
     * remove the redundant ones. <p>
     *
     * @param id should be either null, or one of the following:
     * {@link #CACHE_PART_DATATYPES}, {@link #CACHE_PART_SERVICES},
     * {@link #CACHE_PART_SERVICETYPES}, and {@link
     * #CACHE_PART_NAMESPACES}.
     *
     *************************************************************************/
    public void updateCache (String id)
	throws MobyException {
	if (cacheDir != null) {
	    initCache();
	    if (id == null || id.equals (CACHE_PART_SERVICES)) {
// 		remove (servicesCache, LIST_FILE);
		fillServicesCache();
	    } else if (id == null || id.equals (CACHE_PART_DATATYPES)) {
// 		remove (dataTypesCache, LIST_FILE);
		fillDataTypesCache();
	    } else if (id == null || id.equals (CACHE_PART_SERVICETYPES)) {
// 		remove (serviceTypesCache, LIST_FILE);
		fillServiceTypesCache();
	    } else if (id == null || id.equals (CACHE_PART_NAMESPACES)) {
		fillNamespacesCache();
	    }
	}
    }


    /**
     * Create a cache directory from 'cacheDirectory' and 'registryId' if it
     * does not exist yet. Make sure that it is writable. Return a
     * File representing created directory.
     *
     * 'registryId' (which may be null) denotes what registry this
     * cache is going to be created for. If null, an endpoint of a
     * default Moby registry is used.
     */
    protected File createCacheDir (String cacheDirectory, String registryId)
	throws MobyException {
	if (registryId == null || registryId.equals (""))
	    registryId = CentralImpl.getDefaultURL();
	File cache = new File (cacheDirectory + File.separator + clean (registryId));
	try {
	    if (! cache.exists())
		if (! cache.mkdirs())
		    throw new MobyException ("Cannot create '" + cache.getAbsolutePath() + "'.");
	    if (! cache.isDirectory())
		throw new MobyException ("Cache location '" + cache.getAbsolutePath() + "' exists but it is not a directory.");
	    if (! cache.canWrite())
		throw new MobyException ("Cache location '" + cache.getAbsolutePath() + "' is not writable for me.");
	    return cache;
	} catch (SecurityException e) {
	    throw new MobyException ("Cannot handle cache location '" + cache.getAbsolutePath() + "'. " + e.toString());
	}
    }

    /**
     * Remove cache and all (but given in 'subCacheDirNames') its
     * subdirectories.
     */
    protected void removeCacheDir (String cacheDirectory,
        String registryId,
        String[] subCacheDirNames)
	throws MobyException {
	if (registryId == null || registryId.equals (""))
	    registryId = CentralImpl.getDefaultURL();
	File cache = new File (cacheDirectory + File.separator + clean (registryId));
	try {
	    if (! cache.exists()) return;
	    if (! cache.isDirectory())
		throw new MobyException ("Cache location '" + cache.getAbsolutePath() + "' exists but it is not a directory.");
	    if (! cache.canWrite())
		throw new MobyException ("Cache location '" + cache.getAbsolutePath() + "' is not writable for me.");
	    for (int i = 0; i < subCacheDirNames.length; i++) {
		File cacheSubDir = new File (cache.getAbsolutePath() + File.separator + clean (subCacheDirNames[i]));
		File[] files = cacheSubDir.listFiles();
		for (int f = 0; f < files.length; f++) {
		    if (files[f].isDirectory())
			throw new MobyException ("Found a directory '" + files[f].getAbsolutePath() + "' where no directory should be");
		    if (! files[f].delete())
		    	log.error ("Can't delete file '" + files[f] + "'.");
		}
		cacheSubDir.delete();
	    }
	    cache.delete();

	} catch (SecurityException e) {
	    throw new MobyException ("Cannot handle cache location '" + cache.getAbsolutePath() + "'. " + e.toString());
	}
    }

    //
    protected File createSubCacheDir (File mainCache, String subCacheDirName)
	throws MobyException {
	File cache = new File (mainCache.getAbsolutePath() + File.separator + clean (subCacheDirName));
	try {
	    if (! cache.exists())
		if (! cache.mkdirs())
		    throw new MobyException ("Cannot create '" + cache.getAbsolutePath() + "'.");
	    return cache;
	} catch (SecurityException e) {
	    throw new MobyException ("Cannot handle cache location '" + cache.getAbsolutePath() + "'. " + e.toString());
	}
    }
    
    /**************************************************************************
     * Replace non digit/letter characters in 'toBeCleaned' by their
     * numeric value. If there are more such numeric values side by
     * side, put a dot between them. Return the cleaned string.
     **************************************************************************/
    protected static String clean (String toBeCleaned) {

	char[] chars = toBeCleaned.toCharArray();
	int len = chars.length;
	int i = -1;
	while (++i < len) {
	    char c = chars[i];
	    if (!Character.isLetterOrDigit (c) && c != '_')
		break;
	}
	if (i < len) {
	    StringBuffer buf = new StringBuffer (len*2);
	    for (int j = 0 ; j < i ; j++) {
		buf.append (chars[j]);
	    }
	    boolean lastOneWasDigitalized = false;
	    while (i < len) {
		char c = chars[i];
		if (Character.isLetterOrDigit (c) || c == '_') {
		    buf.append (c);
		    lastOneWasDigitalized = false;
		} else {
		    if (lastOneWasDigitalized)
			buf.append ('.');
		    buf.append ((int)c);
		    lastOneWasDigitalized = true;
		}
		i++;
	    }
	    return new String (buf);
	}
	return toBeCleaned;
    }
    
    // create a file and put into it data to be cached
    protected void store (File cache, String name, String data)
	throws MobyException {
// 	File outputFile = new File (cache.getAbsolutePath() + File.separator + clean (name));
	File outputFile = new File (cache.getAbsolutePath() + File.separator + name);
	PrintWriter fileout = null;
	try {
	    fileout =
		new PrintWriter (new BufferedOutputStream (new FileOutputStream (outputFile)));
	    fileout.write (data);
	    fileout.close();
	} catch (IOException e) {
	    throw new MobyException ("Cannot write to '" + outputFile.getAbsolutePath() + ". " + e.toString());
	} finally {
	    if (fileout != null)
		fileout.close();
	}
    }

    // remove a file from a cache
    protected void remove (File cache, String name) {
	File file = new File (cache, name);
	// do not throw here an exception because a missing file
	// can be a legitimate status (e.g. for LIST_FILE when we
	// are updating)
	file.delete();
    }

    /**************************************************************************
     * Read a cached file
     *************************************************************************/
    protected static String load (File file)
	throws MobyException {
	BufferedReader in = null;
	try {
	    StringBuffer buf = new StringBuffer();
	    in
		= new BufferedReader (new FileReader (file));
	    char[] buffer = new char[1024];
	    int charsRead;

	    while ((charsRead = in.read (buffer, 0, 1024)) != -1) {
		buf.append (buffer, 0, charsRead);
	    }

            return new String (buf);

	} catch (Throwable e) {     // be prepare for "out-of-memory" error
	    throw new MobyException ("Serious error when reading from cache. " + e.toString());

	} finally {
	    if (in != null)
		try {
		    in.close();
		} catch (IOException e) {}
	}
    }

    /**************************************************************************
     * Is the given cache empty (meaning: cache directory does not
     * exist, is empty, or contains only files to be ignored)?
     *************************************************************************/
    protected boolean isCacheEmpty (File cache)
	throws MobyException {
	if (cache == null) return true;
	String[] list = cache.list();
	if (list == null || list.length == 0)
	    return true;
	for (int i = 0; i < list.length; i++) {
	    if ( ! ignoredForEmptiness (new File (list[i])) )
		return false;
	}
	return true;
    }

    /**************************************************************************
     * Update data types from a moby registry:
     *   - get a new LIST_FILE (but do not put it into the cache yet)
     *       if failed do nothing (except reporting it)
     *   - remove LIST_FILE
     *   - compare contents of new LIST_FILE with file names in the cache
     *     and remove them, or fetched missing ones
     *       if success add there new LIST_FILE
     *************************************************************************/
    protected boolean fillDataTypesCache()
	throws MobyException {
	try {
	    fireEvent (DATA_TYPES_START);
	    String typesAsXML = getDataTypeNamesAsXML();

	    // get a list file with all data type names currently in
	    // the cache...
	    Map cachedTypes = new HashMap();
	    String xmlList = getListFile (dataTypesCache);
	    if (xmlList != null)
		cachedTypes = createDataTypeNamesFromXML (xmlList, false);

	    // ...and remove it
 	    remove (dataTypesCache, LIST_FILE);

	    // get a list file with all data types from the registry
	    Map types = createDataTypeNamesFromXML (typesAsXML, false);
	    fireEvent (DATA_TYPES_COUNT, new Integer (types.size()));

	    // list of current files in this cache
	    HashSet currentFiles = new HashSet();
	    File[] list = dataTypesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (dataTypesCache));
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i]))
		    currentFiles.add (list[i].getName());
	    }

	    // iterate over LIST_FILE and fetch missing files
	    for (Iterator it = types.entrySet().iterator(); it.hasNext(); ) {
		Map.Entry entry = (Map.Entry)it.next();
		boolean needToFetch = false;
		String name = (String)entry.getKey();
		if ( ! currentFiles.contains (name)) {
		    // missing file
		    needToFetch = true;
		} else {
		    // check by comparing LSIDs
		    MobyDataType dt = (MobyDataType)entry.getValue();
		    String lsid = dt.getLSID();
		    if (cachedTypes.containsKey (name)) {
			// should always go here - or we have a broken cache, anyway
			String cachedLSID =
			    ( (MobyDataType)cachedTypes.get (name) ).getLSID();
			if (! lsid.equals (cachedLSID)) {
			    needToFetch = true;
			}
		    } else {
			needToFetch = true;
		    }
		}
		if (needToFetch) {
		    // missing file: fetch it from a registry
		    fireEvent (DATA_TYPE_LOADING, name);
		    String xml = getDataTypeAsXML (name);
		    store (dataTypesCache, name, xml);
		    fireEvent (DATA_TYPE_LOADED, name);
		    if (stopDT) {
			return false;
		    }
		}
		currentFiles.remove (name);
	    }

	    // remove files that are not any more needed
	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
		remove (dataTypesCache, (String)it.next());

	    // finally, put there the new LIST_FILE
 	    store (dataTypesCache, LIST_FILE, typesAsXML);
	    return true;

	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (stopDT ? DATA_TYPES_CANCELLED : DATA_TYPES_END);
	    stopDT = false;
	}
    }

    /**************************************************************************
     * Update services from a moby registry:
     *   - get a new LIST_FILE (but do not put it into the cache yet)
     *       if failed do nothing (except reporting it)
     *   - remove LIST_FILE
     *   - compare contents of new LIST_FILE with file names in the cache
     *     and remove them, or fetched missing ones;
     *     in order to compare properly you need to read individual files
     *     and look if they really contain all services mentioned in the
     *     LIST_FILE
     *       if success add there new LIST_FILE
     *************************************************************************/
    protected boolean fillServicesCache()
	throws MobyException {
	try {
	    fireEvent (AUTHORITIES_START);
	    String byAuthorityAsXML = getServiceNamesByAuthorityAsXML();
 	    remove (servicesCache, LIST_FILE);
	    Map authorities = createServicesByAuthorityFromXML (byAuthorityAsXML,
								false);
	    // list of current files in this cache
	    HashSet currentFiles = new HashSet();
	    File[] list = servicesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (servicesCache));
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i]))
		    currentFiles.add (list[i].getName());
	    }

	    // iterate over LIST_FILE and fetch missing files
	    fireEvent (AUTHORITIES_COUNT, new Integer (authorities.size()));
	    for (Iterator it = authorities.entrySet().iterator(); it.hasNext(); ) {
		Map.Entry entry = (Map.Entry)it.next();
		String authority = (String)entry.getKey();
		if (currentFiles.contains (authority)) {
		    MobyService[] servs =
			extractServices (load (new File (servicesCache, authority)));
		    // compare names in 'servs' (those are services we have in cache)
		    // with names in 'entry' (those are the ones we should have)
		    boolean theyAreEqual = true;
		    HashMap currentServices = new HashMap (servs.length);
		    for (int i = 0; i < servs.length; i++)
			currentServices.put (servs[i].getName(), servs[i]);
		    MobyService[] newServices = (MobyService[])entry.getValue();
		    for (int i = 0; i < newServices.length; i++) {
			String currName = newServices[i].getName();
			if (currentServices.containsKey (currName)) {
			    // check whether the old and new ones have the same LSID
			    MobyService current = (MobyService)currentServices.get (currName);
			    if (newServices[i].getLSID().equals (current.getLSID())) {
				currentServices.remove (currName);
			    } else {
				theyAreEqual = false;
			    }
			} else {
			    theyAreEqual = false;
			    break;
			}
		    }
		    if (currentServices.size() > 0)
			theyAreEqual = false;
		    if (! theyAreEqual)
			currentFiles.remove (authority);
		}

		if (! currentFiles.contains (authority)) {
		    // missing file: fetch it from a registry
		    fireEvent (AUTHORITY_LOADING, authority);
		    MobyService pattern = new MobyService (MobyService.DUMMY_NAME, authority);
		    pattern.setCategory ("");
		    String xml = getServicesAsXML (pattern, null, true, true);
		    store (servicesCache, authority, xml);
		    fireEvent (AUTHORITY_LOADED, authority);
		    if (stopS) {
			return false;
		    }
		} else {
		    currentFiles.remove (authority);
		}
	    }

	    // remove files that are not any more needed
	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
		remove (servicesCache, (String)it.next());

	    // finally, put there the new LIST_FILE
	    store (servicesCache, LIST_FILE, byAuthorityAsXML);
	    return true;

	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (stopS ? AUTHORITIES_CANCELLED : AUTHORITIES_END);
	    stopS = false;
	}
    }

    /**************************************************************************
     * Update service types from a moby registry:
     *   - get a new LIST_FILE (but do not put it into the cache yet)
     *       if failed do nothing (except reporting it)
     *   - remove LIST_FILE
     *   - compare contents of new LIST_FILE with file names in the cache
     *     and remove them, or fetched missing ones
     *       if success add there new LIST_FILE
     *************************************************************************/
    protected boolean fillServiceTypesCache()
	throws MobyException {
	try {
	    fireEvent (SERVICE_TYPES_START);
	    String typesAsXML = getServiceTypesAsXML();

	    // get a list file with all service type names currently
	    // in the cache...
	    MobyServiceType[] cachedList = new MobyServiceType[] {};
	    String xmlList = getListFile (serviceTypesCache);
	    if (xmlList != null)
		cachedList = createServiceTypesFromXML (xmlList);

	    HashMap cachedTypes = new HashMap();
	    for (int i = 0; i < cachedList.length; i++) {
		cachedTypes.put (cachedList[i].getName(), cachedList[i]);
	    }

	    // ...and remove it
 	    remove (serviceTypesCache, LIST_FILE);

	    // get a list file with all service types from the
	    // registry
	    MobyServiceType[] types = createServiceTypesFromXML (typesAsXML);
	    fireEvent (SERVICE_TYPES_COUNT, new Integer (types.length));

	    // list of current files in this cache
	    HashSet currentFiles = new HashSet();
	    File[] list = serviceTypesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (serviceTypesCache));
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i]))
		    currentFiles.add (list[i].getName());
	    }

	    // iterate over LIST_FILE and fetch missing files
	    for (int i = 0 ; i < types.length; i++) {
		boolean needToFetch = false;
		String name = types[i].getName();
		if ( ! currentFiles.contains (name)) {
		    // missing file
		    needToFetch = true;
		} else {
		    // check by comparing LSIDs
		    String lsid = types[i].getLSID();
		    if (cachedTypes.containsKey (name)) {
			// should always go here - or we have a broken cache, anyway
			String cachedLSID =
			    ( (MobyServiceType)cachedTypes.get (name) ).getLSID();
			if (! lsid.equals (cachedLSID)) {
			    needToFetch = true;
			}
		    } else {
			needToFetch = true;
		    }
		}
		if (needToFetch) {
		    fireEvent (SERVICE_TYPE_LOADING, name);
		    String xml = getServiceTypeRelationshipsAsXML (name, false);
		    store (serviceTypesCache, name, xml);
		    fireEvent (SERVICE_TYPE_LOADED, name);
		    if (stopST) {
			log.warn ("Service types cache not fully updated");
			return false;
		    }
		}
		currentFiles.remove (name);
	    }

	    // remove files that are not any more needed
	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
		remove (serviceTypesCache, (String)it.next());

	    // finally, put there the new LIST_FILE
	    store (serviceTypesCache, LIST_FILE, typesAsXML);
	    return true;

	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (stopST ? SERVICE_TYPES_CANCELLED :SERVICE_TYPES_END);
	    stopST = false;
	}
    }

    /**************************************************************************
     * Update namespaces from a moby registry - this is easier than with
     * other entities: just get a new LIST_FILE.
     *************************************************************************/
    protected boolean fillNamespacesCache()
	throws MobyException {
	try {
	    fireEvent (NAMESPACES_START);
	    String xml = getNamespacesAsXML();
	    store (namespacesCache, LIST_FILE, xml);
	    return true;
	} catch (Exception e) {
	    throw new MobyException (formatException (e), e);
	} finally {
	    fireEvent (NAMESPACES_END);
	}
    }

    /*************************************************************************
     *
     *************************************************************************/
    public Map getDataTypeNames()
	throws MobyException {
	if (dataTypesCache == null)
	    return super.getDataTypeNames();
	synchronized (dataTypesCache) {
	    if (isCacheEmpty (dataTypesCache)) {
		initCache();
		if (! fillDataTypesCache())
		    // callback stopped filling
		    return new TreeMap();
	    }

	    // get a list file (with all data type names)
	    String xmlList = getListFile (dataTypesCache);
	    if (xmlList == null) {
		initCache();
		if (! fillDataTypesCache())
		    // callback stopped filling
		    return new TreeMap();
		else {
		    xmlList = getListFile (dataTypesCache);
		    if (xmlList == null)
			return new TreeMap();
		}
	    }
	    return createDataTypeNamesFromXML (xmlList, true);
	}
    }

    /*************************************************************************
     *
     *************************************************************************/
    public MobyDataType[] getDataTypes()
	throws MobyException {
	if (dataTypesCache == null)
	    return super.getDataTypes();
	synchronized (dataTypesCache) {
	    Vector v = new Vector();
	    if (isCacheEmpty (dataTypesCache)) {
		initCache();
		if (! fillDataTypesCache())
		    // callback stopped filling
		    return new MobyDataType[] {};
	    }
	    File[] list = dataTypesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (dataTypesCache));
	    Arrays.sort (list, getFileComparator());

	    for (int i = 0; i < list.length; i++) {
		try {
		    if (ignored (list[i])) continue;
		    v.addElement (createDataTypeFromXML (load (list[i]), "-dummy-"));
		} catch (NoSuccessException e) {
			log.error (MSG_CACHE_BAD_FILE (list[i], e));
		}
	    }
	    MobyDataType[] result = new MobyDataType [v.size()];
	    v.copyInto (result);
	    return result;
	}
    }

    /*************************************************************************
     *
     *************************************************************************/
    public Map getServiceNamesByAuthority()
	throws MobyException {
	if (servicesCache == null)
	    return super.getServiceNamesByAuthority();
	synchronized (servicesCache) {
	    if (isCacheEmpty (servicesCache)) {
		initCache();
		if (! fillServicesCache())
		    // callback stopped filling
		    return new TreeMap();
	    }

	    // get a list file (with all service names)
	    String xmlList = getListFile (servicesCache);
	    if (xmlList == null) {
		initCache();
		if (! fillServicesCache())
		    // callback stopped filling
		    return new TreeMap();
		else {
		    xmlList = getListFile (servicesCache);
		    if (xmlList == null)
			return new TreeMap();
		}
	    }
	    return createServicesByAuthorityFromXML (xmlList, true);
	}
    }

    /*************************************************************************
     *
     *************************************************************************/
    public MobyService[] getServices()
	throws MobyException {
	if (servicesCache == null)
	    return super.getServices();
	synchronized (servicesCache) {
	    Vector v = new Vector();
	    if (isCacheEmpty (servicesCache)) {
		initCache();
		if (! fillServicesCache())
		    // callback stopped filling
		    return new MobyService[] {};
	    }
	    File[] list = servicesCache.listFiles();
	    if (list == null)
		throw new MobyException (MSG_CACHE_NOT_DIR (servicesCache));
	    Arrays.sort (list, getFileComparator());
	    for (int i = 0; i < list.length; i++) {
		try {
		    if (ignored (list[i])) continue;
		    MobyService[] servs = extractServices (load (list[i]));
		    for (int j = 0; j < servs.length; j++) {
			v.addElement (servs[j]);
		    }
		} catch (MobyException e) {
		    log.error (MSG_CACHE_BAD_FILE (list[i], e));
		}
	    }
	    MobyService[] result = new MobyService [v.size()];
	    v.copyInto (result);
	    return result;
	}
    }

    /*************************************************************************
     *
     *************************************************************************/
    public MobyNamespace[] getFullNamespaces()
	throws MobyException {
	if (namespacesCache == null)
	    return super.getFullNamespaces();
	synchronized (namespacesCache) {
	    if (isCacheEmpty (namespacesCache)) {
		initCache();
		fillNamespacesCache();
	    }

	    // get a list file (with all namespaces)
	    String xmlList = getListFile (namespacesCache);
	    if (xmlList == null) {
		initCache();
		fillNamespacesCache();
		xmlList = getListFile (namespacesCache);
		if (xmlList == null)
		    return new MobyNamespace[] {};
	    }
	    return createNamespacesFromXML (xmlList);
	}
    }

    /*************************************************************************
     *
     *************************************************************************/
    protected MobyServiceType[] readServiceTypes()
	throws MobyException {
	if (serviceTypesCache == null)
	    return super.readServiceTypes();
	synchronized (serviceTypesCache) {
	    if (isCacheEmpty (serviceTypesCache)) {
		initCache();
		if (! fillServiceTypesCache())
		    // a callback stopped filling
		    return new MobyServiceType[] {};
	    }

	    // get a list file (with all service type names)
	    String xmlList = getListFile (serviceTypesCache);
	    if (xmlList == null) {
		if (! fillServiceTypesCache())
		    // a callback stopped filling
		    return new MobyServiceType[] {};
		else {
		    xmlList = getListFile (serviceTypesCache);
		    if (xmlList == null)
			return new MobyServiceType[] {};
		}
	    }
	    MobyServiceType[] types = createServiceTypesFromXML (xmlList);

	    // add details about relationship to get full service types
	    for (int i = 0; i < types.length; i++) {
		String name = types[i].getName();
		File file = new File (serviceTypesCache, name);
		try {
		    types[i].setParentNames (createServiceTypeRelationshipsFromXML (load (file)));
		} catch (MobyException e) {
		    log.error (MSG_CACHE_BAD_FILE (file, e));
		}
	    }
	    return types;
	}
    }

    /**************************************************************************
     * A LIST_FILE is a TOC of a cache object (each cache part has its
     * own LIST_FILE). Read it and return it. If it does not exist,
     * return null.
     *************************************************************************/
    protected static String getListFile (File cache)
	throws MobyException {
	File listFile = new File (cache, LIST_FILE);
	if (! listFile.exists())
	    return null;
	return load (listFile);
    }

    /**************************************************************************
     * Return a comparator for Files that compares in case-insensitive way.
     *************************************************************************/
    protected static Comparator getFileComparator() {
	return new Comparator() {
		public int compare (Object o1, Object o2) {
		    return o1.toString().compareToIgnoreCase (o2.toString());
		}
	    };
    }

    /**************************************************************************
     * Some file (when being read from a cache directory) are ignored.
     *************************************************************************/
    protected static boolean ignored (File file) {
	String path = file.getPath();
	return
	    path.endsWith ("~")       ||
	    path.endsWith (LIST_FILE) ||
	    path.endsWith (RDF_FILE);
    }

    /**************************************************************************
     * Some file (when a cache is being tested for emptyness) are ignored.
     *************************************************************************/
    protected static boolean ignoredForEmptiness (File file) {
	String path = file.getPath();
	return
	    path.endsWith ("~");
    }

    /**************************************************************************
     *
     *************************************************************************/
    protected static String MSG_CACHE_NOT_DIR (File cache) {
	return
	    "Surprisingly, '" + cache.getAbsolutePath() +
	    "' is not a directory. Strange...";
    }

    /**************************************************************************
     *
     *************************************************************************/
    protected static String MSG_CACHE_BAD_FILE (File file, Exception e) {
	return
	    "Ignoring '" + file.getPath() +
	    "'. It should not be in the cache directory:" +
	    e.getMessage();
    }

    /**************************************************************************
     * It always (if it functions as a cache which is when 'cacheDir'
     * was given) disables caching in the parent (so no memory caching
     * happens there).
     **************************************************************************/
    public void setCacheMode (boolean shouldCache) {
	super.setCacheMode (cacheDir == null ? shouldCache : false);
    }

    /**************************************************************************
     * It always (again, if it functions as a cache which is when
     * 'cacheDir' is given) reports that caching is disabled (even
     * though for the cumulative results is actually always enabled -
     * but that is obvious from the name of this class, isn't it?).
     **************************************************************************/
    public boolean getCacheMode(){
	return (cacheDir == null ? super.getCacheMode() : false);
    }

    /**************************************************************************
     * Return age of the current (whole) cache in millis from the
     * beginning of the Epoch; or -1 if cache is empty, or the age is
     * unknown. <p>
     *
     * @return the cache age which is taken as the oldest (but filled)
     * cache part (part is considered e.g. 'services', or 'data
     * types', not their individual entities)
     **************************************************************************/
    public long getCacheAge() {
	try {
	    long dataTypesCacheAge =
		(isCacheEmpty (dataTypesCache)    ? Long.MAX_VALUE : dataTypesCache.lastModified());
	    long servicesCacheAge =
		(isCacheEmpty (servicesCache)     ? Long.MAX_VALUE : servicesCache.lastModified());
	    long namespacesCacheAge =
		(isCacheEmpty (namespacesCache)   ? Long.MAX_VALUE : namespacesCache.lastModified());
	    long serviceTypesCacheAge =
		(isCacheEmpty (serviceTypesCache) ? Long.MAX_VALUE : serviceTypesCache.lastModified());
	    long age = Math.min (Math.min (dataTypesCacheAge, servicesCacheAge),
				 Math.min (namespacesCacheAge, serviceTypesCacheAge));
	    return (age == Long.MAX_VALUE ? -1 : age);
	} catch (MobyException e) {
	    return -1;
	}
    }

    /**************************************************************************
     * Return as many properties describing the given part of a cache
     * as possible. The key used for returned properties are publicly
     * available from this class but other may be returned as well. <p>
     *
     * @return properties describing a cache
     * @param id is a part of cache to be described, or null if the
     * whole cache should be described (this may return different kind
     * of properties than for individual cache parts)
     **************************************************************************/
    public Properties getCacheInfo (String id) {
	Properties result = new Properties();
	result.put (CACHE_PROP_REGISTRY_URL, getRegistryEndpoint());
	result.put (CACHE_PROP_COUNT, new Integer (0));
	result.put (CACHE_PROP_SIZE, new Long (0));
	if (cacheDir == null) return result;
	String realName = null;
	try {
	    File thisPart = null;
	    if (CACHE_PART_SERVICES.equals (id)) {
		thisPart = servicesCache;
		realName = "Cache for Services (authorities)";
	    } else if (CACHE_PART_DATATYPES.equals (id)) {
		thisPart = dataTypesCache;
		realName = "Cache for Data Types";
	    } else if (CACHE_PART_SERVICETYPES.equals (id)) {
		thisPart = serviceTypesCache;
		realName = "Cache for Service Types";
	    } else if (CACHE_PART_NAMESPACES.equals (id)) {
		thisPart = namespacesCache;
		realName = "Cache for Namespaces";
	    }
	    if (thisPart == null) return result;
	    result.put (CACHE_PROP_NAME, realName);

	    File[] list = thisPart.listFiles();
	    if (list == null) return result;
	    result.put (CACHE_PROP_LOCATION, thisPart.getAbsolutePath());
	    int realCount = 0;
	    long realSize = 0;
	    long ageOfYoungest = -1;
	    long ageOfOldest = Long.MAX_VALUE;
	    for (int i = 0; i < list.length; i++) {
		if (! ignored (list[i])) {
		    realCount++;
		    realSize += list[i].length();
		    long age = list[i].lastModified();
		    ageOfYoungest = Math.max (ageOfYoungest, age);
		    ageOfOldest = Math.min (ageOfOldest, age);
		}
	    }
	    if (! CACHE_PART_NAMESPACES.equals (id)) {
		result.put (CACHE_PROP_COUNT, new Integer (realCount));
		result.put (CACHE_PROP_SIZE, new Long (realSize));
	    }
	    if (ageOfYoungest > 0)
		result.put (CACHE_PROP_YOUNGEST, new Long (ageOfYoungest));
	    if (ageOfOldest < Long.MAX_VALUE)
		result.put (CACHE_PROP_OLDEST, new Long (ageOfOldest));

	} catch (Exception e) {
	    log.error ("Getting cache info failed: " + e.toString());
	}
	return result;
    }

    public String getCacheInfoFormatted (String id) {
	Properties props = getCacheInfo (id);
	StringBuffer buf = new StringBuffer();
	buf.append (props.getProperty (CACHE_PROP_NAME) + "\n");
	add (buf, "Biomoby registry", props.get (CACHE_PROP_REGISTRY_URL));
	if ( ((Integer)props.get (CACHE_PROP_COUNT)).intValue() > 0 )
	    add (buf, "Number of entities", props.get (CACHE_PROP_COUNT));
	Object value = props.get (CACHE_PROP_OLDEST);
	if (value != null) {
	    long age = ((Long)value).longValue();
	    add (buf, "Oldest entry created", new Date (age));
	    add (buf, "Oldest entry has age", Utils.ms2Human (new Date().getTime() - age));
	}
	value = props.get (CACHE_PROP_YOUNGEST);
	if (value != null) {
	    long age = ((Long)value).longValue();
	    add (buf, "Youngest entry created", new Date (age));
	    add (buf, "Youngest entry has age", Utils.ms2Human (new Date().getTime() - age));
	}
	if ( ((Long)props.get (CACHE_PROP_SIZE)).longValue() > 0 )
	    add (buf, "Size (in bytes)", props.get (CACHE_PROP_SIZE));
	add (buf, "Location", props.get (CACHE_PROP_LOCATION));
	return new String (buf);
    }

    protected void add (StringBuffer buf, String name, Object value) {
	if (value != null) {
	    buf.append ("\t");
	    buf.append (name);
	    buf.append ("\t");
	    buf.append (value.toString());
	    buf.append ("\n");
	}
    }

}