package ca.ucalgary.seahawk.services;

import ca.ucalgary.seahawk.util.*;

import org.biomoby.client.*;
import org.biomoby.client.util.SplashScreenStatus;
import org.biomoby.registry.meta.*;
import org.biomoby.shared.*;
import org.biomoby.shared.data.*;
import org.biomoby.shared.parser.MobyTags;

import javax.xml.xpath.*; // compiled xpath statement
import javax.xml.namespace.NamespaceContext;

import org.w3c.dom.*;

import java.io.*;
import java.net.URL;
import java.net.MalformedURLException;
import java.util.*;
import java.util.logging.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.*;

/**
 * The engine that finds MOBY data in unstructured text or DOM of other XML languages according
 * to a rules database.  The format of this database is 
 * <a href="http://biomoby.open-bio.org/CVS_CONTENT/moby-live/Java/docs/seahawkRules.html">described 
 * here</a>. Used in Seahawk to provide the service options popup items.
 */
public class MobyClient{
    public static final String DATA_MAPPING_XML_RESOURCE = "ca/ucalgary/seahawk/resources/mobyBuilderRules.xml";
    public static final String RESOURCE_SYSTEM_PROPERTY = "seahawk.rules";
    public static final String RULE_SET_TAG = "object";
    public static final String RULE_NAME_ATTR = "name";
    public static final String PREFIX_TAG = "prefix";
    public static final String PREFIX_ATTR = "value";
    public static final String ARTICLENAME_RULE_TAG = "articlename";
    public static final String NAMESPACE_RULE_TAG = "namespace";
    public static final String NAMESPACE_MODEL_ATTR = "model";
    public static final String NAMESPACE_VALUE_TAG = "ns";
    public static final String NAMESPACE_VALUE_ATTR = "value";
    public static final String DATATYPE_RULE_TAG = "datatype";
    public static final String DATATYPE_RULE_ATTR = "value";
    public static final String MEMBER_RULE_TAG = "member";
    public static final String MEMBER_RULE_ATTR = "value";
    public static final String MEMBERS_RULE_TAG = "inheritMembers";
    public static final String MEMBERS_RULE_ATTR = "rule";
    public static final String URL_REGEX_TAG = "url_regex";
    public static final String REGEX_TAG = "regex";
    public static final String XPATH_TAG = "xpath";
    public static final String WHITESPACE_ATTR = "whitespace";
    public static final String WHITESPACE_ATTR_STRIP_VAL = "strip";
    public static final String WHITESPACE_ATTR_NORMALIZE_VAL = "normalize";
    public static final String WHITESPACE_ATTR_STRIP_FLANKING_VAL = "flanking";
    public static final String WHITESPACE_ATTR_KEEP_VAL = "keep";
    public static final String DATATYPE_ATTR = "datatype";
    public static final String ENCODING_ATTR = "encoding";
    public static final String ENCODING_ATTR_BASE64_VAL = "Base64";
    public static final String ENCODING_ATTR_NONE_VAL = "none";
    public static final String SINGLE_RETURNED_VALUE_KEY = "_no_acd_param_should_have_this_name";
    public static final String IS_ALIVE_SERVICE_URL = "http://moby.ucalgary.ca/moby/ValidateService";

    private NamespaceContextImpl nsContext;
    private XPath commonXPath;
    private CentralImpl c;
    private Map<String,String> isDeadMap;
    private Map<XPathExpression,MobyComplexBuilder> xpathMap;
    private Map<Pattern,MobyComplexBuilder> urlRegexMap;
    private Map<Pattern,MobyComplexBuilder> regexMap;
    private Map<String,MobyComplexBuilder> builderNameMap;
    private Map<String,Pattern> patternNameMap;
    private Registry registry;
    private URL dataMappingXMLURL;
    private DocumentBuilder docBuilder;
    private static Logger logger = Logger.getLogger(MobyClient.class.getName());

    private int serviceLevel = MobyService.UNCHECKED;

    /**
     * Nullary constructor that differs significantly in behaviour from the one-arg c-tor:
     * it uses the registry from SeahawkOptions, or if not available, the RegistryCache default
     * registry.  If the SeahawkOptions registry choice changes during the life of this object, the
     * new registry will NOT be reflected here.
     */
    public MobyClient() throws MobyException{
	this(SeahawkOptions.getRegistry());
    }

    /**
     * Constructor specifying a particular registry to use.
     *
     * @param reg the registry that will be called to validate ontology terms and determine available services.  If null, the default registry will be used.
     */
    public MobyClient(Registry reg) throws MobyException{
	registry = reg;
	SplashScreenStatus.setStatus("Initializing data recognizer");
        c = CentralImpl.getDefaultCentral(registry);
	c.setCacheMode(true);

	xpathMap = new HashMap<XPathExpression,MobyComplexBuilder>();
	urlRegexMap = new HashMap<Pattern,MobyComplexBuilder>();
	regexMap = new HashMap<Pattern,MobyComplexBuilder>();
	builderNameMap = new HashMap<String,MobyComplexBuilder>(); 
	patternNameMap = new HashMap<String,Pattern>(); 
	nsContext = new NamespaceContextImpl();
	//PG point to xalan until Google App Engine bug is fixed
	//commonXPath = XPathFactory.newInstance().newXPath();
	commonXPath = (new org.apache.xpath.jaxp.XPathFactoryImpl()).newXPath();
	commonXPath.setNamespaceContext(nsContext);

 	DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
	dbf.setNamespaceAware(true);	
	try{
	    docBuilder = dbf.newDocumentBuilder();
	}
	catch(ParserConfigurationException pce){
	    logger.log(Level.SEVERE, "Error: Could not find an XML parser, will not be able to use " +
		       "MOBY xpath and regex mapper default behaviors: ", pce);
	}
	ClassLoader cl = getClass().getClassLoader();
	if(cl == null){
	    cl = ClassLoader.getSystemClassLoader();
	}
        String rulesResource = System.getProperty(RESOURCE_SYSTEM_PROPERTY);
        if(rulesResource == null){
	  dataMappingXMLURL = cl.getResource(DATA_MAPPING_XML_RESOURCE);
        }
	else if(rulesResource.length() != 0){
	    // See if it's a URL
	    try{
		dataMappingXMLURL = new URL(rulesResource);
	    }
	    catch(Exception e){
		dataMappingXMLURL = cl.getResource(rulesResource);
	    }
	}
	if(dataMappingXMLURL == null){
	    if(rulesResource.length() != 0){ // if not left intentionally blank
		logger.log(Level.WARNING, "Could not find MOBY data mapping resource '"+
				          rulesResource+"'");
	    }
	}
	else{
	    try{
		addMappingsFromURL(dataMappingXMLURL);
	    }
	    catch(Exception e){
		logger.log(Level.SEVERE, "Error loading default data mapping rules ("+dataMappingXMLURL+")", e);
                e.printStackTrace();
	    }
	}
    }

    /**
     * Convenience method to backtrack from a mapping rule to the Moby datatype it produces.
     *
     * @param ruleSource where the rule should be loaded from
     * @param ruleURI a unique ID for the rule, in the source XML using Dublin Core
     *
     * @return a template object with the datatype and namespaces produced by the rule
     */
    public static MobyPrimaryDataSimple getObjectProduced(URL ruleSource, String ruleURI, Registry reg) throws Exception{
	MobyPrimaryDataSimple template = null;
	// Load the rule from the URL to make sure it's really accessible
	MobyClient client = null;
	try{
	    System.setProperty(MobyClient.RESOURCE_SYSTEM_PROPERTY, ""); //load no rules by default
	    client = new MobyClient(reg);
	    client.addMappingsFromURL(ruleSource);
	} catch(Exception e){
	    throw new Exception("Internal error: Could not create MobyClient and load the rule (from " + 
				ruleSource+"): "+e.getMessage(), 
				e);
	}
	MobyComplexBuilder mobyBuilder = client.getBuilder(ruleURI);
	if(mobyBuilder == null){
	    throw new Exception("Internal error: loaded the transformation rule (URI " + ruleURI + 
				") from " + ruleSource + " but could not retrieve it from MobyClient " +
				"using the URI.  Make sure the transformation rule contains a Dublin Core " +
				"identifier element specifying this URI.");
	}
	else{
	    template = new MobyPrimaryDataSimple("templateRuleReturnedObject");
	    template.setDataType(mobyBuilder.getDataType());
	    template.setNamespaces(mobyBuilder.getNamespaces());
	}
	return template;	
    }

    public Registry getRegistry(){
	return registry;
    }

    /**
     * Parses the XML formatted rules from the input stream specified by the URL.
     */
    public void addMappingsFromURL(URL u) throws Exception{
	if(docBuilder == null){
	    logger.log(Level.SEVERE, "Asked to add data mappings from file, " +
			             "but no XML parser is available");
	    return;
	}

	SplashScreenStatus.setStatus("Parsing recognition rules");
	// Parse the XML
	Document d = docBuilder.parse(u.openStream());

	// Extract the rules from the document
	SplashScreenStatus.setStatus("Compiling recognition rules");
	addMappingsFromDOM(d);
    }

    public void addNamespaceContext(String prefix, String nsURI){
        nsContext.setPrefix(nsURI, prefix);
    }

    protected void addPrefixMapping(Element e) throws Exception{
	if(!isPrefixRule(e)){
	    throw new Exception("Element provided to addPrefixMapping (" +
				(e == null ? null : e.getLocalName()) + 
				") was not a prefix rule element");
	}

	String prefix = e.getAttribute(PREFIX_ATTR);
	if(prefix == null){
	    throw new Exception("Element provided to addPrefixMapping did not " +
				"have a " + PREFIX_ATTR + 
				" attribute as required");
	}

	if(nsContext.getNamespaceURI(prefix) != null){
	    logger.log(Level.WARNING, "Prefix definition for " + prefix + " already exists, ignoring new definition");
	    return;
	}

	String nsURI = e.getTextContent();
	if(nsURI == null || nsURI.length() == 0){
	    logger.log(Level.WARNING, "Prefix definition for " + prefix + " has a blank URI rule, ignoring");
	    return;
	}
	nsContext.setPrefix(nsURI, prefix);	
    }

    protected void addMappingsFromDOM(Document d) throws Exception{
	Element root = d.getDocumentElement();

	// The document may be either <mappings><object><object>...</mapping>,
	NodeList ruleSets = root.getChildNodes();
	if(root.getLocalName().equals(RULE_SET_TAG)){ // or just <object>, make it its own set
	    ruleSets = new MobyPrefixResolver.MobyNodeList();
	    (( MobyPrefixResolver.MobyNodeList) ruleSets).add(root);
	}    
	for(int i = 0; i < ruleSets.getLength(); i++){
	    Node node = ruleSets.item(i);
	    if(node == null || !(node instanceof Element)){
		// Ignore any other contents
		continue;
	    }

	    Element ruleSet = (Element) node;
	    if(isPrefixRule(ruleSet)){
		addPrefixMapping(ruleSet);
		continue;
	    }

	    if(!RULE_SET_TAG.equals(ruleSet.getLocalName())){
		logger.log(Level.WARNING, "Skipping unexpected top level tag " + ruleSet.getLocalName());
		continue;
	    }

	    String ruleName = ruleSet.getAttribute(RULE_NAME_ATTR);
	    String urn = getURN(ruleSet);

	    Vector<String> regexStrings = new Vector<String>();
	    Vector<String> urlRegexStrings = new Vector<String>();
	    Vector<String> xpathStrings = new Vector<String>();
	    Map<String,String> namespaceMap = new HashMap<String,String>();  // moby nanmespaces -> production rules
	    Map<String,String> modelMap = new HashMap<String,String>();  // Semantic Web URIs -> production rules
	    Map<String,String[]> memberMap = new HashMap<String,String[]>();
	    String dataTypeString = null;
	    String articleNameString = null;

	    NodeList ruleMembers = ruleSet.getChildNodes();
	    for(int j = 0; j < ruleMembers.getLength(); j++){

		Node memNode = ruleMembers.item(j);
		if(memNode == null || !(memNode instanceof Element)){
		    // Ignore any other contents
		    continue;
		}

		Element ruleMember = (Element) memNode;
		if(isRegex(ruleMember)){
		    addRegexString(ruleMember, regexStrings);
		}
		else if(isURLRegex(ruleMember)){
		    addURLRegexString(ruleMember, urlRegexStrings);
		}
		else if(isXPath(ruleMember)){
		    addXPathString(ruleMember, xpathStrings);
		}
		else if(isNamespaceRule(ruleMember)){
		    addNamespaceMapping(ruleMember, namespaceMap, modelMap);
		}
		else if(isArticleNameRule(ruleMember)){
		    if(articleNameString != null && articleNameString.length() != 0){
			logger.log(Level.WARNING, "Skipping unexpected article name definition node, " +
					          "a valid article name rule has already been " +
					          "created for this ruleset");
			continue;
		    }
		    articleNameString = ruleMember.getTextContent();
		}
		else if(isDataTypeRule(ruleMember)){
		    if(dataTypeString != null && dataTypeString.length() != 0){
			logger.log(Level.WARNING, "Skipping unexpected datatype definition node, " +
					          "a valid datatype rule has already been " +
					          "created for this ruleset");
			continue;
		    }
		    dataTypeString = getDataType(ruleMember);
// 		    if(isPrimitiveDataType(dataTypeString)){
// 			addMemberMapping(ruleMember, memberMap, dataTypeString);
// 		    }
		}
		else if(isMemberRule(ruleMember)){
		    addMemberMapping(ruleMember, memberMap, dataTypeString);
		}
		// skip any Dublin Core metadata in validation
		else if(!MobyPrefixResolver.DUBLIN_CORE_NAMESPACE.equals(ruleMember.getNamespaceURI())){
		    logger.log(Level.WARNING, "Skipping unexpected "+RULE_SET_TAG+
				              " child node " + ruleMember.getLocalName());
		    continue;
		} 
	    }

	    if(xpathStrings.size() == 0 && regexStrings.size() == 0 && urlRegexStrings.size() == 0){
		logger.log(Level.WARNING, "Skipping namespace rule that has no xpath " +
				          "url_regex or regex specs associated with it");
		// Next ruleset
		continue;
	    }

	    // Simple object with just namespace & ID
	    if(dataTypeString == null || dataTypeString.length() == 0){
		if(memberMap.size() != 0){
		    logger.log(Level.WARNING, "Warning: ignoring member rules, since " +
				              "no datatype was defined for the ruleset " +
				              "(base Object will be created)");
		}

		for(int j = 0; j < xpathStrings.size(); j++){
		    addXPathMapping((String) xpathStrings.elementAt(j), namespaceMap, articleNameString, urn);
		}
		for(int j = 0; j < regexStrings.size(); j++){
		    addRegexMapping((String) regexStrings.elementAt(j), namespaceMap, articleNameString, ruleName, urn);
		}
		for(int j = 0; j < urlRegexStrings.size(); j++){
		    addURLRegexMapping((String) urlRegexStrings.elementAt(j), namespaceMap, articleNameString, urn);
		}
	    }
	    // Build complex object
	    else{
		if(memberMap.size() == 0){
		    logger.log(Level.WARNING, "Complex datatype (" + dataTypeString + " was defined " +
				              " in the ruleset, but no members were defined");
		}

		for(int j = 0; j < xpathStrings.size(); j++){
		    addXPathMapping((String) xpathStrings.elementAt(j), namespaceMap, dataTypeString, memberMap, articleNameString, urn);
		}
		for(int j = 0; j < regexStrings.size(); j++){
		    addRegexMapping((String) regexStrings.elementAt(j), namespaceMap, dataTypeString, memberMap, articleNameString, ruleName, urn);
		}
		for(int j = 0; j < urlRegexStrings.size(); j++){
		    addURLRegexMapping((String) urlRegexStrings.elementAt(j), namespaceMap, dataTypeString, memberMap, articleNameString, urn);
		}
	    }

	}  // for each ruleset

	// TODO: add warning if no rules at all were added (the file may not be a MOB rules file!)
    }

    // See if the Dublin Core identifier or source child is available for the element, 
    // giving us a unique, referencable name for the rule
    private String getURN(Element e){
	NodeList children = e.getChildNodes();
	for(int i = 0; i < children.getLength(); i++){
	    if(children.item(i) instanceof Element){
		Element child = (Element) children.item(i);
		if(!MobyPrefixResolver.DUBLIN_CORE_NAMESPACE.equals(child.getNamespaceURI())){
		    continue; // not metadata
		}
		if(child.getLocalName().equals("identifier") || 
		   child.getLocalName().equals("source")){
		    String id = child.getTextContent();
		    if(id != null && id.trim().length() > 0){
			return id.trim();
		    }
		}
	    }
	}
	return null;
    }

    public boolean isPrefixRule(Element e){
	return e != null && PREFIX_TAG.equals(e.getLocalName());
    }

    public boolean isRegex(Element e){
	return e != null && REGEX_TAG.equals(e.getLocalName());
    }

    public boolean isURLRegex(Element e){
	return e != null && URL_REGEX_TAG.equals(e.getLocalName());
    }

    public boolean isXPath(Element e){
	return e != null && XPATH_TAG.equals(e.getLocalName());
    }

    public boolean isDataTypeRule(Element e){
	return e != null && DATATYPE_RULE_TAG.equals(e.getLocalName());
    }

    public boolean isArticleNameRule(Element e){
	return e != null && ARTICLENAME_RULE_TAG.equals(e.getLocalName());
    }

    // Defined or inherited rule spec.
    public boolean isMemberRule(Element e){
	return e != null && (MEMBER_RULE_TAG.equals(e.getLocalName()) ||
			     MEMBERS_RULE_TAG.equals(e.getLocalName()));
    }

    // Inherited rule spec.
    public boolean isMembersRule(Element e){
	return e != null && MEMBERS_RULE_TAG.equals(e.getLocalName());
    }

    public boolean isNamespaceRule(Element e){
	return e != null && NAMESPACE_RULE_TAG.equals(e.getLocalName());
    }

    protected String getDataType(Element dtTag) throws Exception{
	if(!isDataTypeRule(dtTag)){
	    throw new Exception("Element provided to getDataType (" +
				(dtTag == null ? null : dtTag.getLocalName()) + 
				") was not a datatype rule element");
	}

	return dtTag.getAttribute(DATATYPE_RULE_ATTR);
    }

    protected String addRegexString(Element regexTag, Vector<String> regexStrings) throws Exception{
	if(!isRegex(regexTag)){
	    throw new Exception("Element provided to addRegexString (" +
				(regexTag == null ? null : regexTag.getLocalName()) + 
				") was not a regex rule element");
	}

	String str = regexTag.getTextContent().trim();
	if(str != null || str.length() != 0){
	    regexStrings.add(str);
	}
	else{
	    logger.log(Level.WARNING, "Skipping blank regex rule"); 
	}

	return str;
    }

    protected String addURLRegexString(Element urlRegexTag, Vector<String> urlRegexStrings) throws Exception{
	if(!isURLRegex(urlRegexTag)){
	    throw new Exception("Element provided to addURLRegexString (" +
				(urlRegexTag == null ? null : urlRegexTag.getLocalName()) + 
				") was not a url_regex rule element");
	}

	String str = urlRegexTag.getTextContent().trim();
	if(str != null || str.length() != 0){
	    urlRegexStrings.add(str);
	}
	else{
	    logger.log(Level.WARNING, "Skipping blank url_regex rule"); 
	}

	return str;
    }

    protected String addXPathString(Element xPathTag, Vector<String> xPathStrings) throws Exception{
	if(!isXPath(xPathTag)){
	    throw new Exception("Element provided to addXPathString (" +
				(xPathTag == null ? null : xPathTag.getLocalName()) + 
				") was not an XPath rule element");
	}

	String str = xPathTag.getTextContent().trim();
	if(str != null || str.length() != 0){
	    xPathStrings.add(str);
	}
	else{
	    logger.log(Level.WARNING, "Skipping blank XPath rule"); 
	}

	return str;
    }

    public boolean isPrimitiveDataType(String dataType){
	return dataType.equals(MobyTags.MOBYSTRING) || dataType.equals(MobyTags.MOBYINTEGER) ||
	    dataType.equals(MobyTags.MOBYFLOAT) || dataType.equals(MobyTags.MOBYDATETIME) ||
	    dataType.equals(MobyTags.MOBYBOOLEAN);
    }

    protected void addMemberMapping(Element memTag, Map<String,String[]> membersMap, String dataTypeName) 
	throws Exception{
	if(!isMemberRule(memTag) && !isDataTypeRule(memTag)){
	    throw new Exception("Element provided to addMemberMapping (" +
				(memTag == null ? null : memTag.getLocalName()) + 
				") was not a member rule element");
	}
	String ruleValue = memTag.getTextContent();
	String memberNameKey = isPrimitiveDataType(dataTypeName) ? 
	    MobyComplexBuilder.PRIMITIVE_VALUE_SENTINEL : memTag.getAttribute(DATATYPE_RULE_ATTR);
	if(ruleValue == null || ruleValue.length() == 0){
	    logger.log(Level.WARNING, "Object member " + memberNameKey + " has a blank value rule");
	}

	String memberDataTypeSetting = memTag.getAttribute(DATATYPE_ATTR);
	if(isMembersRule(memTag)){
	    String membersRuleName = memTag.getAttribute(MEMBERS_RULE_ATTR);
	    if(membersRuleName == null || membersRuleName.length() == 0){
		throw new Exception("Attribute "+MEMBERS_RULE_ATTR+" is missing from the " +
				    "member rule tag '" + memTag.getNodeName()+"'");
	    }
	    MobyComplexBuilder membersBuilder = builderNameMap.get(membersRuleName);
	    if(membersBuilder == null){
		throw new Exception("Attribute "+MEMBERS_RULE_ATTR+" refers to a rule (" +
				    membersRuleName+") that does not exist");
	    }
	    MobyDataType dataType = MobyDataType.getDataType(dataTypeName, getRegistry());
	    if(!dataType.inheritsFrom(membersBuilder.getDataType())){
		throw new Exception("Data type produced by inherited rule (" + membersRuleName + 
				    ") is not a subtype of the current rule (" + dataType.getName() + ")"); 
	    }
	    // Borrow members from the builder, unless they already exist in the
	    // production rule (i.e. "member" rules override "inheritMembers" rules),
	    // BUT, we need to tell the builder that the regex to match is not
	    // the one from the current rule, but the one from the inherited rule
	    // (applied to the value created by ruleValue)
	    // the stering array for the inheritance rule looks like ["ruleName1", "ruleSrcValueExpr1"]
	    if(membersMap.containsKey(MobyComplexBuilder.INHERITED_MEMBERS_SENTINEL)){
		throw new Exception("More than one member inheritance tag was given, which is illegal");
	    }
	    String[] inheritanceRuleSpecs = new String[2];
	    inheritanceRuleSpecs[0] = membersRuleName;
	    inheritanceRuleSpecs[1] = ruleValue;
	    membersMap.put(MobyComplexBuilder.INHERITED_MEMBERS_SENTINEL, inheritanceRuleSpecs);
	    return;
	}

	if(memberNameKey == null || memberNameKey.length() == 0){
	    throw new Exception("Element provided to addMemberMapping did not " +
				"have a non-blank " + DATATYPE_RULE_ATTR + 
				" attribute as required");
	}

	String memberWhitespaceSetting = memTag.getAttribute(WHITESPACE_ATTR);
	if(memberWhitespaceSetting == null || memberWhitespaceSetting.length() == 0){
	    memberWhitespaceSetting = WHITESPACE_ATTR_KEEP_VAL;  // default is to keep whitespace
	}
	else if(!memberWhitespaceSetting.equals(WHITESPACE_ATTR_KEEP_VAL) &&
		!memberWhitespaceSetting.equals(WHITESPACE_ATTR_NORMALIZE_VAL) &&
		!memberWhitespaceSetting.equals(WHITESPACE_ATTR_STRIP_FLANKING_VAL) &&
		!memberWhitespaceSetting.equals(WHITESPACE_ATTR_STRIP_VAL)){
	    logger.log(Level.WARNING, "Object member " + memberNameKey + 
			              " has an unrecognized value for the " + WHITESPACE_ATTR +
			              " attribute (" + memberWhitespaceSetting + 
			              "), overriding with default of " + WHITESPACE_ATTR_KEEP_VAL);
	    memberWhitespaceSetting = WHITESPACE_ATTR_KEEP_VAL;
	}

	String memberEncodingSetting = memTag.getAttribute(ENCODING_ATTR);
	if(memberEncodingSetting == null || memberEncodingSetting.length() == 0){
	    memberEncodingSetting = ENCODING_ATTR_NONE_VAL;  // default is to not encode
	}
	else if(!memberEncodingSetting.equals(ENCODING_ATTR_NONE_VAL) &&
		!memberEncodingSetting.equals(ENCODING_ATTR_BASE64_VAL)){
	    logger.log(Level.WARNING, "Object member " + memberNameKey + 
			              " has an unrecognized value for the " + ENCODING_ATTR +
			              " attribute, overriding with default of " + ENCODING_ATTR_NONE_VAL);
	    memberWhitespaceSetting = ENCODING_ATTR_NONE_VAL;
	}

	if(membersMap.containsKey(memberNameKey)){
	    logger.log(Level.WARNING, "Object member " + memberNameKey + " already exists, ignoring new definition");
	    return;
	}

	// Leave memberRuleName blank unless we inherited members via 
        // a rule attrubute.  If memberRuleName is null,
        // in another method we will see if any capture value in the member
        // rule refers to a \p{ruleName} string in the regex 
	String memberRuleName = memTag.getAttribute(MEMBERS_RULE_ATTR);
	membersMap.put(memberNameKey, new String[]{ruleValue,
						   memberDataTypeSetting,
						   memberWhitespaceSetting, 
						   memberEncodingSetting,
	                                           memberRuleName});
    }

    protected void addNamespaceMapping(Element nsTag, Map<String,String> namespaceStrings, Map<String,String> modelStrings) throws Exception{
	if(!isNamespaceRule(nsTag)){
	    throw new Exception("Element provided to createNamespaceMapping (" +
				(nsTag == null ? null : nsTag.getLocalName()) + 
				") was not a namespace rule element");
	}
	

	NodeList ruleSpecs = nsTag.getChildNodes();
	for(int i = 0; i < ruleSpecs.getLength(); i++){
	    Node specNode = ruleSpecs.item(i);
	    if(specNode == null || !(specNode instanceof Element)){
		// Ignore any other contents
		continue;
	    }

	    Element ruleSpec = (Element) specNode;
	    String specName = ruleSpec.getLocalName();
	    if(NAMESPACE_VALUE_TAG.equals(specName)){
		String keyName = ruleSpec.getAttribute(NAMESPACE_VALUE_ATTR); // moby namespace
		String modelURI = ruleSpec.getAttribute(NAMESPACE_MODEL_ATTR); // Semantic Web URI
		String valueRule = ruleSpec.getTextContent();
		if(valueRule == null || valueRule.length() == 0){
		    throw new Exception("Element " + NAMESPACE_VALUE_TAG + 
					" cannot have empty children");
		}
		if(namespaceStrings.containsKey(keyName)){
		    logger.log(Level.WARNING, "Ignoring duplicate namespace value rule for " +
				              "namespace '" + keyName + "'");
		    continue;
		}
		if(keyName != null && keyName.length() != 0){
		    namespaceStrings.put(keyName, valueRule);
		}
		if(modelURI != null && modelURI.length() != 0){
		    modelStrings.put(modelURI, valueRule);
		}
	    }
	    else{
		logger.log(Level.WARNING, "Skipping unexpected "+NAMESPACE_RULE_TAG+" child node " + specName);
		continue;
	    }
	}
	
	if(namespaceStrings.size() == 0){
	    throw new Exception("Namespace mapping rule did not have any " +
				"namespaces associated with it!");
	}
    }

    public CentralImpl getMobyCentralImpl(){
	return c; 
    }

    /**
     * Performs the moby object creation and service association together (XPath rules).
     */
    public MobyDataServiceAssocInstance[] getServices(Node n) throws MobyException{ 
	return getServices(getMobyObjects(n));
    }

    /**
     * Performs the moby object creation and service association together (regex rules).
     */
    public MobyDataServiceAssocInstance[] getServices(String s) throws MobyException{ 
	return getServices(getMobyObjects(s));
    }

    /**
     * Performs the moby object creation and service association together (url_regex rules).
     */
    public MobyDataServiceAssocInstance[] getServices(URL u) throws MobyException{ 
	return getServices(getMobyObjects(u));
    }

    public MobyDataObject[] getMobyObjects(CharSequence textData, MobyDataType targetDataType){
	return getMobyObjects(textData, targetDataType, null);
    }
    /**
     * Using the regular expression mappings that have been set up, 
     * maps a string using rules producing moby objects of the given type.
     * @param targetDataType matches any data type if null, otherwise restricts returned items to those that are of (or inherit from) the given type
     */
    public MobyDataObject[] getMobyObjects(CharSequence textData, MobyDataType targetDataType, MobyNamespace targetNS){
	Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>();

	if(textData == null){
	    return new MobyDataObject[0];
	}
	for(Pattern pattern: regexMap.keySet()){
	    MobyComplexBuilder rule = regexMap.get(pattern);
	    if((targetDataType == null ||
	       rule.getDataType().inheritsFrom(targetDataType)) &&
	       (targetNS == null || 
	       nsArrayContains(rule.getNamespaces(), targetNS))){
		try{
		    Matcher matcher = pattern.matcher(textData);
		    
		    while(matcher.find()){
			try{
			    MobyDataObject mobyObj = rule.apply(matcher);
			    if(mobyObj != null){
				objectVector.add(mobyObj);		
			    }
			}
			catch(MobyException me){
			    logger.log(Level.SEVERE, "Could not build Moby object from match", me);
			}
		    }
		} catch(Throwable t){  // Principally stack overflows
		    logger.log(Level.SEVERE, "Exception/Error ("+t+") while applying regex '"+pattern.pattern()+"' to text", t);
		}
	    }
	}

	return (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]);	
    }
    
    private boolean nsArrayContains(MobyNamespace[] nss, MobyNamespace tns){
	if(nss == null || nss.length == 0){
	    return false;
	}
	for(MobyNamespace ns: nss){
	    if(tns.equals(ns)){
		return true;
	    }
	}
	return false;
    }

    /**
     * Create a MOBY data instance from a DOM by applying the rules of the given MobyClient (xpaths).
     * The creation of a single object or collection is done according to the dataTemplate provided.
     *
     * @throws MobyServiceException NOTE: these are just warning level exceptions about collection-simple casting that you can add to the service response, or ignore if you like
     */
    public MobyDataInstance getMobyObject(Node n, MobyPrimaryData dataTemplate)
	throws Exception, MobyServiceException{
	Vector<MobyDataObject> results = new Vector<MobyDataObject>();

	for(MobyDataObject resultPart: getMobyObjects(n)){ //, dataTemplate.getDataType())){
	    results.add(resultPart);
	}

	if(results.isEmpty()){
	    throw new Exception("No MOB rules could be applied to the input data");
	}
	else if(results.size() > 1){
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		// In this case, eliminate any higher-up objects that
		// have the exact same contents as the same members who are their
		// subtypes, as these are superfluous.

		logger.log(Level.WARNING, "Multiple Moby objects were found in the text data, " +
				          "but the request was to return a simple. " +
				          "Only the first value in the collection has been returned.");
		for(MobyDataObject result: results){
		    logger.log(Level.WARNING, "Found object: " + result.getDataType().getName()); 
		}
		// TODO: should we instead return the one deepest in the hierarchy (or with the most members)?
		return results.elementAt(0);
	    }
	    else{
		MobyDataObjectSet resultSet = new MobyDataObjectSet(dataTemplate.getName());
		resultSet.addAll(results);
		return resultSet;
	    }
	}
	// One result
	else{
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		return results.elementAt(0);
	    }
	    else{  // Collection of 1
		MobyDataObjectSet resultSet = new MobyDataObjectSet(dataTemplate.getName());
		resultSet.add(results.elementAt(0));
		return resultSet;
	    }
	}
    }

    /**
     * Create a MOBY data instance from a string by applying the rules of the given MobyClient (regexes).
     * The creation of a single object or collection is done according to the dataTemplate provided.
     *
     * @throws MobyServiceException NOTE: these are just warning level exceptions about collection-simple casting that you can add to the service response, or ignore if you like
     */
    public MobyDataInstance getMobyObject(String s, MobyPrimaryData dataTemplate)
	throws Exception, MobyServiceException{
	Vector<MobyDataObject> results = new Vector<MobyDataObject>();

	MobyNamespace[] nss = dataTemplate.getNamespaces();
	MobyNamespace nsRestriction = nss == null || nss.length == 0 ? null : nss[0];
	for(MobyDataObject resultPart: getMobyObjects(s, dataTemplate.getDataType(), nsRestriction)){
	    results.add(resultPart);
	}

	if(results.isEmpty()){
	    throw new Exception("No MOB rules could be applied to the input data");
	}
	else if(results.size() > 1){
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		// In this case, eliminate any higher-up objects that
		// have the exact same contents as the same members who are their
		// subtypes, as these are superfluous.

		logger.log(Level.WARNING, "Multiple Moby objects were found in the text data, " +
				          "but the request was to return a simple. " +
				          "Only the first value in the collection has been returned.");
		for(MobyDataObject result: results){
		    logger.log(Level.WARNING, "Found object: " + result.getDataType().getName()); 
		}
		// TODO: should we instead return the one deepest in the hierarchy (or with the most members)?
		return results.elementAt(0);
	    }
	    else{
		MobyDataObjectSet resultSet = new MobyDataObjectSet(dataTemplate.getName());
		resultSet.addAll(results);
		return resultSet;
	    }
	}
	// One result
	else{
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		return results.elementAt(0);
	    }
	    else{  // Collection of 1
		MobyDataObjectSet resultSet = new MobyDataObjectSet(dataTemplate.getName());
		resultSet.add(results.elementAt(0));
		return resultSet;
	    }
	}
    }

    /**
     * Create a MOBY data instance from a map of name->bytes[] by applying the rules of the given MobyClient.
     * The creation of a single object or collection is done according to the dataTemplate provided.
     *
     * @throws MobyServiceException NOTE: these are just warning level exceptions about collection-simple casting that you can add to the service response, or ignore if you like
     */
    public MobyDataInstance getMobyObject(Map<String, byte[]> resultParts, MobyPrimaryData dataTemplate) 
	throws Exception, MobyServiceException{
	Vector<MobyDataObject> results = new Vector<MobyDataObject>();

	// The output parameter *potentially* has multiple parts 
	// (e.g. multiple .png files from EMBOSS's "banana" program)
	// hence the nested for loops (part iteration, followed by created-object iteration)
	for(String resultPartName: resultParts.keySet()){

	    //System.err.println("About to check part "+ resultPartName);
	    for(MobyDataObject resultPart: getMobyObjects(resultParts.get(resultPartName), dataTemplate.getDataType())){
		// Maintain the name for the object in the collection, as banana.1.png, banana.2.png, etc.
		// is useful information for the end-user for making sense of them together! (left to right)
		if(!resultPartName.equals(SINGLE_RETURNED_VALUE_KEY)){  // except for trivial single-byte-array case
		    resultPart.setName(resultPartName);
		}
		results.add(resultPart);
	    }
	}

	if(results.size() == 0){
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		if(resultParts != null && resultParts.size() > 0){
		    // Is it a single-byte-array response?
		    if(resultParts.containsKey(SINGLE_RETURNED_VALUE_KEY)){
			if(resultParts.get(SINGLE_RETURNED_VALUE_KEY).length != 0){
			    throw new MobyServiceException(MobyServiceException.WARNING, 
							   MobyServiceException.INTERNAL_PROCESSING_ERROR, 
							   null, 
							   dataTemplate.getName(),
							   "The non-blank data provided " +
							   "did not match any MOBY Object rules, " +
							   "therefore a blank response is being returned.  Contact " +
							   " the service provider to fix the MOBY Object rules.");
			}
		    }
		    // Otherwise it's a multi-part result
		    else{
			for(String partName: resultParts.keySet()){
			    if(resultParts.get(partName).length != 0){
				throw new MobyServiceException(MobyServiceException.WARNING, 
							       MobyServiceException.INTERNAL_PROCESSING_ERROR, 
							       null, 
							       dataTemplate.getName(),
							       "The non-blank data provided (" + partName + 
							       ") did not match any MOBY Object rules, " +
							       "therefore a blank response is being returned.  Contact " +
							       " the service provider to fix the MOBY Object rules.");
			    }
			}
		    }  //end multi-part result
		}  //end if some results present
		return null; //nothing to report
	    }
	    else{  // Empty set
		return new MobyDataObjectSet(dataTemplate.getName());		    
	    }
	}
	else if(results.size() > 1){
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		// In this case, eliminate any higher-up objects that 
		// have the exact same contents as the same members who are their
		// subtypes, as these are superfluous.

		logger.log(Level.WARNING, "Multiple Moby objects were found in the text data, " +
			                  "but the request was to return a simple. " +
				          "Only the first value in the collection has been returned.");
		for(MobyDataObject result: results){
		    logger.log(Level.WARNING, "Found object: " + result.getDataType().getName()); 
		}
		// TODO: should we instead return the one deepest in the hierarchy (or with the most members)?
		return results.elementAt(0);
	    }
	    else{
		MobyDataObjectSet resultSet = new MobyDataObjectSet(dataTemplate.getName());
		resultSet.addAll(results);
		return resultSet;
	    }
	}
	// One result
	else{
	    if(dataTemplate instanceof MobyPrimaryDataSimple){
		return results.elementAt(0);
	    }
	    else{  // Collection of 1
		MobyDataObjectSet resultSet = new MobyDataObjectSet(dataTemplate.getName());
		resultSet.add(results.elementAt(0));
		return resultSet;
	    }
	}
    }

    /**
     * Using the regular expression mappings that have been set up, 
     * maps a string using rules producing moby objects of the given type.
     * Particularly, the regex is <b>assumed to include only ASCII characters at first</b>,
     * and the byte-translation of it is checked against the raw data's bytes.
     * If no rules match, the raw data is converted to a String, and the regex
     * are tried as normal...
     * 
     * The net effect is that you specify magic signatures for file types as regular expression rules,
     * and calling this method will match up the file type by byte-wise comparing the data and pattern.
     */
    public MobyDataObject[] getMobyObjects(byte[] rawData, MobyDataType targetDataType){
	// Only do the magic check if the target data type inherits from MOBY's base64 class (i.e. might encode binary data)
	if(!targetDataType.inheritsFrom(MobyDataType.getDataType(MobyDataBytes.BASE64_DATATYPE, targetDataType.getRegistry()))){
	    //System.err.println("Data type does not inherit from Base64, continuing with string-based search...");
	    return getMobyObjects(new String(rawData), targetDataType);
	}
	//System.err.println("Data type " + targetDataType.getName() + " inherits from " + MobyDataBytes.BASE64_DATATYPE +
	//		   ", continuing with byte-based search...");

	String rawDataAsString = null;

	Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>();

	for(Pattern pattern: regexMap.keySet()){
	    MobyComplexBuilder rule = regexMap.get(pattern);
	    //System.err.print("Checking" + rule.getDataType().getName()+":");
	    //for(MobyDataType parent: rule.getDataType().getLineage()){System.err.print(" " + parent.getName());}
	    //System.err.println("");
	    if(rule.getDataType().inheritsFrom(targetDataType)){
		//System.err.println("Checking rule that generates data type " +  rule.getDataType().getName());
		// Only build the string representation of the byte array if we
		// found a rule that applies (since it's a somewhat expensive operation
		// to do the conversion)
		if(rawDataAsString == null){
		    rawDataAsString = bytesToString(rawData);
		    //System.err.println("Start of data is " + rawDataAsString.substring(0, 4));
		}
		try{
		    Matcher matcher = pattern.matcher(rawDataAsString);
		    
		    while(matcher.find()){
			//System.err.println("Found match for binary data");
			try{
			    MobyDataObject mobyObj = rule.apply(matcher, rawData);
			    if(mobyObj != null){
				objectVector.add(mobyObj);		
			    }
			}
			catch(MobyException me){
			    logger.log(Level.SEVERE, "Could not build Moby object from match", me);
			}
		    }
		} catch(Throwable t){ // Principally stack overflows
		    logger.log(Level.SEVERE, "Exception/Error ("+t+") while applying regex '"+pattern.pattern()+"' to binary data", t);
		}
	    }
	}

	if(objectVector.size() != 0){
	    return (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]);	
	}
	// Didn't find anything in the raw form, but there was a rule that applies to the data type, so try as a string...
	//else if(rawDataAsString != null){
	//    return getMobyObjects(new String(rawData));
	//}
	else{
	    return new MobyDataObject[0];
	}
    }

    //Pad out the bytes (8-bit) into chars (16-bits), for regex checking of the data
    private String bytesToString(byte[] bytes){
//       	return new String(bytes);
 	StringBuffer stringBuffer = new StringBuffer(bytes.length);

 	for(int i = 0; i < bytes.length; i++){
 	    // Casting byte to char pads it out with a byte e.g. 0x34 become 0x0034 (Unicode 16-bit)
 	    // Do this because regex use unicode, not ASCII
 	    stringBuffer.append((char) bytes[i]);
 	}
	
 	return stringBuffer.toString();
    }

    /**
     * Using the regular expression mappings that have been set up, 
     * maps a string to moby objects.
     */
    public MobyDataObject[] getMobyObjects(CharSequence textData){
	return getMobyObjects(textData, (byte[]) null);
    }

    /**
     * Same as getMobyObjects(String), but returns only those generated Moby objects that 
     * are derived from URN-addressable rules (which are the keys), mostly for provenance 
     * and reproducibility purposes.
     */
    public Map<String,MobyDataObject[]> getMobyObjectsURNMap(CharSequence textData){
	return getMobyObjectsURNMap(textData, (byte[]) null);
    }

    public Map<String,MobyDataObject[]> getMobyObjectsURNMap(CharSequence textData, byte[] bytes){
	Map results = new HashMap<String,MobyDataObject[]>();
	if(regexMap.isEmpty()){
	    logger.log(Level.WARNING, "The MOBY Client has not been provided any regex->moby data mappings!");
	    return results;
	}

	for(Pattern pattern: regexMap.keySet()){
	    // Vector of moby objects we construct based on the regex mappings found
	    Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>(); 
	    MobyComplexBuilder rule = (MobyComplexBuilder) regexMap.get(pattern);
	    String urn = rule.getURN();
	    try{
		Matcher matcher = pattern.matcher(textData);
		while(urn != null && matcher.find()){
		    try{
			MobyDataObject mobyObj = rule.apply(matcher, bytes);
			if(mobyObj != null){
			    objectVector.add(mobyObj);		
			}
		    }
		    catch(MobyException me){
			logger.log(Level.SEVERE, "Could not build Moby object from match (rule "+urn+")", me);
		    }
		}
	    }catch(Throwable t){ // Principally stack overflows
		logger.log(Level.SEVERE, "Exception/Error ("+t+") while applying regex '"+pattern.pattern()+"' to text/binary", t);
	    }
	    if(urn != null && !objectVector.isEmpty()){
		results.put(urn, (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]));
	    }
	}

	return results;
    }

    public MobyDataObject[] getMobyObjects(CharSequence textData, byte[] bytes){
	if(regexMap.isEmpty()){
	    logger.log(Level.WARNING, "The MOBY Client has not been provided any regex->moby data mappings!");
	    return new MobyDataObject[0];
	}

	// Vector of moby objects we construct based on the regex mappings found
	Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>(); 

	Iterator regex_keys = regexMap.keySet().iterator();
	while(regex_keys.hasNext()){ //map still has untraversed regex entry to test
	    Pattern pattern = (Pattern) regex_keys.next();
	    try{
		Matcher matcher = pattern.matcher(textData);
		
		while(matcher.find()){
		    MobyComplexBuilder rule = (MobyComplexBuilder) regexMap.get(pattern);
		    
		    try{
			MobyDataObject mobyObj = rule.apply(matcher, bytes);
			if(mobyObj != null){
			    objectVector.add(mobyObj);		
			}
		    }
		    catch(MobyException me){
			logger.log(Level.SEVERE, "Could not build Moby object from match", me);
		    }
		}
	    }catch(Throwable t){ // Principally stack overflows
		logger.log(Level.SEVERE, "Exception/Error ("+t+") while applying regex '"+pattern.pattern()+"' to text/binary", t);
	    }
	}

	return (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]);
    }

    /**
     * Using the URL regular expression mappings that have been set up, 
     * maps a string to moby objects.  URL regexs are special because they are
     * flexible in the order in which query parameters are given.  They
     * may also have requirements based on the hypertext linked. This method
     * does not look at the contents of the URL, but rather the URL itself.
     */
    public MobyDataObject[] getMobyObjects(URL url){
	return getMobyObjects(url, null, null);
    }

    public MobyDataObject[] getMobyObjects(URL url, String linkText){
	return getMobyObjects(url, linkText, null);
    }

    public MobyDataObject[] getMobyObjects(URL url, MobyDataType targetDataType){
	return getMobyObjects(url, null, targetDataType);
    }

    public MobyDataObject[] getMobyObjects(URL url, String linkText, MobyDataType targetDataType){
	if(urlRegexMap.isEmpty()){
	    logger.log(Level.WARNING, "The MOBY Client has not been provided any url regex->moby data mappings!");
	    return new MobyDataObject[0];
	}

	// Vector of moby objects we construct based on the regex mappings found
	Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>(); 

	String urlString = url.toString();
	Iterator url_regex_keys = urlRegexMap.keySet().iterator();
	while(url_regex_keys.hasNext()){ // map still has untraversed regex entry to test
	    Pattern pattern = (Pattern) url_regex_keys.next();
	    MobyComplexBuilder rule = (MobyComplexBuilder) urlRegexMap.get(pattern);

	    // If we asked for a particular data type, make sure the rule can produce it before we
	    // start to go matching it.
	    if(targetDataType != null && !rule.getDataType().inheritsFrom(targetDataType)){
		continue;
	    }
	    Matcher matcher = pattern.matcher(urlString);

	    while(matcher.find()){

		try{
		    MobyDataObject mobyObj = rule.apply(matcher);
		    if(mobyObj != null){
			objectVector.add(mobyObj);		
		    }
		}
		catch(MobyException me){
		    logger.log(Level.SEVERE, "Could not build Moby object from url regex match", me);
		}
	    }
	}

	return (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]);
    }

    /**
     * Using the XPath mappings that have been set up, maps a DOM node to 
     * moby objects.
     */
    public MobyDataObject[] getMobyObjects(Node n){

	// No xpaths to check
	if(xpathMap.isEmpty()){
	    logger.log(Level.WARNING, "The MOBY Client has not been provided any xpath->moby data mappings!");
	    return new MobyDataObject[0];
	}
	
	// Vector of moby objects we construct based on the xpath mappings found
	Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>(); 
	for(XPathExpression xpath: xpathMap.keySet()){ //map still has untraversed xpath entry to test
	    objectVector.addAll(applyXPath(n, xpath));
	}
	return (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]);
    }
	
    private Vector<MobyDataObject> applyXPath(Node current_node, XPathExpression xpath){
	Vector<MobyDataObject> objectVector = new Vector<MobyDataObject>(); 

	Object result = null;
	try{
	    MobyComplexBuilder rule = (MobyComplexBuilder) xpathMap.get(xpath);
	    
	    result = xpath.evaluate(current_node, XPathConstants.NODESET);
	    
	    // More than one hit?
	    if(result != null){
		try{
		    NodeList node_list = (NodeList) result; 
		    //System.out.println("number of nodes in the nodelist: " + node_list.getLength()); 
		    if(node_list != null && node_list.getLength() != 0){
			for(int i = 0; i < node_list.getLength(); i++){
			    MobyDataObject mobyObj = rule.applyXPath(node_list.item(i), nsContext);
			    if(mobyObj != null){
				objectVector.add(mobyObj);
			    }
			}
		    }
		    
		}catch(XPathExpressionException xpe2){
		    logger.log(Level.WARNING, "Warning: Cannot access resulting node list "+
			       "due to exception in its retrieval: ", xpe2);
		    return new Vector<MobyDataObject>();
		}
	    }
	    else{
		//todo: re-eval why we do this! result = xpath.evaluate(current_node, XPathConstants.STRING);
		if(result != null){
		    if(!result.toString().equals("")){
			//System.out.println("Got string result: " + result);
			MobyDataObject mobyObj = rule.applyXPath(result, nsContext);
			if(mobyObj != null){
			    objectVector.add(mobyObj);
			}
		    }
		}
		else{ //not a node sequence
		    logger.log(Level.WARNING, "Warning: the XPath expression ("+ xpath +
			                      ") did not return a node set, cannot select document elements."+
				              " The returned object was of type "+ result.getClass().getName() );
		    return new Vector<MobyDataObject>();
		}
	    }		
	}catch(javax.xml.xpath.XPathExpressionException xpe){
	    //System.err.println("Warning: Cannot select nodes due to exception "+
	    //"in XPath expression ("+xpath+"):" + xpe);
	//xpe.printStackTrace();
	    return new Vector<MobyDataObject>();
	}catch(MobyException me){
	    logger.log(Level.WARNING, "Warning: Cannot create objects from select nodes due " +
		       " to exception in MOBY logic:", me);
	    return new Vector<MobyDataObject>();
	}
	
	return objectVector;
    }

    /**
     * Same as getMobyObjects(Node), but applies only those rules that have a URN associated with them,
     * mainly for provenance and reuse purposes.Map<String,MobyDataObject[]>
     */
    public Map<String,MobyDataObject[]> getMobyObjectsURNMap(Node n){
	Map results = new HashMap<String,MobyDataObject[]>();

	// No xpaths to check
	if(xpathMap.isEmpty()){
	    logger.log(Level.WARNING, "The MOBY Client has not been provided any xpath->moby data mappings!");
	    return results;
	}
	
	// Vector of moby objects we construct based on the xpath mappings found
	for(XPathExpression xpath: xpathMap.keySet()){ //map still has untraversed xpath entry to test
	    String urn = xpathMap.get(xpath).getURN();
	    if(urn == null){
		continue; // we only want named rules
	    }
	    Vector<MobyDataObject> objectVector = applyXPath(n, xpath); 
	    if(!objectVector.isEmpty()){
		results.put(urn, (MobyDataObject[]) objectVector.toArray(new MobyDataObject[objectVector.size()]));
	    }
	}
	return results;
    }

    public void setRequiredServiceLevel(int level){
	serviceLevel = level;
    }

    public MobyDataServiceAssocInstance getServices(MobyDataObjectSet mdos) throws MobyException{
	return (getServices(new MobyDataObjectSet[]{mdos}))[0];
    }

    public MobyDataServiceAssocInstance getServices(MobyDataObject mdo) throws MobyException{
	return (getServices(new MobyDataObject[]{mdo}))[0];
    }

    public MobyDataServiceAssocInstance[] getServices(MobyDataObjectSet[] mdoss) throws MobyException{ 
	return getServices((MobyPrimaryData[]) mdoss);
    }

    public MobyDataServiceAssocInstance[] getServices(MobyDataObject[] mdos) throws MobyException{ 
	return getServices((MobyPrimaryData[]) mdos);
    }

    protected MobyDataServiceAssocInstance[] getServices(MobyPrimaryData[] mdos) throws MobyException{     
	MobyDataServiceAssocInstance[] mobyDataServiceAssocInstances = new MobyDataServiceAssocInstance[mdos.length]; 

	for(int i = 0; i < mdos.length; i++){
	    MobyService templateServices = new MobyService("");
	    templateServices.addInput(mdos[i]);
	    // Normally category is "moby", which excludes async services. 
	    // Turn it into a wildcard instead by making the template category blank
	    templateServices.setCategory(""); 
	    MobyService[] mService = c.findService(templateServices);

	    Vector<MobyService> filteredServices = new Vector<MobyService>();
	    // Let's make sure all of the services have their service type properly instantiated
	    // for ontology checks later on
	    for(int j = 0; j < mService.length; j++){
		mService[j].setStatus(MobyService.ALIVE, isServiceAlive(mService[j]));
		mService[j].setServiceType(MobyServiceType.getServiceType(mService[j].getServiceType().getName(), 
		                                                          getRegistry()));
		if(serviceLevel != MobyService.UNCHECKED){
		    int serviceStatus = mService[j].getStatus();
		    //System.err.println("Service " + mService[j].getName() + " has status " + serviceStatus);
		    if((serviceStatus & serviceLevel) >= serviceLevel){
			filteredServices.add(mService[j]);		    
		    }
		}
	    }

	    if(serviceLevel != MobyService.UNCHECKED){
		mService = (MobyService []) filteredServices.toArray(new MobyService[filteredServices.size()]);
	    }

	    if(mdos[i] instanceof MobyDataObject){
		if(mdos[i] instanceof MobyDataObjectDeferred){
		    mobyDataServiceAssocInstances[i] = new MobyDataObjectDeferredSAI((MobyDataObjectDeferred) mdos[i], mService);
		}
		else{
		    mobyDataServiceAssocInstances[i] = new MobyDataObjectSAI((MobyDataObject) mdos[i], mService);
		}
	    }
	    else if(mdos[i] instanceof MobyDataObjectSet){
		if(mdos[i] instanceof MobyDataObjectSetDeferred){
		    mobyDataServiceAssocInstances[i] = new MobyDataObjectSetDeferredSAI((MobyDataObjectSetDeferred) mdos[i], mService);
		}
		else{
		    mobyDataServiceAssocInstances[i] = new MobyDataObjectSetSAI((MobyDataObjectSet) mdos[i], mService);
		}
	    }
	    else{
		logger.log(Level.WARNING, "MobyClient could not handle service-associating an instance of " + mdos[i].getClass());
		//System.err.println("MobyClient could not handle service-associating an instance of " + mdos[i].getClass());
	    }
	}
	return mobyDataServiceAssocInstances; 
    }

    public boolean isMapped(Node n){
	return true;
    }

    public void addRegexMapping(String regexp, String[] mobyObj){ //mobyObj<--mobyNamespaces
	addRegexMapping(regexp, mobyObj, null, null);
    }

    public void addRegexMapping(String regexp, String[] mobyObj, String articleName, String urn){ //mobyObj<--mobyNamespaces
	if(mobyObj == null){
	    logger.log(Level.WARNING, "Ignoring empty namespace-only regex rule mappings");
	    return;
	}

	Map<String,String> nsRules = new HashMap<String,String>();
	// Default the rule to taking the whole matched expression ($0)
	for(int i = 0; i < mobyObj.length; i++){
	    nsRules.put(mobyObj[i], "$0");
	}
	addRegexMapping(regexp, nsRules, articleName, (String) null, urn);
    }

    public void addURLRegexMapping(String url_regexp, String[] mobyObj, String articleName, String urn){ //mobyObj<--mobyNamespaces
	if(mobyObj == null){
	    logger.log(Level.WARNING, "Ignoring empty namespace-only url regex rule mappings");
	    return;
	}

	Map<String,String> url_nsRules = new HashMap<String,String>();
	// Default the rule to take the first saved grouping of the regex (whole URL won't be a good ID)
	for(int i = 0; i < mobyObj.length; i++){
	    url_nsRules.put(mobyObj[i], "$1");
	}
	addURLRegexMapping(url_regexp, url_nsRules, articleName, urn);
    }

    /**
     * Converts seahawk-specific regex syntax into generic Java syntax (e.g \N becomes a
     * match for any IUPAC DNA character, \P any amino acid, \M for any monosacchiride abbreviation).  
     * We pass in the member map so that
     * this method can populate the last field of rules that inherit complex members (by virtue 
     * of using the capture group whose values is derived from \p{ruleName}), with ruleName 
     */
    protected String processRegExp(String regex, Map<String,String[]> membersMap) throws Exception{
	String returnValue = regex.replaceAll("\\\\N", "[acgtunxACGTUNX]")
	    .replaceAll("\\\\P", "[ARNDCQEGHILKMFPSTWYVBZXarndcqeghilkmfpstwyvbz*]")
	    .replaceAll("\\\\M", "(?:Abe|IdoA|All|Lyx|Alt|Man|Api|Mur|Ara|Neu|Ara-ol|Neu5Ac|dRib|Neu2en5Ac|Fru|Neu5Gc|Fuc|Kdo|Fuc-ol|Rha|Gal|Rha3,4Me2|GalN|Psi|GalNAc|Qui|B-D-Galp4S|Rib|Glc|Rib5P|GlcN|Ribulo|Rul|GlcN3N|Sor|Glc-ol|Tag|GlcNAc|Tal|GlcA|Xyl|GlcpA6Et|Xylulo|Xul|Gul|Xyl2CMe|Ido)");
	// Now see if there are any references to other patterns (by rule name) 
	// with the \p{} syntax specific to Seahawk
	Pattern charClassPattern = Pattern.compile("\\\\p\\{([A-Za-z0-9]+)\\}");
	Matcher charClassMatcher = charClassPattern.matcher(returnValue);
	Map<Integer,String> capGroup2RuleReference = new HashMap<Integer,String>();
	while(charClassMatcher.find()){
	    String reference = charClassMatcher.group(1);
	    if(RegexParser.isPosixCharacterClass(reference)){
		continue;
	    }

	    if(!patternNameMap.containsKey(reference)){
		throw new Exception("\\p{"+reference+"} in regex does not refer " +
				    "to a known Seahawk rule, cannot build the regex");
	    }

	    int capGroup = RegexParser.locationToCaptureGroupNumber(regex, charClassMatcher.start(1));
	    if(capGroup > 0){  // sanity check
		capGroup2RuleReference.put(capGroup, reference);
	    }

	    // Replace ref with regex, elinating any nested capture groups, for efficiency
	    // (otherwise we'd need to shift all the $# refs in the rules map to compensate)
	    returnValue = returnValue.replaceFirst("\\\\p\\{"+reference+"\\}", 
						   patternNameMap.get(reference).pattern().replaceAll("\\\\", "\\\\\\\\").replaceAll("\\((?!\\?)", "(?:"));
	}

	if(!capGroup2RuleReference.isEmpty()){
	    // Update any member rule that uses one of the capture values referring to a
	    // \p{ruleName} reference
	    for(String[] rule: membersMap.values()){
		for(Integer capGroupNum: capGroup2RuleReference.keySet()){
		    if(rule[0].matches("^\\s*\\$"+capGroupNum+"\\s*$")){
			// This is where the rule actually gets the subrule reference update,
			// UNLESS it was already specified (probably by a inheritsMembers tag)
			if(rule[4] == null || rule[4].length() == 0){
			    rule[4] = capGroup2RuleReference.get(capGroupNum);
			}
			break;
		    }
		}
	    }
	}
	return returnValue;
    }

    /**
     * First check that the string has proper URL syntax, then builds a regex that ignores
     * the order of the query parameters (as this is inconsequential to the results the URL 
     * produces).
     */
    protected String processURLRegExp(String url_regex) throws MalformedURLException{
	// Populate an example matching URL to check the syntax
	String exampleURL = url_regex.replaceAll("\\\\d\\+?", "1");
	exampleURL = exampleURL.replaceAll("\\[.*?\\]", "foo");
	URL urlTest = new URL(exampleURL);  // May throw exception

	// Account for all permutations of the query string
	String url_regex_flexible = url_regex.replaceAll("", "");
	//url_regex_flexible = 

	return url_regex_flexible;
    }

    protected void addRegexMapping(String regexp, Map<String,String> nsRules, String articleName, String ruleName, String urn){ //nsRules = Map<String ns, String regex_replacement>
	if(nsRules == null || nsRules.size() == 0){
	    logger.log(Level.WARNING, "Ignoring empty namespace-only regex rule mappings");
	    return;
	}

	addRegexMapping(regexp, nsRules, (String) null, (Map<String,String[]>) null, articleName, null, urn);
    }

    protected void addURLRegexMapping(String url_regexp, Map<String,String> url_nsRules, String articleName, String urn){ //nsRules = Map<String ns, String regex_replacement>
	if(url_nsRules == null || url_nsRules.size() == 0){
	    logger.log(Level.WARNING, "Ignoring empty namespace-only url regex rule mappings");
	    return;
	}

	addURLRegexMapping(url_regexp, url_nsRules, (String) null, (Map<String,String[]>) null, articleName, urn);
    }

    public void addRegexMapping(String regexp, Map<String,String> nsRules, 
				String mobyDataType, Map<String,String[]> membersMap){
	addRegexMapping(regexp, nsRules, mobyDataType, membersMap, null, null, null);
    }

    /**
     * This method looks for HAS members in a rule, and if they exist creates new
     * capture groups around the accessed capture groups so that they can be further
     * processed in MobyComplexBuilder (e.g. (\d)+ become ((\d+)) so we can deconstruct
     * that part of the regex as capture *each* \d for the HAS (0 or more) member relationship.
     * We also need to modify all of the rules that access capture groups to bump up their
     * numbers due to the added capture groups here (transparent to the user).
     *
     * membersMap, nsRules, and articleName get their capture group references modified accordingly, 
     * and the new regex is returned with its extract capture groups.
     */
    private String handleHASMembers(String regexp, Map<String,String> nsRules, String mobyDataType,
				    Map<String,String[]> membersMap, StringBuffer articleName) throws Exception{
	if(mobyDataType == null){
	    //System.err.println("Got null data type for regex "+regexp);
	    return regexp; //must be a base object
	}
	MobyDataType dataType = MobyDataType.getDataType(mobyDataType, getRegistry());	
	if(dataType == null){
	    throw new Exception("Cannot find definition of data type "+mobyDataType+
				" in the ontology, therefore the rule cannot be properly parsed");
	}
	MobyRelationship[] memberRelationships = dataType.getAllChildren();
	
	String newRegexp = processRegExp(regexp, membersMap);
	Map<Integer,Boolean> captured = new HashMap<Integer,Boolean>();
	for(MobyRelationship memberRelationship: memberRelationships){
	    if(memberRelationship.getRelationshipType() == Central.iHAS){
		String[] rule = membersMap.get(memberRelationship.getName());
		if(rule == null){
		    logger.log(Level.WARNING, "Skipping HAS member "+memberRelationship.getName() + 
				              " without an explicit rule (may be defined by inheritance?)");		    
		    continue;
		}
		Pattern pattern = Pattern.compile(newRegexp, Pattern.DOTALL | Pattern.COMMENTS);
		int groupCount = RegexParser.groupCount(pattern);
		for(int i = 0; i < groupCount; i++){
		    if(captured.containsKey(i)){ //autoboxed int
			// Already encapsulated the capture group due to another
			// HAS member, don't need to add anything
			logger.log(Level.WARNING, "Skipping processing of capture group "+i+
					          ", it's already been processed by another member in this rule");
			continue;
		    }
		    if(rule[0].matches("^.*\\$"+i+"(?=\\D.*|\\z)")){
			//System.err.println("Substituting "+i+" with encapsulating capture group, " +
			//		   "due to HAS condition of member "+memberRelationship.getName() +
			//		   " with rule " + rule[0]);
			
			// Now actually update the regex with the new cap group
			// including any quantity modifier associated with it.
			boolean INCL_QUANTIFIER = true;
			int capGroupRange[] = RegexParser.getCaptureGroupRange(pattern, i, INCL_QUANTIFIER);
			newRegexp = newRegexp.substring(0, capGroupRange[0])+"("+
			    newRegexp.substring(capGroupRange[0], capGroupRange[1]+1) + ")" +
			    (capGroupRange[1]+1 < newRegexp.length() ? newRegexp.substring(capGroupRange[1]+1) : "");
			// Bump up all the capture group reference higher than this one, in all rules
			for(int j = i; j < groupCount; j++){
			    for(String memberName: membersMap.keySet()){
				String[] memberRule = membersMap.get(memberName);
				memberRule[0] = memberRule[0].replaceAll("\\$"+i+"(?=\\D.*|\\z)", "\\$"+(i+1));
			    }
			    for(String nsName: nsRules.keySet()){
				String nsRule = nsRules.get(nsName);
				nsRules.put(nsName, nsRule.replaceAll("\\$"+i+"(?=\\D.*|\\z)", "\\$"+(i+1)));
			    }
			    articleName.replace(0, articleName.length(), 
						articleName.toString().replaceAll("\\$"+i+"(?=\\D.*|\\z)", "\\$"+(i+1)));
			}
			captured.put(i, true); //autobox both key and value
			for(int j = groupCount; j >= i; j--){
			    captured.remove(j);
			    captured.put(j+1, true);
			}
		    }  //end if (rule contains group reference)
		} // end for(group count)
	    }  // end if (member's relationship is HAS)
	    else{
		//System.err.println("Relationship for member " + memberRelationship.getName() + " of "+ mobyDataType +
		//		   "is *not* HAS");
	    }
	}  //end for(member relationships)
	return newRegexp;
    }

    public void addRegexMapping(String regexp, Map<String,String> nsRules, String mobyDataType, 
				Map<String,String[]> membersMap, String articleName,
				String ruleName, String urn){ 
	try{
	    // Use a StringBuffer so it's mutable by handleHASMembers()
	    StringBuffer articleNameBuffer = new StringBuffer(articleName == null ? "" : articleName); 
	    regexp = handleHASMembers(regexp, nsRules, mobyDataType, membersMap, articleNameBuffer);

	    // Pattern.DOTALL to allow ".*" to span multiple lines, also allow comments (# to EOL) and whitespace
	    // for better readability in the rules file.
	    Pattern pattern = Pattern.compile(processRegExp(regexp, membersMap), Pattern.DOTALL | Pattern.COMMENTS);	

	    // Base object
	    MobyComplexBuilder mcb = null;
	    if(mobyDataType == null || mobyDataType.length() == 0){
		mcb = new MobyComplexBuilder("Object", 
					     membersMap, 
					     nsRules,
					     this,
					     articleNameBuffer.toString());
	    }
	    // Complex Object
	    else{
		mcb = new MobyComplexBuilder(mobyDataType, 
					     membersMap, 
					     nsRules,
					     this,
					     articleNameBuffer.toString());
	    }
	    mcb.setURN(urn);
	    regexMap.put(pattern, mcb);

	    if(ruleName != null && ruleName.length() != 0){
		patternNameMap.put(ruleName, pattern);
		builderNameMap.put(ruleName, regexMap.get(pattern));
	    }
	    if(urn != null && urn.length() != 0){
		patternNameMap.put(urn, pattern);
		builderNameMap.put(urn, regexMap.get(pattern));
	    }

	}catch(Exception e){
	    logger.log(Level.SEVERE, "Could not create regular expression statement from '" + regexp + "'", e);
	}
    }

    public void addURLRegexMapping(String url_regexp, Map<String,String> url_nsRules, 
				   String mobyDataType, Map<String,String[]> membersMap, 
				   String articleName, String urn){ //mobyObj<--mobyNamespaces
	//System.out.println("url regex addMapping: " + url_regexp);
	try{
	    Pattern pattern = Pattern.compile(processURLRegExp(url_regexp));	

	    // Base object
	    MobyComplexBuilder mcb = null;
	    if(mobyDataType == null || mobyDataType.length() == 0){
		mcb = new MobyComplexBuilder("Object", 
					     membersMap, 
					     url_nsRules,
					     this,
					     articleName);
	    }
	    else{
		// Complex Object
		mcb = new MobyComplexBuilder(mobyDataType, 
					     membersMap, 
					     url_nsRules,
					     this,
					     articleName);
	    }
	    mcb.setURN(urn);
	    urlRegexMap.put(pattern, mcb);

	    if(urn != null && urn.length() != 0){
		builderNameMap.put(urn, mcb);
	    }
	}catch(Exception e){
	    logger.log(Level.SEVERE, "Could not create URL regular expression statement from '" +
			             url_regexp + "': " + e);
	    e.printStackTrace();
	}
    }

    public void addXPathMapping(String xpath, String[] mobyObj){ //mobyObj<--mobyNamespaces
	if(mobyObj == null){
	    logger.log(Level.WARNING, "Ignoring empty namespace-only regex rule mappings");
	    return;
	}

	Map<String,String> nsRules = new HashMap<String,String>();
	// Defautl the rule to taking the whole matched expression ($0)
	for(int i = 0; i < mobyObj.length; i++){
	    nsRules.put(mobyObj[i], ".");
	}
	addXPathMapping(xpath, nsRules, null, null);
    }

    public void addXPathMapping(String xpath_exp, Map<String,String> nsRules, String articleName, String urn){ //mobyObj<--mobyNamespaces
	addXPathMapping(xpath_exp, nsRules, (String) null, (Map<String,String[]>) null, articleName, urn);
    }

    public void addXPathMapping(String xpath_exp, Map<String,String> nsRules, String mobyDataType, 
				Map<String,String[]> membersMap, String articleName, String urn){ //mobyObj<--mobyNamespaces
	//System.out.println("xpath addMapping: " + xpath_exp);
	try{
	    XPathExpression xpath = commonXPath.compile(xpath_exp);

	    // Base object
	    MobyComplexBuilder mcb = null;
	    if(mobyDataType == null || mobyDataType.length() == 0){
		mcb = new MobyComplexBuilder("Object", 
					     membersMap, 
					     nsRules,
					     this,
					     articleName);		
	    }
	    else{
		// Complex Object
		mcb = new MobyComplexBuilder(mobyDataType, 
					     membersMap, 
					     nsRules,
					     this,
					     articleName);
	    }
	    mcb.setURN(urn);
	    xpathMap.put(xpath, mcb);

	    if(urn != null && urn.length() != 0){
		builderNameMap.put(urn, mcb);
	    }
	}catch(Exception e){
	    logger.log(Level.WARNING, 
		       "Could not create XPath select statement from '" +
		       xpath_exp + "': ", e);
	}
    }

    /**
     * Removes all mapping rules.
     */
    public void clear(){
	xpathMap.clear();
	regexMap.clear();
	(new Exception()).printStackTrace();
    }

    public void clearXPaths(){
	xpathMap.clear();
    }

    public void clearRegexs(){
	regexMap.clear();
	(new Exception()).printStackTrace();
    }

    public void clearURLRegexs(){
	urlRegexMap.clear();
    }

    /**
     * Indicates whether at least one production rule exists for the data type or one of its children.
     */
    public boolean canProduceDataTypeFromString(MobyDataType targetDataType){
	//System.err.println("The rules database has " + regexMap.size() + " entries");
	for(MobyComplexBuilder rule: regexMap.values()){
	    //System.err.println("Checking regex rule " + rule);
	    if(rule.getDataType().inheritsFrom(targetDataType)){
		return true;
	    }
	}
	return false;
    }

    /**
     * @return true unless we can contact ValidateService, and it tells us the service is dead 
     */
    private boolean isServiceAlive(MobyService service){
	// first time method is called
	if(isDeadMap == null){
	    isDeadMap = new HashMap<String,String>(); // keys are authority:serviceName
	    try{
		// Note, since it's not part of the standard API, this only 
		// works for services listed in the default Central
		URL u = new URL(IS_ALIVE_SERVICE_URL);
		LineNumberReader reader = new LineNumberReader(new InputStreamReader(u.openStream()));
		String currentAuthority = null;
		for(String line = reader.readLine();
		    line != null; 
		    line = reader.readLine()){
		    // authority declarations look like "  host.domain.tld"
		    if(line.indexOf("  ") == 0){
			currentAuthority = line.trim();
		    }
		    else if(line.indexOf("\t") == 0){
			String[] fields = line.trim().split(",");
			if(fields.length != 2){ 
			    logger.log(Level.WARNING, "Unrecognized line (not 2 comma delimited fields) " +
					              "from ValidateService: " + line.trim());
			}
			else if(fields[1].equals("true")){
			    // service is okay, don't add to dead map
			}
			else if(fields[1].equals("false")){
			    isDeadMap.put(currentAuthority+":"+fields[0], "dead");
			}
			else{
			    logger.log(Level.WARNING, "Unrecognized line (second field not 'true' or 'false') " +
					              "from ValidateService: " + line.trim());			    
			}
		    }
		}
		    
	    } catch(Exception e){
		e.printStackTrace();
		// will not attempt to retrieve map again, as it's not null anymore
		return true;
	    }
	}

	if(isDeadMap.containsKey(service.getAuthority()+":"+service.getName())){
	    return false;  // dead for sure
	}
	return true;  // be optimistic by default, assume it's alive
    }

    /**
     * If a rule was given this name, or url/lsid, the MOBY object builder for the rule is returned.
     */
    public MobyComplexBuilder getBuilder(String ruleName){
	return builderNameMap.get(ruleName);
    }

    /**
     * If a rule was given this name, the regex pattern for the rule is returned.
     */
    public Pattern getPattern(String ruleName){
	return patternNameMap.get(ruleName);
    }
}