package ca.ucalgary.services;

import ca.ucalgary.services.util.*;

import ca.ucalgary.seahawk.services.MobyClient;
import ca.ucalgary.seahawk.services.TextClient;

import org.biomoby.service.MobyServlet;
import org.biomoby.shared.*;
import org.biomoby.shared.data.*;

import java.io.*;
import java.math.*;
import java.net.URL;
import java.util.*;
import java.util.regex.Pattern;

/**
 * A servlet that will read in an EMBOSS ACD file (which describes
 * the command-line usage of a program in the suite) and publishes
 * it as a MOBY service.
 * 
 * MOBY XML data is converted to regular input to EMBOSS input via an XSLT
 * stylesheet.  EMBOSS results are conveted to MOBY datatypes using some
 * built-in heuristics.
 */
public class ACDService extends LegacyService{
    // Used for web.xml configuration
    public final static String EMBOSS_ROOT_PARAM = "embossRoot";
    public final static String EMBOSS_PARAMS_PARAM = "embossParams";
    public final static String EMBOSS_ADV_PARAMS_PARAM = "embossUseAdvancedParams";
    public final static String EMBOSS_OUTPUT_PARAM = "embossOutput";
    public final static String ACD_FILE_PARAM = "acdFile";    

    // Used for program execution
    private File programBinaryFile;
    private String embossRootDirName;
    private String acdRootDirName;

    // Keep track of what MOBY input parameters are to be converted to what ACD types
    private Map<String,String> acdTypes;
    private Map<String,String> acdBasicTypes;  //holds e.g. string, when acdType for same key is "nucleotide sequence"
    // For launching command-line programs
    private Runtime runtime;
    Vector<MobyDataSecondaryInstance> fixedSecondaryData;
    Pattern outputPattern;
    String outputPatternName;

    public void init(){
	super.init();
	acdTypes = new HashMap<String,String>();
	acdBasicTypes = new HashMap<String,String>();
	runtime = Runtime.getRuntime();
	fixedSecondaryData = new Vector<MobyDataSecondaryInstance>();
    }

    /**
     * Runs the command-line program specified by the servlet config,
     * reformatting along the way.
     */
    public void processRequest(MobyDataJob request, MobyDataJob result) throws Exception{
	// Run the program.  The result is for each output param, a list (vector) of the raw data (byte array)
	// Maps are used because, for example, an output may be composed of several png files.
	Map<String, Map<String, byte[]>> resultData = configAndRunProgram(request);
	MobyPrimaryData[] mobyOutputTemplates = getService().getPrimaryOutputs();

	if((resultData == null || resultData.size() == 0) && 
	   mobyOutputTemplates != null && mobyOutputTemplates.length > 0){
	    throw new MobyServiceException(MobyServiceException.WARNING, 
					   MobyServiceException.INTERNAL_PROCESSING_ERROR, 
					   request.getID(),
					   "No data was retrieved from the command-line program, even though it "+
					   " appears to have executed properly." + 
					   "Cannot return any results (code bug?).");
	}

	// Turn the results text into MOBY XML	
	for(MobyPrimaryData mobyOutputTemplate: mobyOutputTemplates){
	    String outputName = mobyOutputTemplate.getName();

	    Map<String, byte[]> resultParts = resultData.get(outputName);
	    if(resultParts == null){
		String paramChoices = "";
		for(String paramName: resultData.keySet()){
		    paramChoices += " \"" + paramName + "\"";
		}
		throw new MobyServiceException(MobyServiceException.WARNING, 
					       MobyServiceException.INTERNAL_PROCESSING_ERROR, 
					       request.getID(), 
					       "The requested output parameter in the servlet configuration (" +
					       outputName + " does not exist in the program output, please " +
					       " ask the service provider to correct the output parameter name." +
					       "Valid choices from this run are:" + paramChoices);
	    }

	    // Transform the bits of binary data into a MOBY object or collection
	    MobyDataInstance mdi = getMobyData(resultParts, mobyOutputTemplate);
	    if(mdi == null){
		throw new MobyServiceException(MobyServiceException.WARNING, 
					       MobyServiceException.INTERNAL_PROCESSING_ERROR, 
					       request.getID(), 
					       "The requested output parameter in the servlet configuration (" +
					       outputName + ") could not be created from the program's results data, please " +
					       " ask the service provider to correct the output transformation rules");
	    }
	    result.put(outputName, mdi);
	}

    }

    /**
     * Convert MOBY data (as required by the service) to text and
     * formulate the command from the given options.  Run the command
     * and return all of the output given.
     */
    private Map<String,Map<String, byte[]>> configAndRunProgram(MobyDataJob request) throws Exception{
	Vector<String> command = new Vector<String>();
	command.add(programBinaryFile.getAbsolutePath());

	MobyPrimaryData[] inputs = getService().getPrimaryInputs();

	// Param:tempFile Map
	Map<String, File> tempFiles = new HashMap<String, File>(); 

	// Primary input
	byte[] stdin = null;
	for(MobyPrimaryData mobyInputTemplate: inputs){
	    // Retrieve the input with the same name as the service template specifies
	    String paramName = mobyInputTemplate.getName();
	    MobyDataInstance inputData = request.get(paramName);

	    // Transform the moby data to text, unless it's binary data, which will be passed as decoded bytes
	    // Now, for binary data, we have to ignore any fields other than the Base64 encoded one.  Sorry!
	    String tempFileSuffix = ".txt";
	    if(inputData instanceof MobyDataBytes){
		tempFileSuffix = ".bin";
	    }
	    byte[] inputDataBytes = getLegacyData(inputData, acdTypes.get(mobyInputTemplate.getName()));
	    if(inputDataBytes == null){
		throw new NullPointerException("The TextClient returned null after transforming the " +
					       "input parameter "  + mobyInputTemplate.getName() + 
					       " to text type " + acdTypes.get(mobyInputTemplate.getName()));
	    }

	    // Create the required command-line flag for the parameter

	    // Is it a primary param whose value must be passed in directly, as opposed to in a file?
	    String basicType = acdBasicTypes.get(paramName);
	    if(basicType != null && 
	       (basicType.equals("string") || basicType.equals("integer") ||
		basicType.equals("float") || basicType.equals("boolean"))){
		// If only one input, we can pass it via stdin 
		command.add("-"+paramName);
		command.add(new String(inputDataBytes));
	    }
	    else{
		// Otherwise we need to create temporary files to store the data
		File tempFile = File.createTempFile("ACDService."+programBinaryFile.getName()+".input."+paramName, 
						    tempFileSuffix);
		command.add("-"+paramName);
		command.add(tempFile.toString());
		tempFiles.put(paramName, tempFile);
		// Write the data to the file
		FileOutputStream fileOS = new FileOutputStream(tempFile);
		fileOS.write(inputDataBytes);
		fileOS.close();
	    }
	}

	// User-selected Secondary input
 	for(MobyDataSecondaryInstance mobySecondary: request.getSecondaryData()){
 	    // ASSUMPTION: Don't need to make sure that no funny business is 
	    // going on with shell escape characters in the secondary names and 
	    // values (potential security problem), because we call the string 
	    // array form of Runtime.exec() later, which does not invoke a shell.
 	    command.add("-"+mobySecondary.getName());
	    command.add(mobySecondary.getObject().toString());
 	}

	// Fixed secondary inputs (via the web.xml)
	for(MobyDataSecondaryInstance mobySecondary: fixedSecondaryData){
	    command.add("-"+mobySecondary.getName());
	    command.add(mobySecondary.getObject().toString());	    
	}

	MobyPrimaryData[] outputs = getService().getPrimaryOutputs();
	Map<File,Boolean> outputFile = new HashMap<File,Boolean>();

	// Primary output
	for(MobyPrimaryData mobyOutputTemplate: outputs){
	    // Retrieve the output with the same name as the service template specifies
	    String paramName = mobyOutputTemplate.getName();

	    // Create the required command-line flag for the parameter
	    if(outputs.length == 1){
		// If only one output, we can grab it via stdout 
		command.add("-"+paramName);
		command.add("stdout");
	    }
	    else{
		// Otherwise we need to create temporary files to store the data
		String tempFileSuffix = ".txt";
		if(mobyOutputTemplate.getDataType().inheritsFrom(binaryDataType)){
		    tempFileSuffix = ".bin";
		}
		File tempFile = File.createTempFile("ACDService."+programBinaryFile.getName()+".output."+paramName,
						    tempFileSuffix);
		command.add("-"+paramName);
		command.add(tempFile.toString());
		tempFiles.put(paramName, tempFile);
		outputFile.put(tempFile, true);
	    }
	}

	// If the command creates fixed-name outputs, we need to make sure we run
	// the command in a unique temporary directory so that concurrent runs don't 
	// interfere with each other.
	File workingDir = null;  // null = use default, shared temp dir
	if(outputPattern != null){
	    workingDir = TempDir.createTempDir("ACDService."+programBinaryFile.getName()+"."+outputPatternName, null);
	}

	String[] commandStringArray = (String[]) command.toArray(new String[command.size()]);
	String stdout = runProgram(request, commandStringArray, workingDir, stdin);

	Map<String, Map<String, byte[]>> results = new HashMap<String, Map<String, byte[]>>();

	if(outputs.length == 1){ 
	    Map<String, byte[]> map = new HashMap<String, byte[]>();
	    map.put(MobyClient.SINGLE_RETURNED_VALUE_KEY, stdout.getBytes());
	    results.put(outputs[0].getName(), map);
	}

	// Load up the fixed-name results files, if any exist
	if(outputPattern != null){
	    // Determine if files matching the pattern were created by the program
	    Vector<File> fixedNameOutputFiles = new Vector<File>();
	    for(String fileName: workingDir.list()){
		// Skip hidden files, current and parent dirs
		if(fileName.indexOf(".") == 0){
		    continue;
		}
		if(outputPattern.matcher(fileName).find()){
		    fixedNameOutputFiles.add(new File(workingDir, fileName));
		}
	    }

	    // Load the files
	    Map<String, byte[]> map = new HashMap<String, byte[]>();
	    for(File fixedNameFile: fixedNameOutputFiles){
		if(outputFile.containsKey(fixedNameFile)){
		    // Name each value according to the file it came from
		    map.put(fixedNameFile.getName(), getFileData(fixedNameFile));
		}
	    }
	    // 0 to n byte arrays (representing 0 to n files' contents) added as a single output parameter
	    results.put(outputPatternName, map);

	    // Remove the directory and all of its contents
	    TempDir.delete(workingDir);
	}

	// Cleanup temp files used during program invocation
	for(Map.Entry<String,File> tempFileMapping: tempFiles.entrySet()){
	    File tempFile = tempFileMapping.getValue();
	    if(outputFile.containsKey(tempFile)){		
		Map<String, byte[]> map = new HashMap<String, byte[]>();
		map.put(MobyClient.SINGLE_RETURNED_VALUE_KEY, getFileData(tempFile));
		results.put(tempFileMapping.getKey(), map);
	    }
	    tempFile.delete();
	}
	return results;
    }

    private byte[] getFileData(File file) throws Exception{
	// Slurp the result file all in at once (assumes file smaller than 2 GB, the max value of an int)
	byte[] tempFileContents = new byte[(int) file.length()];
	FileInputStream fileInputStream = new FileInputStream(file);
	fileInputStream.read(tempFileContents, 0, (int) file.length());
	fileInputStream.close();
	return tempFileContents;
    }

    /**
     * @return the standard output of the command
     */
    private String runProgram(MobyDataJob request, String[] command, File workingDir, final byte[] input) throws Exception{
	// Ensure $embossRootDirName/lib is in the LD_LIBRARY_PATH, 
	// what is the equivalent in Windows?  Windows searches just PATH...
	// PATH should therefore include the lib and bin dirs of EMBOSS (bin because
	// programs like emma call other programs such as clustalw
	String libDir = embossRootDirName+File.separator+"lib";
	String binDir = programBinaryFile.getParent();
	String dataDir = embossRootDirName+File.separator+"share"+File.separator+"EMBOSS"+File.separator+"data";
	final Process process = runtime.exec(command, 
					     new String[]{"EMBOSS="+embossRootDirName,
							  "EMBOSS_ACDROOT="+acdRootDirName,
							  "EMBOSS_DATA="+dataDir,
					                  "LD_LIBRARY_PATH="+libDir,
					                  "PATH="+libDir+File.pathSeparator+binDir},
					     workingDir);
	final OutputStream stdin = process.getOutputStream();
	final InputStream stderr = process.getErrorStream();
	final InputStream stdout = process.getInputStream();
	final StringBuffer output = new StringBuffer();

	// Echo the program's stderr
	new Thread(){
		public void run(){
		    try{
			int c;
			while((c = stderr.read()) != -1){
			    System.err.write(c);
			}
			stderr.close();
		    }
		    catch(Exception e){
			e.printStackTrace();
		    }
		}
	    }.start();

	// Write the data to the command's stdin, in one fell swoop
	new Thread(){
		public void run(){
		    try{
			stdin.write(input);
			stdin.flush();
			stdin.close();
		    }
		    catch(Exception e){
			// Okay, so sometimes Java sits there waiting to write data
			// even after the write call (due to buffering?), then throws
			// an Exception because the stream was closed in the main thread
			// below.  How do we distinguish between genuine write issues and this?
			// By checking if the job is finished already and exited normally and produced data.  
			// If so, don't bother reporting the error, otherwise report it
			// because the program may be missing data!
			try{
			    Thread.sleep(1000);}
			catch(Exception te){
			    System.err.println("Warning: grace period for stdin close " +
					       "not met, thread was interrupted: " + te);
			}
			//grace period between stream close and process end 
			boolean completed = true;
			int eVal = 0;
			try{
			    eVal = process.exitValue();
			} catch(IllegalThreadStateException itse){
			    completed = false;
			}
			// 3 conditions, from more most severe to least
			if(!completed){
			    System.err.println("Caught exception while sending data to the command" +
					       "(while process is still running): "+e);
			}
			else if(eVal != 0){
			    System.err.println("Caught exception while sending data to the command" +
					       "(and the process had a non-zero return value): "+e);
			}
			else if(output.length() == 0){
			    System.err.println("Caught exception while sending data to the command " +
					       "(and the process output is blank): "+e);
			    // Not *necessarily* a real error 
			    // (grace time may be exceeded), but maybe, 
			    // so print it anyway
			    e.printStackTrace();
			}
			else{
			    return; //all okay, we only got here becuase of the java superfluous buffering issue
			}
			e.printStackTrace();
		    }
		}
	    }.start();
	
	try{
	    byte[] data = new byte[1024];
	    for(int c = stdout.read(data, 0, data.length); c != -1; c = stdout.read(data, 0, 1024)){
		output.append(new String(data, 0, c));
	    }
	}
	catch(Exception e){
	    throw new Exception("Caught exception while receiving data from the command-line program: "+e);
	}
		    
	int exitVal = process.waitFor();
	if(exitVal != 0){
	    addException(new MobyServiceException(MobyServiceException.WARNING, 
						  MobyServiceException.UNKNOWN_STATE, 
						  request.getID(), 
						  "The underlying program returned a non-zero " +
						  "value upon exiting, indicating abnormal termination " +
						  "therefore the results may not be correct"));
	}

	try{
	    //Thread.sleep(500);  // give the reading thread a generous 0.5 seconds to read the last 1024 bytes
	    stdout.close(); // this should flush any remaining data to go to the file in the thread above
	}
	catch(Exception e){
	    System.err.println("Caught exception while closing the command's output stream: "+e);
	    e.printStackTrace();
	}
	
	return output.toString();
    }

    /**
     * Determines the command-line program to execute when service requests come in,
     * based on the values for the EMBOSS_ROOT_PARAM and ACD_FILE_PARAM parameters in
     * the servlet configuration.
     */
    public MobyService createServiceFromConfig(javax.servlet.http.HttpServletRequest request)
	throws java.lang.Exception{
	MobyService service = super.createServiceFromConfig(request);

	if(getCoCInitParameter(EMBOSS_ROOT_PARAM) != null){
	    embossRootDirName = getCoCInitParameter(EMBOSS_ROOT_PARAM);
	}
	else{
	    throw new Exception("No parameter called " + EMBOSS_ROOT_PARAM + 
				" was found in the servlet configuration");
	}
	if(embossRootDirName.length() == 0){
	    throw new Exception("Parameter " + EMBOSS_ROOT_PARAM +
				" was blank in the servlet configuration");
	}
	File embossRootDir = new File(embossRootDirName);
	if(!embossRootDir.exists()){
	    throw new Exception("Parameter " + EMBOSS_ROOT_PARAM +
				" in the servlet configuration (" + embossRootDirName + 
				") doesn't exist on the filesystem");
	}
	if(!embossRootDir.isDirectory()){
	     throw new Exception("Parameter " + EMBOSS_ROOT_PARAM +
				" in the servlet configuration (" + embossRootDirName + 
				") exists, but is not a directory, as expected");
	}
	// See if it's really the emboss root (has bin dir)
	String slash = File.separator;
	String embossBinDirName = embossRootDirName+slash+"bin";
	File embossBinDir = new File(embossBinDirName);
	if(!embossBinDir.exists()){
	    throw new Exception("The EMBOSS binaries directory inferred from " +
				"the servlet configuration (" + embossBinDirName + 
				") doesn't exist on the filesystem");
	}
	if(!embossBinDir.isDirectory()){
	     throw new Exception("The EMBOSS binaries directory inferred from " +
				"the servlet configuration (" + embossBinDirName + 
				") exists, but is not a directory, as expected");
	}

	String programName = null;
	if(getServletConfig() != null){
	    programName = getServletConfig().getServletName();
	}
	if(programName == null && getServletContext() != null){
	    programName = getServletContext().getServletContextName();
	}
	if(programName == null || programName.length() == 0){
	    throw new Exception("Could not determine the program name, no servlet " +
				"name or servlet context name is available");
	}
	if(programName.length() == 0){
	    throw new Exception("The program name is blank, based on the available " +
				"servlet and servlet context names");
	}

	// See that the emboss root and program work together
	programBinaryFile = new File(embossBinDir, programName);
	if(!programBinaryFile.exists()){
	    throw new Exception("The program name inferred from the servlet " +
				"configuration (" + programBinaryFile.getPath() + 
				") does not exist");
	}
	if(!programBinaryFile.isFile()){
	    throw new Exception("The program name inferred from the servlet " +
				"configuration (" + programBinaryFile.getPath() + 
				") exists, but is not a file, as expected");
	}

	// The default location EMBOSS uses is the default we'll use...
	String acdFileName = embossRootDir.getPath()+slash+"share"+slash+"EMBOSS"+slash+
	                     "acd"+slash+programName+".acd";
	// Unless overriden in the config...
	if(getCoCInitParameter(ACD_FILE_PARAM) != null){
	    acdFileName = getCoCInitParameter(ACD_FILE_PARAM);
	    if(acdFileName.length() == 0){
		throw new Exception("Parameter " + ACD_FILE_PARAM +
				    " was blank in the servlet configuration");
	    }
	}
	File acdFile = new File(acdFileName);
	if(!acdFile.exists()){
	    throw new Exception("The ACD file (" + acdFileName + 
				") doesn't exist on the filesystem");
	}
	if(!acdFile.isFile()){
	    throw new Exception("The ACD file (" + acdFileName + 
				") exists, but is not a file, as expected");
	}
	acdRootDirName = acdFile.getParent();

	boolean useAdvancedParams = Boolean.parseBoolean(getCoCInitParameter(EMBOSS_ADV_PARAMS_PARAM));

	// All the parameters have been specified correctly, now check the ACD file
	// and create the MOBYService signature from it.
	try{
	    configureServiceFromACDFile(service, acdFile, useAdvancedParams);
	}
	catch(Exception e){
	    log("While parsing the ACD file (" + acdFileName + ")", e);
	    throw new Exception("While parsing the ACD file (" + acdFileName + "): " + e);
	}

	// Override params? (will also add non-existing params if you really want)
	if(getCoCInitParameter(EMBOSS_PARAMS_PARAM) != null){
	    for(String param: getCoCInitParameter(EMBOSS_PARAMS_PARAM).split(",")){
		String[] specs = param.split(":");
		if(specs.length != 2){		    
		    log("While parsing the " + EMBOSS_PARAMS_PARAM + " specs, item \""+
			param +"\" did not have the expected \"flag:value\" format, ignoring");
		    continue;
		}
		MobyDataSecondaryInstance fixedSecondary = 
		    new MobyDataSecondaryInstance(new MobySecondaryData(specs[0]), specs[1]);
		// Will remove existing value specified by the ACD file, if any exists
		service.removeInput(fixedSecondary);
		fixedSecondaryData.add(fixedSecondary);
	    }
	}

	// A parameter telling us what hard-coded output file name patterns
	// are used in the ACD program (i.e. ones we can't specify ourselves on the command line,
	// so we need to know them in order to hand them back to the user).
	if(getCoCInitParameter(EMBOSS_OUTPUT_PARAM) != null){
	    String[] specs = getCoCInitParameter(EMBOSS_OUTPUT_PARAM).split(":");
	    if(specs.length != 2){
		throw new Exception("While parsing the " + EMBOSS_OUTPUT_PARAM + "  specs, "+
				    "the value did not have the expected \"name:regex\" format"); 
	    }
	    if(specs[0] == null || specs[0].length() == 0){
		throw new Exception("While parsing the " + EMBOSS_OUTPUT_PARAM + "  specs, "+
				    "the value did not have the expected \"name:regex\" format (name length was 0)");
	    }
	    if(specs[1] == null || specs[1].length() == 0){
		throw new Exception("While parsing the " + EMBOSS_OUTPUT_PARAM + "  specs, "+
				    "the value did not have the expected \"name:regex\" format (regex length was 0)");
	    }
	    outputPatternName = specs[0];
	    try{
		outputPattern = Pattern.compile(specs[1]);
	    }
	    catch(Exception e){
		log("While compiling the " + EMBOSS_OUTPUT_PARAM + 
		    " pattern (" + specs[1] + "):", e);
		throw new Exception("While compiling the " + EMBOSS_OUTPUT_PARAM + 
				    " pattern (" + specs[1] + "):" + e);
	    }	    
	}
	
	return service;
    }

    /**
     * Parses the ACD file and sets the MOBY signature parameters appropriately.
     */
    public void configureServiceFromACDFile(MobyService service, File acdFile, boolean useAdvancedParams) 
	throws Exception{	
	ACDFile parsedACDData = new ACDFile(acdFile);

	configureServiceFromACDApplication(service, parsedACDData.getApplicationSection());
	List<Map<String,String>> combinedInput = new ArrayList<Map<String,String>>(parsedACDData.getInputSection());
	combinedInput.addAll(parsedACDData.getRequiredParamsSection());
	configureServiceFromACDInput(service, combinedInput);
	//configureServiceFromACDInput(service, parsedACDData.getInputSection());
	//configureServiceFromACDInput(service, parsedACDData.getRequiredParamsSection());
	configureServiceFromACDParams(service, parsedACDData.getAdditionalParamsSection());
	if(useAdvancedParams){
	    configureServiceFromACDParams(service, parsedACDData.getAdvancedParamsSection());
	}
	configureServiceFromACDOutput(service, parsedACDData.getOutputSection());
    }
    
    protected void configureServiceFromACDApplication(MobyService service, 
						      List<Map<String,String>> blocks) throws Exception{
	if(blocks == null){
	    throw new Exception("The application definition section of the ACD file is missing (null)");
	}

	//System.err.println("application data is:\n"+spec);
	if(blocks.size() != 1){
	    throw new Exception("Expected 1 block of data for the ACD file's application section, but got " + blocks);
	}

	Map<String,String> entries = blocks.get(0);
	if(!"application".equals(entries.get(ACDFile.BLOCK_TYPE_KEY))){
	    throw new Exception("Expected the ACD file's application section to be called \"application\", " +
				"but got \"" + entries.get(ACDFile.BLOCK_TYPE_KEY) + "\"");
	}

	// service.setName(entries.get(ACDFile.BLOCK_NAME_KEY));
	// Get the service description
	if(entries.containsKey("documentation")){
	    service.setDescription(entries.get("documentation"));
	}
    }

    /**
     * We must ensure that every MOBY input object has a corresponding ACD input
     * parameter that matches its name, and that we aren't stuffing an MOBY collection
     * into a service that takes a simple ACD input (the other way is okay, 
     * a MOBY simple coerced into a collection of 1).
     */
    protected void configureServiceFromACDInput(MobyService service, 
						List<Map<String,String>> spec) throws Exception{
	MobyPrimaryData[] mobyInputTypes = service.getPrimaryInputs();
	Map<String,MobyData> paramUnused = new HashMap<String,MobyData>();

	for(MobyPrimaryData mobyPrimaryInput: mobyInputTypes){
	    paramUnused.put(mobyPrimaryInput.getName(), mobyPrimaryInput);
	}
	// There is a chance that the input was fixed as a secondary
	for(MobySecondaryData mobySecondaryInput: service.getSecondaryInputs()){
	    paramUnused.put(mobySecondaryInput.getName(), mobySecondaryInput);
	}	

	for(MobyData acdInput: specToMoby(spec)){
	    if(acdInput instanceof MobyPrimaryData){
		String acdInputName = acdInput.getName();

		// Make sure it maps to a MOBY Input
		if(!paramUnused.containsKey(acdInputName)){
		    String declInputNames = "";
		    for(String inputName: paramUnused.keySet()){
			declInputNames = inputName+" ";
		    }
		    throw new Exception("A required ACD input parameter ("+acdInputName+
					") does not have a matching declaration in the " +
					"servlet configuration ( "+declInputNames+")");
		}
		MobyData mobyPrimaryInput = paramUnused.get(acdInputName);
		// A fixed value was given rather than supplying a param...
		if(paramUnused.get(acdInputName) instanceof MobySecondaryData){
		    paramUnused.remove(acdInputName);
		    continue;
		}

		if(!canProduceTextTypeFromMoby(acdTypes.get(acdInputName), 
					       (MobyPrimaryData) mobyPrimaryInput)){
		    throw new Exception("No XSLT rules exist that can produce the requested " +
					"text type '" + acdTypes.get(acdInputName) + 
					"' (acd input parameter " + acdInputName + 
					") from the given MOBY object type (" + 
					((MobyPrimaryData) mobyPrimaryInput).getDataType().getName()+")");
		}

		// If it exists, make sure it's of the right type
		if(acdInput instanceof MobyPrimaryDataSimple &&
		   paramUnused.get(acdInputName) instanceof MobyPrimaryDataSet){
		    System.err.println("Potential issue: The MOBY input parameter \""+acdInputName+
					"\" is a Collection, but the ACD input parameter appears " +
					"to be a simple (expected a declaration like \""+acdInputName+":"+
					((MobyPrimaryData) paramUnused.get(acdInputName)).getDataType().getName()+
					"\")");
		}
		paramUnused.remove(acdInputName);
	    }
	    // Secondary params read from the ACD spec always get included
	    else if(acdInput instanceof MobySecondaryData){
		service.addInput(acdInput);
		paramUnused.remove(acdInput.getName());
	    }
	    else{		
		throw new Exception("While parsing ACD input section: input field was not interpreted as " +
				    "primary or secondary data as expected (" + acdInput.getName() + " is of class " + 
				    acdInput.getClass().getName() + ")");
	    }
	}

	if(!paramUnused.isEmpty()){
	    String unmappedParams = "";
	    for(String param: paramUnused.keySet()){
		unmappedParams += " "+param;
	    }
	    throw new Exception("There are superfluous MOBY input parameters specified that do " +
				"not map to ACD input parameters:" + unmappedParams);	    
	}
    }
    
    protected void configureServiceFromACDParams(MobyService service,
						 List<Map<String,String>> spec) throws Exception{
	List<MobySecondaryData> paramTypes = new ArrayList<MobySecondaryData>();
	for(MobyData param: specToMoby(spec)){
	    if(!(param instanceof MobySecondaryData)){		
		throw new Exception("While parsing ACD input section: parameter field was not interpreted as " +
				    "secondary data as expected (" + param.getName() + " is of class " + 
				    param.getClass().getName() + ")");
	    }
	    paramTypes.add((MobySecondaryData) param);
	}

	service.setInputs(paramTypes.toArray(new MobySecondaryData[paramTypes.size()]));
    }

    /**
     * We must ensure that every MOBY output object has a corresponding ACD output
     * parameter that matches its name, and that we aren't stuffing an ACD output collection
     * into a MOBY simple (the other way is okay, a MOBY collection of 1).
     */
    protected void configureServiceFromACDOutput(MobyService service, 
						 List<Map<String,String>> spec) throws Exception{
	Map<String,MobyPrimaryData> paramUsed = new HashMap<String,MobyPrimaryData>();

	for(MobyPrimaryData mobyPrimaryOutput: service.getPrimaryOutputs()){
	    if(!canProduceDataTypeFromString(mobyPrimaryOutput.getDataType())){
		throw new Exception("No data mapping rules exist that can produce the requested " +
				    "data type (" + mobyPrimaryOutput.getDataType().getName() + 
				    ") from plain text");
	    }
	    paramUsed.put(mobyPrimaryOutput.getName(), mobyPrimaryOutput);
	}

	for(MobyData acdOutput: specToMoby(spec)){
	    String acdOutputName = acdOutput.getName(); 
	    if(acdOutput instanceof MobyPrimaryData){
		// See if it was defined as being returned in the MOBY service
		if(paramUsed.containsKey(acdOutputName)){
		    // And it is correctly a simple or collection
		    if(acdOutput instanceof MobyPrimaryDataSet &&
		       paramUsed.get(acdOutputName) instanceof MobyPrimaryDataSimple){
			throw new Exception("The MOBY output parameter \""+acdOutputName+
					    "\" is a Simple, but the ACD output parameter is " +
					    " a set (expected a declaration like \""+acdOutputName+":"+
					    "Collection("+
					    paramUsed.get(acdOutputName).getDataType().getName()+
					    ")\")");
		    }
		    paramUsed.remove(acdOutputName);  // Keep track of the ones mapped
		}
		else{
		    log("Output parameter from ACD (" + acdOutputName + 
			") will not be used in MOBY output (no param with " +
			"same name in servlet config)");
		}
	    }
	    // In the output section of ACD files they sometimes define optional
	    // *input parameters* controlling the output format
	    else if(acdOutput instanceof MobySecondaryData){
		service.addInput(acdOutput);
	    }
	    else{		
		throw new Exception("While parsing ACD input section: input field was not interpreted as " +
				    "primary or secondary data as expected (" + acdOutputName + " is of class " + 
				    acdOutput.getClass().getName() + ")");
	    }
	}

	// See if there are any unmapped MOBY outputs
	if(!paramUsed.isEmpty()){
	    String unmappedParams = "";
	    for(String param: paramUsed.keySet()){
		unmappedParams += " "+param;
	    }
	    throw new Exception("There are MOBY output parameters specified that do " +
				"not map to ACD output parameters:" + unmappedParams);
	}
    }

    protected List<MobyData> specToMoby(List<Map<String,String>> blocks) throws Exception{
	List<MobyData> mobyDataTypes = new ArrayList<MobyData>();

	for(Map<String,String> block: blocks){
	    String acdType = block.get(ACDFile.BLOCK_TYPE_KEY);
	    String acdName = block.get(ACDFile.BLOCK_NAME_KEY);

	    // ACD datatypes to MOBY datatypes mapping
	    MobyData mobyDataType = null;
	    
	    // Secondary Parameter types
	    String additional = block.get("additional");
	    String deFault = block.get("default");
	    //System.err.println("param: " + acdName + "/" + acdType +" " + additional +" ("+deFault+")");
	    if((additional != null && !additional.toUpperCase().equals("N")) ||
	       deFault != null){
		if("boolean".equals(acdType) || "toggle".equals(acdType)){
		    MobySecondaryData bool = new MobySecondaryData(acdName);
		    bool.setDataType(MobySecondaryData.BOOLEAN_TYPE);
		    bool.setDefaultValue(""+Boolean.parseBoolean(deFault));
		    bool.setDescription(block.get("help"));
		    mobyDataType = bool;
		}
		else if("string".equals(acdType) || "range".equals(acdType)){
		    MobySecondaryData string = new MobySecondaryData(acdName);
		    string.setDataType(MobySecondaryData.STRING_TYPE);
		    string.setDefaultValue(deFault);
		    string.setDescription(block.get("help"));
		    mobyDataType = string;
		}
		else if("list".equals(acdType)){
		    MobySecondaryData enumeration = new MobySecondaryData(acdName);
		    enumeration.setDataType(MobySecondaryData.STRING_TYPE);
		    enumeration.setDefaultValue(deFault);
		    enumeration.setDescription(block.get("information"));
		    processACDList(enumeration, block.get("values"), 
				   block.get("delimiter"), block.get("codedelimiter"));
		    mobyDataType = enumeration;
		}
		else if("integer".equals(acdType)){
		    MobySecondaryData integer = new MobySecondaryData(acdName);
		    integer.setDataType(MobySecondaryData.INTEGER_TYPE);
		    integer.setDescription(block.get("information"));
		    String m = block.get("maximum");
		    try{m = (new BigInteger(m)).toString();}catch(Exception e){m = ""+Integer.MAX_VALUE;}
		    integer.setMaxValue(m);
		    m = block.get("minimum");
		    try{m = (new BigInteger(m)).toString();}catch(Exception e){m = ""+Integer.MIN_VALUE;}
		    integer.setMinValue(m);
		    // Set default to 0, or minimum value if it's not Integer.MIN_VALUE
		    try{
			deFault = (new BigInteger(deFault)).toString();
		    }
		    catch(NumberFormatException nfe){
			deFault = (Integer.parseInt(m) == Integer.MIN_VALUE ? 
				   (Integer.parseInt(integer.getMaxValue()) == Integer.MAX_VALUE ? 
				    "0" : integer.getMaxValue()): m);
		    }
		    integer.setDefaultValue(deFault);
		    mobyDataType = integer;
		}
		else if("float".equals(acdType)){
		    MobySecondaryData floating = new MobySecondaryData(acdName);
		    floating.setDataType(MobySecondaryData.FLOAT_TYPE);
		    floating.setDefaultValue(deFault);
		    floating.setDescription(block.get("information"));
		    String m = block.get("maximum");
		    try{m = (new BigDecimal(m)).toString();}catch(Exception e){m = ""+Double.MAX_VALUE;}
		    floating.setMaxValue(m);
		    m = block.get("minimum");
		    try{m = (new BigDecimal(m)).toString();}catch(Exception e){m = ""+Double.MIN_VALUE;}
		    floating.setMinValue(m);
		    // Set default to 0, or minimum value if it's not Double.MIN_VALUE
		    try{
			deFault = (new BigDecimal(deFault)).toString();
		    }
		    catch(NumberFormatException nfe){
			deFault = (Double.parseDouble(m) == Double.MIN_VALUE ? 
				   (Double.parseDouble(floating.getMaxValue()) == Double.MAX_VALUE ? 
				    "0.0" : floating.getMaxValue()): m);
		    }
		    mobyDataType = floating;
		}
		else{
		    // If non-secondary with a default value, this is weird
		    if(deFault.length() != 0){
			System.err.println("Found secondary parameter that is not a primitive in MOBY:" + acdType);
		    }
		    //continue;  fall through to mobyDataType == null (Primary parameter)
		}
	    }  //end if(additional)
	    // else it's a required parameter
	    if(mobyDataType == null){
		// If it's an input or output parameter, just fill in a basic object to keep
		// track if the name of the ACD parameter and whether it's a collection or not.
		// The calling methods must make sure these match the MOBY service description provided
		String canBeNull = block.get("nullok");
		if(canBeNull != null && canBeNull.equals("Y")){
		    // do nothing, hence not add as parameter.  TODO: evaluate "standard" condition instead
		}
		else if(acdType.endsWith("set")){
		    mobyDataType = new MobyPrimaryDataSet(acdName);
		}
		else{
		    mobyDataType = new MobyPrimaryDataSimple(acdName);
		}

		// There are instances where the ACD data type is generic, such as 
		// "infile", "outfile", "string", and "datafile"
		// We will not handle the case of "list" data type, because this was enumerated anyway.
		if((acdType.equals("infile") ||
		    acdType.equals("outfile") ||
		    acdType.equals("string") ||
		    acdType.equals("datafile")) &&
		    block.containsKey("knowntype")){
		    // In such cases, keep track of the basic type (e.g. string, so when we build
		    // the command line we know how to represent the data)
		    acdBasicTypes.put(acdName, acdType);
		    acdType = block.get("knowntype");
		}
		acdTypes.put(acdName, acdType);
	    }

	    if(mobyDataType != null){
		mobyDataTypes.add(mobyDataType);
	    }
	}

	return mobyDataTypes;
    }

    private void processACDList(MobySecondaryData param, String values, 
				String itemDelim, String tagValueDelim){
	// Sometimes, delimiters are not given in the ACD file, use the defaults instead
	if(tagValueDelim == null){
	    tagValueDelim = ":";
	}
	if(itemDelim == null){
	    itemDelim = ",";
	}
	for(String item: values.split(itemDelim)){
	    String[] tagValuePair = item.split(tagValueDelim);
	    param.addAllowedValue(tagValuePair[0].trim());
	}
    }


}