path: root/src/model/Model.java

// Copyright 2016 Huawei Technologies Co. Ltd.
//
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model;

import java.io.*;
import java.util.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;

import org.apache.log4j.*;

import org.apache.commons.math3.stat.descriptive.moment.*;
import org.apache.commons.math3.distribution.NormalDistribution;

import weka.core.*;
import weka.classifiers.evaluation.*;
import weka.core.converters.*;

import input.ARFFReader;
import predictor.*;


public class Model implements ModelInterface {
	protected Logger logger = Logger.getLogger(Model.class);
	
	protected String datapath;
	protected Instances trainingInstances;
	protected Instances preprocessedInstances;
	protected ArrayList<PredictorInterface> predictors;

	protected static DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
	protected static Date date = new Date();
	protected static String resultFilename = dateFormat.format(date);

	public Model(){
		this.datapath = "";
		this.trainingInstances = null;
		this.preprocessedInstances = null;
		this.predictors = new ArrayList<PredictorInterface>();
	}

	public Model (ModelInterface model) {
		this.datapath = model.getDatapath();
		this.trainingInstances = new Instances(model.getTrainingInstances());
		this.preprocessedInstances = new Instances(model.getPreprocessedInstances());
		this.predictors = new ArrayList<PredictorInterface>(model.getPredictors());
	}

	@Override
	public Instances getPreprocessedInstances() {
		return this.preprocessedInstances;
	}

	@Override
	public Instances getTrainingInstances() {
		return this.trainingInstances;
	}


	@Override
	public String getDatapath() {
		return this.datapath;
	}


	@Override
	public ArrayList<PredictorInterface> getPredictors() {
		return this.predictors;
	}


	@Override
	public void loadTrainingData(String path) {
		logger.debug("Reading training data from " + path);
		this.datapath = path;
		try {
			trainingInstances = ARFFReader.read(path);
			preprocessedInstances = trainingInstances;
			logger.debug("Training data is read");
			logger.trace(trainingInstances);
		} catch (Exception e) {
			logger.warn(e.toString());
		}
	}

	@Override
	public void loadRawLog(String path) {
		logger.debug("Reading logfile from " + path);
		try {
			BufferedReader in = new BufferedReader(new FileReader(path));
			String line = in.readLine();
			System.out.println(line);
		} catch (Exception e) {
			logger.warn(e.toString());
		}
	}



	@Override
	public void setPreprocessedInstances(Instances instances){
		this.preprocessedInstances=new Instances(instances);
	}

	@Override
	public void savePreprocessedInstances(String path) {
		ArffSaver saver = new ArffSaver();
		saver.setInstances(this.preprocessedInstances);
		try {
			saver.setFile(new File(path));
			saver.writeBatch();
		} catch (Exception e) {
			logger.error("Cannot save preprocessed instances to file " + path);
		}
	}

	@Override
	public void addPredictor(String shortName) {
		for (PredictorFactory.PredictionTechnique pTechnique : PredictorFactory.PredictionTechnique.values()) {
			if (pTechnique.getShortName().equals(shortName)) {
				predictors.add(PredictorFactory.createPredictor(pTechnique));
				logger.info("Added predictor: " + pTechnique.getName());
				return;
			}
		}
		logger.warn("Added predictor: None");
		logger.warn(shortName + " is not in the list.");
	}

	@Override
	public void selectTrainingMethod() {

	}

	@Override
	public void trainPredictors() throws Exception{
		if (preprocessedInstances == null) {
			throw new Exception("No training data");
		}
		if (predictors.size() == 0) {
			throw new Exception("No predictors selected");
		}
		for (PredictorInterface p : predictors) {
			try {
				logger.info("Training " + p.getName());
				p.train(preprocessedInstances);
				logger.debug(p.toString());
			} catch (Exception e) {
				logger.error(e.toString());
			}
		}
	}

	@Override
	public void crossValidatePredictors(int numFold) {
		long seed = 1;
		this.crossValidatePredictors(numFold, seed);
	}

	@Override
	public void crossValidatePredictors(int numFold, long seed) {
		for (PredictorInterface p : predictors) {
			try {
				logger.debug(numFold + "-fold cross-validating: " + p.getName());
				Random rand = new Random(seed);
				p.crossValidate(preprocessedInstances, numFold, rand);

				///* 
				ThresholdCurve tc = new ThresholdCurve();
				int classIndex = 1;
				Instances result = tc.getCurve(p.getEvaluationPredictions(), classIndex);


				// Save ROC

				BufferedWriter br = new BufferedWriter(new FileWriter(resultFilename + "_" + p.getName().replace(" ", "_") + "_ROC.arff"));
				br.write(result.toString());
				br.close();
			} catch (Exception e) {
				logger.error(e.toString());
			}
		}
	}

	@Override
	public void benchmark(int rounds, String filename) throws Exception {
		ArrayList<ArrayList<Double>> trainingTime = new ArrayList<ArrayList<Double>>(this.predictors.size());
		ArrayList<ArrayList<Double>> predictionTime = new ArrayList<ArrayList<Double>>(this.predictors.size());

		Runtime runtime = Runtime.getRuntime();

		for (int i=0; i<this.predictors.size(); i++) {
			trainingTime.add(new ArrayList<Double>(rounds));
			predictionTime.add(new ArrayList<Double>(rounds));
		}

		// Benchmark - using preprocessed instances
		long startTime;
		long endTime;
		double elapsedTime;
		for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) {
			for (int rIndex=0; rIndex<rounds; rIndex++) {
				logger.debug("Benchmarking " + this.predictors.get(pIndex).getName() + " round " + rIndex);

				// Training time
				startTime = System.currentTimeMillis();
				this.predictors.get(pIndex).train(this.preprocessedInstances);
				endTime = System.currentTimeMillis();
				elapsedTime = ((double)(endTime - startTime))/1000;
				trainingTime.get(pIndex).add(elapsedTime);
				logger.debug("Training time = " + elapsedTime + " seconds");

				// Prediction time
				startTime = System.currentTimeMillis();
				this.predictors.get(pIndex).predict(this.preprocessedInstances);
				endTime = System.currentTimeMillis();
				elapsedTime = ((double)(endTime - startTime))/1000;
				predictionTime.get(pIndex).add(elapsedTime);
				logger.debug("Prediction time = " + elapsedTime + " seconds");
			}
		}

		// Save time results to file
		BufferedWriter br = new BufferedWriter(new FileWriter(filename+"_benchmark"));
		// Write header
		String header = "";
		for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) {
			header += "\"" + this.predictors.get(pIndex).getName() + " Training\" ";
			header += "\"" + this.predictors.get(pIndex).getName() + " Prediction\" ";
		}
		header += "\n";
		br.write(header);

		for (int rIndex=0; rIndex<rounds; rIndex++) {
			String line = "";
			for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) {
				line += trainingTime.get(pIndex).get(rIndex).toString() + " " + predictionTime.get(pIndex).get(rIndex).toString() + " ";
			}
			line += "\n";
			br.write(line);
		}
		br.close();

		// TODO: move to another function
		// refactor
		// Calculate and save summary results
		BufferedWriter brSummary = new BufferedWriter(new FileWriter(filename+"_benchmark_time_summary"));
		//header = "Algorithms tMean tError pMean pError\n";
		//brSummary.write(header);
		for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) {
			String line = "\"" + this.predictors.get(pIndex).getName() + "\" ";
			// Calculate mean
			double [] training = new double[rounds];
			// Convert ArrayList to array for Mean.evaluate
			for (int rIndex=0; rIndex<rounds; rIndex++) {
				//System.out.println("pIndex = " + pIndex + " rIndex = " + rIndex);
				//System.out.println("array size = " + training.length);
				training[rIndex] = trainingTime.get(pIndex).get(rIndex);
			}
			double meanTraining = new Mean().evaluate(training);
			double varTraining = new Variance().evaluate(training);
			double stdTraining = Math.sqrt(varTraining);
			double errorTraining = new NormalDistribution().inverseCumulativeProbability(0.975)*(stdTraining/Math.sqrt(rounds));
			double lowerCITraining = meanTraining - errorTraining;
			double upperCITraining = meanTraining + errorTraining;
			line += meanTraining + " " + errorTraining + " ";
			//brSummary.write(line);

			//line = "\"" + this.predictors.get(pIndex).getName() + " Prediction\" ";
			// Calculate mean
			double [] prediction = new double[rounds];
			// Convert ArrayList to array for Mean.evaluate
			for (int rIndex=0; rIndex<rounds; rIndex++) {
				prediction[rIndex] = predictionTime.get(pIndex).get(rIndex);
			}
			double meanPrediction = new Mean().evaluate(prediction);
			double varPrediction = new Variance().evaluate(prediction);
			double stdPrediction = Math.sqrt(varPrediction);
			double errorPrediction = new NormalDistribution().inverseCumulativeProbability(0.975)*(stdPrediction/Math.sqrt(rounds));
			double lowerCIPrediction = meanPrediction - errorPrediction;
			double upperCIPrediction = meanPrediction + errorPrediction;
			line += meanPrediction + " " + errorPrediction + "\n";
			brSummary.write(line);
		}
		brSummary.close();
	}


	@Override
	public String getPredictorNames() {
		String str = "";
		for (PredictorInterface p : this.predictors) {
			str += p.getName() + "\n";
		}
		return str;
	}

	@Override
	public String toString() {
		String str = "";
		str += "Training data:\n" + this.datapath + "\n";
		str += "Training data summary:\n" + this.getTrainingInstances().toSummaryString() + "\n";
		str += "Preprocessed data summary:\n" + this.getPreprocessedInstances().toSummaryString() + "\n";
		str += "Predictors:\n" + this.getPredictorNames() + "\n";
		return str;
	}

	@Override
	public void saveSettings(String filename) throws Exception {
		BufferedWriter br = new BufferedWriter(new FileWriter(filename,true));
		br.write(this.toString());
		br.close();
	}

	@Override
	public void saveResults(String filename) throws Exception {
		BufferedWriter br = new BufferedWriter(new FileWriter(filename,true));

		for (PredictorInterface p : this.predictors) {
			logger.info(p.getEvaluationResults());
			br.write(p.getEvaluationResults());
		}

		br.close();
	}
}