diff options
Diffstat (limited to 'src/model')
| -rw-r--r-- | src/model/Model.java | 325 | ||||
| -rw-r--r-- | src/model/ModelInterface.java | 32 |
2 files changed, 357 insertions, 0 deletions
diff --git a/src/model/Model.java b/src/model/Model.java new file mode 100644 index 0000000..ca9897b --- /dev/null +++ b/src/model/Model.java @@ -0,0 +1,325 @@ +package model; + +import java.io.*; +import java.util.*; +import java.text.DateFormat; +import java.text.SimpleDateFormat; + +import org.apache.log4j.*; + +import org.apache.commons.math3.stat.descriptive.moment.*; +import org.apache.commons.math3.distribution.NormalDistribution; + +import weka.core.*; +import weka.classifiers.evaluation.*; +import weka.core.converters.*; + +import input.ARFFReader; +import predictor.*; + + +public class Model implements ModelInterface { + protected Logger logger = Logger.getLogger(Model.class); + + protected String datapath; + protected Instances trainingInstances; + protected Instances preprocessedInstances; + protected ArrayList<PredictorInterface> predictors; + + protected static DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); + protected static Date date = new Date(); + protected static String resultFilename = dateFormat.format(date); + + public Model(){ + this.datapath = ""; + this.trainingInstances = null; + this.preprocessedInstances = null; + this.predictors = new ArrayList<PredictorInterface>(); + } + + public Model (ModelInterface model) { + this.datapath = model.getDatapath(); + this.trainingInstances = new Instances(model.getTrainingInstances()); + this.preprocessedInstances = new Instances(model.getPreprocessedInstances()); + this.predictors = new ArrayList<PredictorInterface>(model.getPredictors()); + } + + @Override + public Instances getPreprocessedInstances() { + return this.preprocessedInstances; + } + + @Override + public Instances getTrainingInstances() { + return this.trainingInstances; + } + + + @Override + public String getDatapath() { + return this.datapath; + } + + + @Override + public ArrayList<PredictorInterface> getPredictors() { + return this.predictors; + } + + + @Override + public void loadTrainingData(String path) { + logger.debug("Reading training data from " + path); + this.datapath = path; + try { + trainingInstances = ARFFReader.read(path); + preprocessedInstances = trainingInstances; + logger.debug("Training data is read"); + logger.trace(trainingInstances); + } catch (Exception e) { + logger.warn(e.toString()); + } + } + + @Override + public void loadRawLog(String path) { + logger.debug("Reading logfile from " + path); + try { + BufferedReader in = new BufferedReader(new FileReader(path)); + String line = in.readLine(); + System.out.println(line); + } catch (Exception e) { + logger.warn(e.toString()); + } + } + + + + @Override + public void setPreprocessedInstances(Instances instances){ + this.preprocessedInstances=new Instances(instances); + } + + @Override + public void savePreprocessedInstances(String path) { + ArffSaver saver = new ArffSaver(); + saver.setInstances(this.preprocessedInstances); + try { + saver.setFile(new File(path)); + saver.writeBatch(); + } catch (Exception e) { + logger.error("Cannot save preprocessed instances to file " + path); + } + } + + @Override + public void addPredictor(String shortName) { + for (PredictorFactory.PredictionTechnique pTechnique : PredictorFactory.PredictionTechnique.values()) { + if (pTechnique.getShortName().equals(shortName)) { + predictors.add(PredictorFactory.createPredictor(pTechnique)); + logger.info("Added predictor: " + pTechnique.getName()); + return; + } + } + logger.warn("Added predictor: None"); + logger.warn(shortName + " is not in the list."); + } + + @Override + public void selectTrainingMethod() { + + } + + @Override + public void trainPredictors() throws Exception{ + if (preprocessedInstances == null) { + throw new Exception("No training data"); + } + if (predictors.size() == 0) { + throw new Exception("No predictors selected"); + } + for (PredictorInterface p : predictors) { + try { + logger.info("Training " + p.getName()); + p.train(preprocessedInstances); + logger.debug(p.toString()); + } catch (Exception e) { + logger.error(e.toString()); + } + } + } + + @Override + public void crossValidatePredictors(int numFold) { + long seed = 1; + this.crossValidatePredictors(numFold, seed); + } + + @Override + public void crossValidatePredictors(int numFold, long seed) { + for (PredictorInterface p : predictors) { + try { + logger.debug(numFold + "-fold cross-validating: " + p.getName()); + Random rand = new Random(seed); + p.crossValidate(preprocessedInstances, numFold, rand); + + ///* + ThresholdCurve tc = new ThresholdCurve(); + int classIndex = 1; + Instances result = tc.getCurve(p.getEvaluationPredictions(), classIndex); + + + // Save ROC + + BufferedWriter br = new BufferedWriter(new FileWriter(resultFilename + "_" + p.getName().replace(" ", "_") + "_ROC.arff")); + br.write(result.toString()); + br.close(); + } catch (Exception e) { + logger.error(e.toString()); + } + } + } + + @Override + public void benchmark(int rounds, String filename) throws Exception { + ArrayList<ArrayList<Double>> trainingTime = new ArrayList<ArrayList<Double>>(this.predictors.size()); + ArrayList<ArrayList<Double>> predictionTime = new ArrayList<ArrayList<Double>>(this.predictors.size()); + + Runtime runtime = Runtime.getRuntime(); + + for (int i=0; i<this.predictors.size(); i++) { + trainingTime.add(new ArrayList<Double>(rounds)); + predictionTime.add(new ArrayList<Double>(rounds)); + } + + // Benchmark - using preprocessed instances + long startTime; + long endTime; + double elapsedTime; + for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) { + for (int rIndex=0; rIndex<rounds; rIndex++) { + logger.debug("Benchmarking " + this.predictors.get(pIndex).getName() + " round " + rIndex); + + // Training time + startTime = System.currentTimeMillis(); + this.predictors.get(pIndex).train(this.preprocessedInstances); + endTime = System.currentTimeMillis(); + elapsedTime = ((double)(endTime - startTime))/1000; + trainingTime.get(pIndex).add(elapsedTime); + logger.debug("Training time = " + elapsedTime + " seconds"); + + // Prediction time + startTime = System.currentTimeMillis(); + this.predictors.get(pIndex).predict(this.preprocessedInstances); + endTime = System.currentTimeMillis(); + elapsedTime = ((double)(endTime - startTime))/1000; + predictionTime.get(pIndex).add(elapsedTime); + logger.debug("Prediction time = " + elapsedTime + " seconds"); + } + } + + // Save time results to file + BufferedWriter br = new BufferedWriter(new FileWriter(filename+"_benchmark")); + // Write header + String header = ""; + for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) { + header += "\"" + this.predictors.get(pIndex).getName() + " Training\" "; + header += "\"" + this.predictors.get(pIndex).getName() + " Prediction\" "; + } + header += "\n"; + br.write(header); + + for (int rIndex=0; rIndex<rounds; rIndex++) { + String line = ""; + for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) { + line += trainingTime.get(pIndex).get(rIndex).toString() + " " + predictionTime.get(pIndex).get(rIndex).toString() + " "; + } + line += "\n"; + br.write(line); + } + br.close(); + + // TODO: move to another function + // refactor + // Calculate and save summary results + BufferedWriter brSummary = new BufferedWriter(new FileWriter(filename+"_benchmark_time_summary")); + //header = "Algorithms tMean tError pMean pError\n"; + //brSummary.write(header); + for (int pIndex=0; pIndex<this.predictors.size(); pIndex++) { + String line = "\"" + this.predictors.get(pIndex).getName() + "\" "; + // Calculate mean + double [] training = new double[rounds]; + // Convert ArrayList to array for Mean.evaluate + for (int rIndex=0; rIndex<rounds; rIndex++) { + //System.out.println("pIndex = " + pIndex + " rIndex = " + rIndex); + //System.out.println("array size = " + training.length); + training[rIndex] = trainingTime.get(pIndex).get(rIndex); + } + double meanTraining = new Mean().evaluate(training); + double varTraining = new Variance().evaluate(training); + double stdTraining = Math.sqrt(varTraining); + double errorTraining = new NormalDistribution().inverseCumulativeProbability(0.975)*(stdTraining/Math.sqrt(rounds)); + double lowerCITraining = meanTraining - errorTraining; + double upperCITraining = meanTraining + errorTraining; + line += meanTraining + " " + errorTraining + " "; + //brSummary.write(line); + + //line = "\"" + this.predictors.get(pIndex).getName() + " Prediction\" "; + // Calculate mean + double [] prediction = new double[rounds]; + // Convert ArrayList to array for Mean.evaluate + for (int rIndex=0; rIndex<rounds; rIndex++) { + prediction[rIndex] = predictionTime.get(pIndex).get(rIndex); + } + double meanPrediction = new Mean().evaluate(prediction); + double varPrediction = new Variance().evaluate(prediction); + double stdPrediction = Math.sqrt(varPrediction); + double errorPrediction = new NormalDistribution().inverseCumulativeProbability(0.975)*(stdPrediction/Math.sqrt(rounds)); + double lowerCIPrediction = meanPrediction - errorPrediction; + double upperCIPrediction = meanPrediction + errorPrediction; + line += meanPrediction + " " + errorPrediction + "\n"; + brSummary.write(line); + } + brSummary.close(); + } + + + @Override + public String getPredictorNames() { + String str = ""; + for (PredictorInterface p : this.predictors) { + str += p.getName() + "\n"; + } + return str; + } + + @Override + public String toString() { + String str = ""; + str += "Training data:\n" + this.datapath + "\n"; + str += "Training data summary:\n" + this.getTrainingInstances().toSummaryString() + "\n"; + str += "Preprocessed data summary:\n" + this.getPreprocessedInstances().toSummaryString() + "\n"; + str += "Predictors:\n" + this.getPredictorNames() + "\n"; + return str; + } + + @Override + public void saveSettings(String filename) throws Exception { + BufferedWriter br = new BufferedWriter(new FileWriter(filename,true)); + br.write(this.toString()); + br.close(); + } + + @Override + public void saveResults(String filename) throws Exception { + BufferedWriter br = new BufferedWriter(new FileWriter(filename,true)); + + for (PredictorInterface p : this.predictors) { + logger.info(p.getEvaluationResults()); + br.write(p.getEvaluationResults()); + } + + br.close(); + } +} + diff --git a/src/model/ModelInterface.java b/src/model/ModelInterface.java new file mode 100644 index 0000000..a4d1b7b --- /dev/null +++ b/src/model/ModelInterface.java @@ -0,0 +1,32 @@ +package model; + +import java.util.ArrayList; + +import weka.core.Instances; + +import predictor.*; + + +public interface ModelInterface { + public String getDatapath(); + public ArrayList<PredictorInterface> getPredictors(); + + public void loadTrainingData(String path); + public void loadRawLog(String path); + public Instances getTrainingInstances(); + public void setPreprocessedInstances(Instances instances); + public Instances getPreprocessedInstances(); + public void savePreprocessedInstances(String path); + public void addPredictor(String shortName); + public void crossValidatePredictors(int numFold); + public void crossValidatePredictors(int numFold, long seed); + public void selectTrainingMethod(); + public void trainPredictors() throws Exception; + public void benchmark(int rounds, String filename) throws Exception; + + public String getPredictorNames(); + public String toString(); + public void saveSettings(String filename) throws Exception; + public void saveResults(String filename) throws Exception; +} + |