001/* 002 * #%L 003 * Netarchivesuite - archive 004 * %% 005 * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library, 006 * the National Library of France and the Austrian National Library. 007 * %% 008 * This program is free software: you can redistribute it and/or modify 009 * it under the terms of the GNU Lesser General Public License as 010 * published by the Free Software Foundation, either version 2.1 of the 011 * License, or (at your option) any later version. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Lesser Public License for more details. 017 * 018 * You should have received a copy of the GNU General Lesser Public 019 * License along with this program. If not, see 020 * <http://www.gnu.org/licenses/lgpl-2.1.html>. 021 * #L% 022 */ 023package dk.netarkivet.archive.webinterface; 024 025import java.io.File; 026import java.io.FileWriter; 027import java.io.IOException; 028import java.util.Collection; 029import java.util.Collections; 030import java.util.Date; 031import java.util.HashMap; 032import java.util.Map; 033import java.util.Set; 034 035import org.slf4j.Logger; 036import org.slf4j.LoggerFactory; 037 038import dk.netarkivet.common.distribute.arcrepository.ArcRepositoryClientFactory; 039import dk.netarkivet.common.distribute.arcrepository.BatchStatus; 040import dk.netarkivet.common.distribute.arcrepository.Replica; 041import dk.netarkivet.common.distribute.arcrepository.ViewerArcRepositoryClient; 042import dk.netarkivet.common.exceptions.ArgumentNotValid; 043import dk.netarkivet.common.exceptions.IOFailure; 044import dk.netarkivet.common.utils.StreamUtils; 045import dk.netarkivet.common.utils.batch.FileBatchJob; 046import dk.netarkivet.common.utils.batch.FileBatchJob.ExceptionOccurrence; 047import dk.netarkivet.common.utils.batch.LoadableJarBatchJob; 048 049/** 050 * Class for execution of a batchjob in a separate thread. 051 */ 052public class BatchExecuter extends Thread { 053 /** Whether the results should be appended to the file. */ 054 private static final boolean APPEND = true; 055 056 /** The log. */ 057 //private Log log = LogFactory.getLog(BatchExecuter.class); 058 protected final Logger log = LoggerFactory.getLogger(BatchExecuter.class); 059 060 /** The batchjob to execute. */ 061 private FileBatchJob batchJob; 062 063 /** The regular expression for the execution. */ 064 private String regex; 065 066 /** The replica where the batchjob should be sent. */ 067 private Replica rep; 068 069 /** 070 * Map for containing the ids for the running batchjobs. Map between the name of the batchjob and the ID of the 071 * batch message. 072 */ 073 private static Map<String, String> batchjobs = Collections.synchronizedMap(new HashMap<String, String>()); 074 075 /** 076 * The constructor. 077 * 078 * @param job The batchjob to execute. 079 * @param pattern The regular expression pattern. 080 * @param replica The replica where the batchjob should be executed. 081 * @throws ArgumentNotValid If any of the arguments are null. 082 */ 083 public BatchExecuter(FileBatchJob job, String pattern, Replica replica) throws ArgumentNotValid { 084 ArgumentNotValid.checkNotNull(job, "FileBatchJob job"); 085 ArgumentNotValid.checkNotNull(pattern, "String pattern"); 086 ArgumentNotValid.checkNotNull(replica, "Replica replica"); 087 088 batchJob = job; 089 rep = replica; 090 regex = pattern; 091 } 092 093 /** 094 * Execution of the batchjob in its own thread (use start() instead). 095 * 096 * @throws IOFailure If an IOException is caught while writing the results. 097 */ 098 public void run() throws IOFailure { 099 ViewerArcRepositoryClient arcrep = ArcRepositoryClientFactory.getViewerInstance(); 100 // get the timestamp in milliseconds 101 String timestamp = Long.valueOf(new Date().getTime()).toString(); 102 // get the batchjob name without the classpath. 103 String jobName = BatchGUI.getJobName(batchJob.getClass().getName()); 104 105 // handle if loaded batchjob. 106 if (batchJob instanceof LoadableJarBatchJob) { 107 LoadableJarBatchJob ljbj = (LoadableJarBatchJob) batchJob; 108 jobName = BatchGUI.getJobName(ljbj.getLoadedJobClass()); 109 log.debug("LoadableJarBatchJob is actually the batchjob '" + jobName + "'."); 110 } 111 112 try { 113 // create output and error files. 114 File outputFile = new File(BatchGUI.getBatchDir(), jobName + Constants.NAME_TIMSTAMP_SEPARATOR + timestamp 115 + Constants.OUTPUT_FILE_EXTENSION); 116 outputFile.createNewFile(); 117 File eventLogFile = new File(BatchGUI.getBatchDir(), jobName + Constants.NAME_TIMSTAMP_SEPARATOR 118 + timestamp + Constants.ERROR_FILE_EXTENSION); 119 eventLogFile.createNewFile(); 120 121 // set pattern 122 batchJob.processOnlyFilesMatching(regex); 123 124 // write the output to the log file. 125 FileWriter fw = new FileWriter(eventLogFile, APPEND); 126 127 // execute the batchjob. 128 String processInfo = "Starting batchjob '" + jobName + "' at time '" + timestamp + "' on replica '" 129 + rep.getId() + "' with pattern '" + regex + "'."; 130 log.info(processInfo); 131 fw.write(processInfo + "\n"); 132 133 BatchStatus status = arcrep.batch(batchJob, rep.getId()); 134 final Collection<File> failedFiles = status.getFilesFailed(); 135 Collection<ExceptionOccurrence> exceptions = status.getExceptions(); 136 137 // log results. 138 processInfo = "Successfully finished BatchJob '" + jobName + "' on " + status.getNoOfFilesProcessed() 139 + " files."; 140 log.info(processInfo); 141 fw.write(processInfo + "\n"); 142 // log status. 143 processInfo = "BatchJob '" + jobName + "' has failed on '" + failedFiles.size() 144 + "' files and has gotten '" + exceptions.size() + "' exceptions."; 145 log.info(processInfo); 146 fw.write(processInfo + "\n"); 147 148 // copy results to outputfile, or log if problems with outputfile. 149 if (outputFile != null && outputFile.exists()) { 150 status.copyResults(outputFile); 151 } else { 152 log.warn("Could not print output to file. Logging it instead: " + "'\n:" 153 + StreamUtils.getInputStreamAsString(status.getResultFile().getInputStream())); 154 } 155 156 // print failed files to errorfile 157 if (!failedFiles.isEmpty()) { 158 fw.write("File failed: " + failedFiles.size() + "\n"); 159 for (File f : failedFiles) { 160 fw.write(f.getPath() + "\n"); 161 } 162 } 163 164 // print exceptions 165 log.info("BatchJob '" + jobName + "' encountered " + exceptions.size() + " exceptions."); 166 fw.write("Number of exceptions: " + exceptions.size() + "\n"); 167 if (!exceptions.isEmpty()) { 168 // print filename and exception, with empty line between. 169 for (ExceptionOccurrence e : exceptions) { 170 fw.write(e.getFileName() + "\n"); 171 fw.write(e.getException() + "\n \n"); 172 } 173 } 174 175 fw.flush(); 176 fw.close(); 177 } catch (IOException e) { 178 String errMsg = "Could not handle batchjob '" + jobName + "' with timestamp '" + timestamp + "'."; 179 log.error(errMsg, e); 180 throw new IOFailure(errMsg, e); 181 } catch (Throwable e) { 182 log.error("Fatal error", e); 183 throw new IOFailure("Fatal error", e); 184 } 185 } 186 187 /** 188 * Method for retrieving the data for the running batchjobs. 189 * 190 * @return The set of entries in the map. 191 */ 192 public static Set<Map.Entry<String, String>> getRunningBatchjobs() { 193 return batchjobs.entrySet(); 194 } 195}