001/* 002 * #%L 003 * Netarchivesuite - common 004 * %% 005 * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library, 006 * the National Library of France and the Austrian National Library. 007 * %% 008 * This program is free software: you can redistribute it and/or modify 009 * it under the terms of the GNU Lesser General Public License as 010 * published by the Free Software Foundation, either version 2.1 of the 011 * License, or (at your option) any later version. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Lesser Public License for more details. 017 * 018 * You should have received a copy of the GNU General Lesser Public 019 * License along with this program. If not, see 020 * <http://www.gnu.org/licenses/lgpl-2.1.html>. 021 * #L% 022 */ 023 024package dk.netarkivet.common.distribute; 025 026import java.io.File; 027import java.io.FileOutputStream; 028import java.io.InputStream; 029import java.io.OutputStream; 030import java.util.Calendar; 031 032import org.slf4j.Logger; 033import org.slf4j.LoggerFactory; 034 035import dk.netarkivet.common.exceptions.ArgumentNotValid; 036import dk.netarkivet.common.exceptions.IOFailure; 037import dk.netarkivet.common.utils.FileUtils; 038import dk.netarkivet.common.utils.StreamUtils; 039import dk.netarkivet.common.utils.TimeUtils; 040 041/** 042 * Abstract superclass for easy implementation of remote file. 043 * <p> 044 * Sub classes should override this class, and do the following: - Implement getChecksum. - Implement getInputStream. - 045 * Implement cleanup. - Add getInstance(File, Boolean, Boolean, Boolean)-method to make the file work with the factory. 046 */ 047@SuppressWarnings({"serial"}) 048public abstract class AbstractRemoteFile implements RemoteFile { 049 050 /** A named logger for this class. */ 051 private static final transient Logger log = LoggerFactory.getLogger(AbstractRemoteFile.class); 052 053 /** The file this is remote file for. */ 054 protected final File file; 055 /** If true, communication is checksummed. */ 056 protected final boolean useChecksums; 057 /** If true, the file may be deleted after all transfers are done. */ 058 protected final boolean fileDeletable; 059 /** 060 * If true, the file may be downloaded multiple times. Otherwise, the remote file is invalidated after first 061 * transfer. 062 */ 063 protected final boolean multipleDownloads; 064 /** The size of the file. */ 065 protected final long filesize; 066 067 /** 068 * Initialise common fields in remote file. Overriding classes should also initialise checksum field. 069 * 070 * @param file The file to make remote file for. 071 * @param useChecksums If true, communications should be checksummed. 072 * @param fileDeletable If true, the file may be downloaded multiple times. Otherwise, the remote file is 073 * invalidated after first transfer. 074 * @param multipleDownloads If useChecksums is true, contains the file checksum. 075 */ 076 public AbstractRemoteFile(File file, boolean useChecksums, boolean fileDeletable, boolean multipleDownloads) { 077 ArgumentNotValid.checkNotNull(file, "File file"); 078 if (!file.isFile() || !file.canRead()) { 079 throw new ArgumentNotValid("File '" + file.getAbsolutePath() + "' is not a readable file"); 080 } 081 this.file = file; 082 this.fileDeletable = fileDeletable; 083 this.multipleDownloads = multipleDownloads; 084 this.useChecksums = useChecksums; 085 this.filesize = file.length(); 086 } 087 088 /** 089 * Copy this remote file to the given file. This method will make a fileoutputstream, and use appendTo to write the 090 * remote file to this stream. 091 * 092 * @param destFile The file to write the remote file to. 093 * @throws ArgumentNotValid on null destFile, or parent to destfile is not a writeable directory, or destfile exists 094 * and cannot be overwritten. 095 * @throws IOFailure on I/O trouble writing remote file to destination. 096 */ 097 public void copyTo(File destFile) { 098 ArgumentNotValid.checkNotNull(destFile, "File destFile"); 099 destFile = destFile.getAbsoluteFile(); 100 if ((!destFile.isFile() || !destFile.canWrite()) 101 && (!destFile.getParentFile().isDirectory() || !destFile.getParentFile().canWrite())) { 102 throw new ArgumentNotValid("Destfile '" + destFile + "' does not point to a writable file for " 103 + "remote file '" + file + "'"); 104 } 105 try { 106 FileOutputStream fos = null; 107 int retry = 0; 108 boolean success = false; 109 110 // retry if it fails, but always make at least one attempt. 111 do { 112 try { 113 try { 114 fos = new FileOutputStream(destFile); 115 appendTo(fos); 116 success = true; 117 } finally { 118 if (fos != null) { 119 fos.close(); 120 } 121 } 122 } catch (IOFailure e) { 123 if (retry == 0) { 124 log.warn("Could not retrieve the file '{}' on first attempt. Will retry up to '{}' times.", 125 getName(), getNumberOfRetries(), e); 126 } else { 127 log.warn("Could not retrieve the file '{}' on retry number '{}' of '{}' retries.", getName(), 128 retry, getNumberOfRetries(), e); 129 } 130 } 131 ++retry; 132 if (!success && retry < getNumberOfRetries()) { 133 log.debug("CopyTo attempt #{} of max {} failed. Will sleep a while before trying to copyTo again.", 134 retry, getNumberOfRetries()); 135 TimeUtils.exponentialBackoffSleep(retry, Calendar.MINUTE); 136 } 137 } while (!success && retry < getNumberOfRetries()); 138 139 // handle case when the retrieval is unsuccessful. 140 if (!success) { 141 throw new IOFailure("Unable to retrieve the file '" + getName() + "' in '" + getNumberOfRetries() 142 + "' attempts."); 143 } 144 } catch (Exception e) { 145 FileUtils.remove(destFile); 146 throw new IOFailure("IO trouble transferring file", e); 147 } 148 } 149 150 /** 151 * Append this remote file to the given output stream. This method will use getInputStream to get the remote stream, 152 * and then copy that stream to the given output stream. 153 * 154 * @param out The stream to write the remote file to. 155 * @throws ArgumentNotValid if outputstream is null. 156 * @throws IOFailure on I/O trouble writing remote file to stream. 157 */ 158 public void appendTo(OutputStream out) { 159 ArgumentNotValid.checkNotNull(out, "OutputStream out"); 160 StreamUtils.copyInputStreamToOutputStream(getInputStream(), out); 161 } 162 163 /** 164 * Get an input stream representing the remote file. The returned input stream should throw IOFailure on close, if 165 * checksums are requested, but do not match. The returned inputstream should call cleanup on close, if 166 * multipleDownloads is not true. 167 * 168 * @return An input stream for the remote file. 169 * @throws IOFailure on I/O trouble generating inputstream for remote file. 170 */ 171 public abstract InputStream getInputStream(); 172 173 /** 174 * Get the name of the remote file. 175 * 176 * @return The name of the remote file. 177 */ 178 public String getName() { 179 return file.getName(); 180 } 181 182 /** 183 * Get checksum for file, or null if checksums were not requested. 184 * 185 * @return checksum for file, or null if checksums were not requested. 186 */ 187 public abstract String getChecksum(); 188 189 /** 190 * Invalidate all file handles. If file is deletable, it should be deleted after this method is called. This method 191 * should never throw exceptions, but only log a warning on trouble. It should be idempotent, meaning it should be 192 * safe to call this method twice. 193 */ 194 public abstract void cleanup(); 195 196 /** 197 * Method for retrieving the number of retries for retrieving a file. 198 * 199 * @return The number of retries for retrieving a file. 200 */ 201 public abstract int getNumberOfRetries(); 202 203 /** 204 * Get the size of this remote file. 205 * 206 * @return The size of this remote file. 207 */ 208 public long getSize() { 209 return filesize; 210 } 211 212}