Source code

001/*
002 * #%L
003 * Netarchivesuite - common - test
004 * %%
005 * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library,
006 *             the National Library of France and the Austrian National Library.
007 * %%
008 * This program is free software: you can redistribute it and/or modify
009 * it under the terms of the GNU Lesser General Public License as
010 * published by the Free Software Foundation, either version 2.1 of the
011 * License, or (at your option) any later version.
012 * 
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Lesser Public License for more details.
017 * 
018 * You should have received a copy of the GNU General Lesser Public
019 * License along with this program.  If not, see
020 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
021 * #L%
022 */
023
024package dk.netarkivet.common.arcrepository;
025
026import java.io.File;
027import java.io.FileOutputStream;
028import java.io.FilenameFilter;
029import java.io.IOException;
030import java.io.OutputStream;
031import java.util.regex.Pattern;
032
033import org.archive.io.ArchiveReader;
034import org.archive.io.ArchiveReaderFactory;
035import org.archive.io.ArchiveRecord;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039import dk.netarkivet.common.distribute.RemoteFileFactory;
040import dk.netarkivet.common.distribute.arcrepository.ArcRepositoryClient;
041import dk.netarkivet.common.distribute.arcrepository.BatchStatus;
042import dk.netarkivet.common.distribute.arcrepository.BitarchiveRecord;
043import dk.netarkivet.common.distribute.arcrepository.Replica;
044import dk.netarkivet.common.distribute.arcrepository.ReplicaStoreState;
045import dk.netarkivet.common.exceptions.ArgumentNotValid;
046import dk.netarkivet.common.exceptions.IOFailure;
047import dk.netarkivet.common.exceptions.NotImplementedException;
048import dk.netarkivet.common.utils.FileUtils;
049import dk.netarkivet.common.utils.batch.BatchLocalFiles;
050import dk.netarkivet.common.utils.batch.FileBatchJob;
051
052/**
053 * A minimal implementation of ArcRepositoryClient that just has one local directory that it keeps its files in, no
054 * checking no nothing.
055 */
056public class TrivialArcRepositoryClient implements ArcRepositoryClient {
057    /** The directory name of the local arcrepository. */
058    private static final String ARC_REPOSITORY_DIR_NAME = "ArcRepository";
059    /** Store files in this dir -- might later use a separate setting. */
060    private final File dir = new File(FileUtils.getTempDir(), ARC_REPOSITORY_DIR_NAME);
061    /** The class logger. */
062    //private Log log = LogFactory.getLog(getClass());
063    private static final Logger log = LoggerFactory.getLogger(TrivialArcRepositoryClient.class);
064
065    /**
066     * Constructor for this class. Creates a local directory for the arcrepository.
067     */
068    public TrivialArcRepositoryClient() {
069        FileUtils.createDir(dir);
070    }
071
072    /** Call on shutdown to release external resources. */
073    public void close() {
074    }
075
076    /**
077     * Store the given file in the ArcRepository. After storing, the file is deleted.
078     *
079     * @param file A file to be stored. Must exist.
080     * @throws IOFailure thrown if store is unsuccessful, or failed to clean up files after the store operation.
081     * @throws ArgumentNotValid if file parameter is null or file is not an existing file.
082     */
083    public void store(File file) throws IOFailure, ArgumentNotValid {
084        ArgumentNotValid.checkNotNull(file, "file");
085        FileUtils.copyFile(file, new File(dir, file.getName()));
086        FileUtils.remove(file);
087    }
088
089    /**
090     * Gets a single ARC record out of the ArcRepository.
091     *
092     * @param arcfile The name of a file containing the desired record.
093     * @param index The offset of the desired record in the file
094     * @return a BitarchiveRecord-object, or null if request times out or object is not found.
095     * @throws ArgumentNotValid if arcfile is null or empty, or index is negative
096     * @throws IOFailure If the get operation failed.
097     */
098    public BitarchiveRecord get(String arcfile, long index) throws ArgumentNotValid {
099        ArgumentNotValid.checkNotNullOrEmpty(arcfile, "arcfile");
100        ArgumentNotValid.checkNotNegative(index, "index");
101        ArchiveReader reader = null;
102        ArchiveRecord record = null;
103        try {
104            reader = ArchiveReaderFactory.get(new File(dir, arcfile), index);
105            record = reader.get();
106            return new BitarchiveRecord(record, arcfile);
107        } catch (IOException e) {
108            throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e);
109        } finally {
110            if (record != null) {
111                try {
112                    record.close();
113                } catch (IOException e) {
114                    log.info("Error closing ARC record '" + record + "'", e);
115                }
116            }
117            if (reader != null) {
118                try {
119                    reader.close();
120                } catch (IOException e) {
121                    log.info("Error closing ARC reader '" + reader + "'", e);
122                }
123            }
124        }
125    }
126
127    /**
128     * Retrieves a file from an ArcRepository and places it in a local file.
129     *
130     * @param arcfilename Name of the arcfile to retrieve.
131     * @param replica The bitarchive to retrieve the data from (not used in this implementation)
132     * @param toFile Filename of a place where the file fetched can be put.
133     * @throws IOFailure if there are problems getting a reply or the file could not be found.
134     */
135    public void getFile(String arcfilename, Replica replica, File toFile) {
136        ArgumentNotValid.checkNotNullOrEmpty(arcfilename, "arcfilename");
137        ArgumentNotValid.checkNotNull(toFile, "toFile");
138        FileUtils.copyFile(new File(dir, arcfilename), toFile);
139    }
140
141    /**
142     * Runs a batch batch job on each file in the ArcRepository.
143     *
144     * @param job An object that implements the FileBatchJob interface. The initialize() method will be called before
145     * processing and the finish() method will be called afterwards. The process() method will be called with each File
146     * entry. An optional function postProcess() allows handling the combined results of the batchjob, e.g. summing the
147     * results, sorting, etc.
148     * @param replicaId The archive to execute the job on (not used in this implementation)
149     * @param args The arguments for the batchjob.
150     * @return The status of the batch job after it ended.
151     */
152    public BatchStatus batch(final FileBatchJob job, String replicaId, String... args) {
153        ArgumentNotValid.checkNotNull(job, "job");
154        OutputStream os = null;
155        File resultFile;
156        try {
157            resultFile = File.createTempFile("batch", replicaId, FileUtils.getTempDir());
158            os = new FileOutputStream(resultFile);
159            File[] files = dir.listFiles(new FilenameFilter() {
160                public boolean accept(File dir, String name) {
161                    Pattern filenamePattern = job.getFilenamePattern();
162                    return new File(dir, name).isFile()
163                            && (filenamePattern == null || filenamePattern.matcher(name).matches());
164                }
165            });
166            BatchLocalFiles batcher = new BatchLocalFiles(files);
167            batcher.run(job, os);
168        } catch (IOException e) {
169            throw new IOFailure("Cannot perform batch job '" + job + "'", e);
170        } finally {
171            if (os != null) {
172                try {
173                    os.close();
174                } catch (IOException e) {
175                    log.info("Error closing batch output stream '" + os + "'", e);
176                }
177            }
178        }
179        return new BatchStatus(replicaId, job.getFilesFailed(), job.getNoOfFilesProcessed(),
180                RemoteFileFactory.getMovefileInstance(resultFile), job.getExceptions());
181    }
182
183    /**
184     * Updates the administrative data in the ArcRepository for a given file and replica. (not implemented)
185     *
186     * @param fileName The name of a file stored in the ArcRepository.
187     * @param bitarchiveId The id of the replica that the administrative data for fileName is wrong for.
188     * @param newval What the administrative data will be updated to.
189     */
190    public void updateAdminData(String fileName, String bitarchiveId, ReplicaStoreState newval) {
191        throw new NotImplementedException("Function has not been implemented");
192    }
193
194    /**
195     * Updates the checksum kept in the ArcRepository for a given file. It is the responsibility of the ArcRepository
196     * implementation to ensure that this checksum matches that of the underlying files.
197     *
198     * @param filename The name of a file stored in the ArcRepository.
199     * @param checksum The new checksum.
200     */
201    public void updateAdminChecksum(String filename, String checksum) {
202        throw new NotImplementedException("Function has not been implemented");
203    }
204
205    /**
206     * Remove a file from one part of the ArcRepository, retrieving a copy for security purposes. This is typically used
207     * when repairing a file that has been corrupted.
208     *
209     * @param fileName The name of the file to remove.
210     * @param bitarchiveId The id of the replica from which to remove the file (not used)
211     * @param checksum The checksum of the file to be removed (not used)
212     * @param credentials A string that shows that the user is allowed to perform this operation (not used)
213     * @return A local copy of the file removed.
214     */
215    public File removeAndGetFile(String fileName, String bitarchiveId, String checksum, String credentials) {
216        ArgumentNotValid.checkNotNullOrEmpty(fileName, "fileName");
217        // Ignores bitarchiveId, checksum, and credentials for now
218        File copiedTo = null;
219        try {
220            copiedTo = File.createTempFile("removeAndGetFile", fileName);
221        } catch (IOException e) {
222            throw new IOFailure("Cannot make temp file to copy '" + fileName + "' into", e);
223        }
224        File file = new File(dir, fileName);
225        FileUtils.copyFile(file, copiedTo);
226        FileUtils.remove(file);
227        return copiedTo;
228    }
229
230    public File getAllChecksums(String replicaId) {
231        // TODO Auto-generated method stub
232        throw new NotImplementedException("TODO: Implement me!");
233    }
234
235    public File getAllFilenames(String replicaId) {
236        // TODO Auto-generated method stub
237        throw new NotImplementedException("TODO: Implement me!");
238    }
239
240    public File correct(String replicaId, String checksum, File file, String credentials) {
241        // TODO Auto-generated method stub
242        throw new NotImplementedException("TODO: Implement me!");
243    }
244
245    @Override
246    public String getChecksum(String replicaId, String filename) {
247        // TODO Auto-generated method stub
248        return null;
249    }
250}