001/*
002 * #%L
003 * Netarchivesuite - common
004 * %%
005 * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library,
006 *             the National Library of France and the Austrian National Library.
007 * %%
008 * This program is free software: you can redistribute it and/or modify
009 * it under the terms of the GNU Lesser General Public License as
010 * published by the Free Software Foundation, either version 2.1 of the
011 * License, or (at your option) any later version.
012 * 
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Lesser Public License for more details.
017 * 
018 * You should have received a copy of the GNU General Lesser Public
019 * License along with this program.  If not, see
020 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
021 * #L%
022 */
023
024package dk.netarkivet.common.utils;
025
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.FileOutputStream;
029import java.io.IOException;
030import java.io.InputStream;
031import java.util.Enumeration;
032import java.util.zip.GZIPInputStream;
033import java.util.zip.GZIPOutputStream;
034import java.util.zip.ZipEntry;
035import java.util.zip.ZipFile;
036import java.util.zip.ZipOutputStream;
037
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041import dk.netarkivet.common.exceptions.ArgumentNotValid;
042import dk.netarkivet.common.exceptions.IOFailure;
043
044/**
045 * Utilities for interfacing with the (fairly low-level) java.util.zip package.
046 */
047public final class ZipUtils {
048
049    /** The class logger. */
050    private static final Logger log = LoggerFactory.getLogger(ZipUtils.class);
051
052    /** The standard suffix for a gzipped file. */
053    public static final String GZIP_SUFFIX = ".gz";
054
055    /** Default constructor to avoid initialization. */
056    private ZipUtils() {
057    }
058
059    /**
060     * Zip the contents of a directory into a file. Does *not* zip recursively.
061     *
062     * @param dir The directory to zip.
063     * @param into The (zip) file to create. The name should typically end in .zip, but that is not required.
064     */
065    public static void zipDirectory(File dir, File into) {
066        ArgumentNotValid.checkNotNull(dir, "File dir");
067        ArgumentNotValid.checkNotNull(into, "File into");
068        ArgumentNotValid.checkTrue(dir.isDirectory(), "directory '" + dir + "' to zip is not a directory");
069        ArgumentNotValid.checkTrue(into.getAbsoluteFile().getParentFile().canWrite(), "cannot write to '" + into + "'");
070
071        File[] files = dir.listFiles();
072        FileOutputStream out;
073        try {
074            out = new FileOutputStream(into);
075        } catch (IOException e) {
076            throw new IOFailure("Error creating ZIP outfile file '" + into + "'", e);
077        }
078        ZipOutputStream zipout = new ZipOutputStream(out);
079        try {
080            try {
081                for (File f : files) {
082                    if (f.isFile()) {
083                        ZipEntry entry = new ZipEntry(f.getName());
084                        zipout.putNextEntry(entry);
085                        FileUtils.writeFileToStream(f, zipout);
086                    } // Not doing directories yet.
087                }
088            } finally {
089                zipout.close();
090            }
091        } catch (IOException e) {
092            throw new IOFailure("Failed to zip directory '" + dir + "'", e);
093        }
094    }
095
096    /**
097     * Unzip a zipFile into a directory. This will create subdirectories as needed.
098     *
099     * @param zipFile The file to unzip
100     * @param toDir The directory to create the files under. This directory will be created if necessary. Files in it
101     * will be overwritten if the filenames match.
102     */
103    public static void unzip(File zipFile, File toDir) {
104        ArgumentNotValid.checkNotNull(zipFile, "File zipFile");
105        ArgumentNotValid.checkNotNull(toDir, "File toDir");
106        ArgumentNotValid
107                .checkTrue(toDir.getAbsoluteFile().getParentFile().canWrite(), "can't write to '" + toDir + "'");
108        ArgumentNotValid.checkTrue(zipFile.canRead(), "can't read '" + zipFile + "'");
109        InputStream inputStream = null;
110        ZipFile unzipper = null;
111        try {
112            try {
113                unzipper = new ZipFile(zipFile);
114                Enumeration<? extends ZipEntry> entries = unzipper.entries();
115                while (entries.hasMoreElements()) {
116                    ZipEntry ze = entries.nextElement();
117                    File target = new File(toDir, ze.getName());
118                    // Ensure that its dir exists
119                    FileUtils.createDir(target.getCanonicalFile().getParentFile());
120                    if (ze.isDirectory()) {
121                        target.mkdir();
122                    } else {
123                        inputStream = unzipper.getInputStream(ze);
124                        FileUtils.writeStreamToFile(inputStream, target);
125                        inputStream.close();
126                    }
127                }
128            } finally {
129                if (unzipper != null) {
130                    unzipper.close();
131                }
132                if (inputStream != null) {
133                    inputStream.close();
134                }
135            }
136        } catch (IOException e) {
137            throw new IOFailure("Failed to unzip '" + zipFile + "'", e);
138        }
139    }
140
141    /**
142     * GZip each of the files in fromDir, placing the result in toDir (which will be created) with names having .gz
143     * appended. All non-file (directory, link, etc) entries in the source directory will be skipped with a quiet little
144     * log message.
145     *
146     * @param fromDir An existing directory
147     * @param toDir A directory where gzipped files will be placed. This directory must not previously exist. If the
148     * operation is not successful, the directory will not be created.
149     */
150    public static void gzipFiles(File fromDir, File toDir) {
151        ArgumentNotValid.checkNotNull(fromDir, "File fromDir");
152        ArgumentNotValid.checkNotNull(toDir, "File toDir");
153        ArgumentNotValid.checkTrue(fromDir.isDirectory(), "source '" + fromDir + "' must be an existing directory");
154        ArgumentNotValid.checkTrue(!toDir.exists(), "destination directory '" + toDir + "' must not exist");
155
156        File tmpDir = null;
157        try {
158            tmpDir = FileUtils.createUniqueTempDir(toDir.getAbsoluteFile().getParentFile(), toDir.getName());
159            File[] fromFiles = fromDir.listFiles();
160            for (File f : fromFiles) {
161                if (f.isFile()) {
162                    gzipFileInto(f, tmpDir);
163                } else {
164                    log.trace("Skipping non-file '{}'", f);
165                }
166            }
167            if (!tmpDir.renameTo(toDir)) {
168                throw new IOFailure("Failed to rename temp dir '" + tmpDir + "' to desired target '" + toDir + "'");
169            }
170        } finally {
171            if (tmpDir != null) {
172                try {
173                    FileUtils.removeRecursively(tmpDir);
174                } catch (IOFailure e) {
175                    log.debug("Error removing temporary directory '{}' after gzipping of '{}'", tmpDir, toDir, e);
176                }
177            }
178        }
179    }
180
181    /**
182     * GZip a file into a given dir. The resulting file will have .gz appended.
183     *
184     * @param f A file to gzip. This must be a real file, not a directory or the like.
185     * @param toDir The directory that the gzipped file will be placed in.
186     */
187    private static void gzipFileInto(File f, File toDir) {
188        try {
189            GZIPOutputStream out = null;
190            try {
191                File outF = new File(toDir, f.getName() + GZIP_SUFFIX);
192                out = new GZIPOutputStream(new FileOutputStream(outF));
193                FileUtils.writeFileToStream(f, out);
194            } finally {
195                if (out != null) {
196                    try {
197                        out.close();
198                    } catch (IOException e) {
199                        // Not really a problem to not be able to close,
200                        // so don't abort
201                        log.debug("Error closing output file for '{}'", f, e);
202                    }
203                }
204            }
205        } catch (IOException e) {
206            throw new IOFailure("Error while gzipping file '" + f + "'", e);
207        }
208    }
209
210    /**
211     * Gunzip all .gz files in a given directory into another. Files in fromDir not ending in .gz or not real files will
212     * be skipped with a log entry.
213     *
214     * @param fromDir The directory containing .gz files
215     * @param toDir The directory to place the unzipped files in. This directory must not exist beforehand.
216     * @throws IOFailure if there are problems creating the output directory or gunzipping the files.
217     */
218    public static void gunzipFiles(File fromDir, File toDir) {
219        ArgumentNotValid.checkNotNull(fromDir, "File fromDir");
220        ArgumentNotValid.checkNotNull(toDir, "File toDir");
221        ArgumentNotValid.checkTrue(fromDir.isDirectory(), "source directory '" + fromDir + "' must exist");
222        ArgumentNotValid.checkTrue(!toDir.exists(), "destination directory '" + toDir + "' must not exist");
223        File tempDir = FileUtils.createUniqueTempDir(toDir.getAbsoluteFile().getParentFile(), toDir.getName());
224        try {
225            File[] gzippedFiles = fromDir.listFiles();
226            for (File f : gzippedFiles) {
227                if (f.isFile() && f.getName().endsWith(GZIP_SUFFIX)) {
228                    gunzipInto(f, tempDir);
229                } else {
230                    log.trace("Non-gzip file '{}' found in gzip dir", f);
231                }
232            }
233            if (!tempDir.renameTo(toDir)) {
234                throw new IOFailure("Error renaming temporary directory '" + tempDir + "' to target directory '"
235                        + toDir);
236            }
237        } finally {
238            FileUtils.removeRecursively(tempDir);
239        }
240    }
241
242    /**
243     * Gunzip a single file into a directory. Unlike with the gzip() command-line tool, the original file is not
244     * deleted.
245     *
246     * @param f The .gz file to unzip.
247     * @param toDir The directory to gunzip into. This directory must exist.
248     * @throws IOFailure if there are any problems gunzipping.
249     */
250    private static void gunzipInto(File f, File toDir) {
251        String fileName = f.getName();
252        File outFile = new File(toDir, fileName.substring(0, fileName.length() - GZIP_SUFFIX.length()));
253        gunzipFile(f, outFile);
254    }
255
256    /**
257     * Gunzip a single gzipped file into the given file. Unlike with the gzip() command-line tool, the original file is
258     * not deleted.
259     *
260     * @param fromFile A gzipped file to unzip.
261     * @param toFile The file that the contents of fromFile should be gunzipped into. This file must be in an existing
262     * directory. Existing contents of this file will be overwritten.
263     */
264    public static void gunzipFile(File fromFile, File toFile) {
265        ArgumentNotValid.checkNotNull(fromFile, "File fromFile");
266        ArgumentNotValid.checkTrue(fromFile.canRead(), "fromFile must be readable");
267        ArgumentNotValid.checkNotNull(toFile, "File toFile");
268        ArgumentNotValid.checkTrue(toFile.getAbsoluteFile().getParentFile().canWrite(),
269                "toFile must be in a writeable dir");
270        try {
271            GZIPInputStream in = new LargeFileGZIPInputStream(new FileInputStream(fromFile));
272            FileUtils.writeStreamToFile(in, toFile);
273        } catch (IOException e) {
274            throw new IOFailure("Error ungzipping '" + fromFile + "'", e);
275        }
276    }
277
278}