001/*
002 * #%L
003 * Netarchivesuite - common - test
004 * %%
005 * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library,
006 *             the National Library of France and the Austrian National Library.
007 * %%
008 * This program is free software: you can redistribute it and/or modify
009 * it under the terms of the GNU Lesser General Public License as
010 * published by the Free Software Foundation, either version 2.1 of the
011 * License, or (at your option) any later version.
012 * 
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Lesser Public License for more details.
017 * 
018 * You should have received a copy of the GNU General Lesser Public
019 * License along with this program.  If not, see
020 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
021 * #L%
022 */
023
024package dk.netarkivet.common.utils;
025
026import static org.junit.Assert.assertEquals;
027
028import java.io.BufferedInputStream;
029import java.io.BufferedOutputStream;
030import java.io.File;
031import java.io.FileInputStream;
032import java.io.FileOutputStream;
033import java.io.IOException;
034import java.io.InputStream;
035import java.io.OutputStream;
036import java.util.zip.GZIPOutputStream;
037
038import org.apache.commons.io.IOUtils;
039import org.junit.Ignore;
040import org.junit.Test;
041
042import dk.netarkivet.common.CommonSettings;
043import dk.netarkivet.common.Constants;
044import dk.netarkivet.common.distribute.FTPRemoteFile;
045import dk.netarkivet.common.distribute.RemoteFile;
046import dk.netarkivet.common.distribute.RemoteFileFactory;
047import dk.netarkivet.testutils.FileAsserts;
048import dk.netarkivet.testutils.TestFileUtils;
049import dk.netarkivet.testutils.preconfigured.ReloadSettings;
050
051/**
052 * Integrity tests for the package dk.netarkivet.common.utils.
053 */
054@Ignore("Needs to be run in deploy-test module according to junit 3 test suite")
055public class IntegrityTester {
056    ReloadSettings rs = new ReloadSettings();
057    private static final File BASE_DIR = new File("tests/dk/netarkivet/common/utils");
058    private static final File ORIGINALS = new File(BASE_DIR, "fileutils_data");
059    private static final File WORKING = new File(BASE_DIR, "working");
060    private static final File SUBDIR = new File(WORKING, "subdir");
061    private static final File SUBDIR2 = new File(WORKING, "subdir2");
062    private static final File SUBDIR3 = new File(WORKING, "subdir3");
063    private static final int BLOCKSIZE = 32768;
064    private static final long LARGE = ((long) Integer.MAX_VALUE) + 1L;
065    public static final String LARGE_FILE = "largeFile";
066
067    public void setUp() {
068        rs.setUp();
069        FileUtils.removeRecursively(WORKING);
070        TestFileUtils.copyDirectoryNonCVS(ORIGINALS, WORKING);
071        /** Do not send notifications by email. Print them to STDOUT. */
072        Settings.set(CommonSettings.NOTIFICATIONS_CLASS, RememberNotifications.class.getName());
073    }
074
075    public void tearDown() {
076        FileUtils.removeRecursively(WORKING);
077        rs.tearDown();
078    }
079
080    /**
081     * test that FileUtils.append can append between two remote files using ftp.
082     */
083    @Test
084    public void failingTestAppendRemoteFiles() throws IOException {
085        Settings.set(CommonSettings.REMOTE_FILE_CLASS, FTPRemoteFile.class.getName());
086        File in_1 = new File(WORKING, "append_data/file1");
087        File in_2 = new File(WORKING, "append_data/file2");
088        File out_file = new File(WORKING, "append_data/output");
089        RemoteFile rf1 = RemoteFileFactory.getInstance(in_1, true, false, true);
090        RemoteFile rf2 = RemoteFileFactory.getInstance(in_2, true, false, true);
091        OutputStream out = new FileOutputStream(out_file);
092        rf1.appendTo(out);
093        rf2.appendTo(out);
094        out.close();
095        FileAsserts.assertFileNumberOfLines("File has wrong number of lines", out_file, 2);
096        FileAsserts.assertFileContains("Missing content", "1", out_file);
097        FileAsserts.assertFileContains("Missing content", "2", out_file);
098    }
099
100    /** Test that files larger than 2GB can be copied! */
101    @Test
102    public void failingTestCopyLargeFiles() throws IOException {
103        byte[] block = new byte[BLOCKSIZE];
104        SUBDIR.mkdirs();
105        File largeFile = new File(SUBDIR, LARGE_FILE);
106        OutputStream os = null;
107        try {
108            os = new BufferedOutputStream(new FileOutputStream(largeFile));
109            System.out.println("Creating file - this will take a long time");
110            for (long l = 0; l < LARGE / ((long) BLOCKSIZE) + 1L; l++) {
111                os.write(block);
112            }
113            System.out.println("Copying file - this will take a long time");
114            FileUtils.copyDirectory(SUBDIR, SUBDIR2);
115            File file1 = new File(SUBDIR, LARGE_FILE);
116            File file2 = new File(SUBDIR2, LARGE_FILE);
117            assertEquals("Should have same file sizes", file1.length(), file2.length());
118        } finally {
119            IOUtils.closeQuietly(os);
120        }
121    }
122
123    /**
124     * This tests that we are actually able to write and read more than 4GB worth of data using GZip.
125     *
126     * @throws IOException
127     */
128    @Test
129    public void failingTestGzipLargeFile() throws IOException {
130        byte[] block = new byte[BLOCKSIZE];
131        File largeFile = new File(WORKING, LARGE_FILE);
132        OutputStream os = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(largeFile)));
133        System.out.println("Creating " + 5 * LARGE + " bytes file " + "- this will take a long time");
134        block[1] = 'a';
135        block[2] = 'b';
136        for (long l = 0; l < 5 * LARGE / ((long) BLOCKSIZE) + 1L; l++) {
137            os.write(block);
138        }
139        os.close();
140
141        InputStream is = new LargeFileGZIPInputStream(new BufferedInputStream(new FileInputStream(largeFile)));
142        System.out.println("Reading " + 5 * LARGE + " bytes file " + "- this will take a long time");
143        byte[] buf = new byte[BLOCKSIZE];
144        for (long l = 0; l < 5 * LARGE / ((long) BLOCKSIZE) + 1L; l++) {
145            int totalRead = 0;
146            int read = 0;
147            while (totalRead != block.length && read != -1) {
148                read = is.read(buf, totalRead, buf.length - totalRead);
149                totalRead += read;
150            }
151            assertEquals("Should have read full length of block " + l, block.length, totalRead);
152            for (int i = 0; i < 8; i++) {
153                assertEquals("Read block " + l + " should be equals at " + i, block[i], buf[i]);
154            }
155        }
156        assertEquals("This should be the end of the stream.", -1, is.read());
157        is.close();
158    }
159
160    /** Test that files larger than 2GB can be gzipped and gunzipped! */
161    @Test
162    public void failingTestGZipGUnZipLargeFiles() throws IOException {
163        byte[] block = new byte[BLOCKSIZE];
164        SUBDIR.mkdirs();
165        File largeFile = new File(SUBDIR, LARGE_FILE);
166        OutputStream os = null;
167        try {
168            os = new FileOutputStream(largeFile);
169            System.out.println("Creating file - this will take a long time");
170            for (long l = 0; l < LARGE / ((long) BLOCKSIZE) + 1L; l++) {
171                os.write(block);
172            }
173            System.out.println("Zipping file - this will take a long time");
174            ZipUtils.gzipFiles(SUBDIR, SUBDIR2);
175            System.out.println("UnZipping file - this will take a long time");
176            ZipUtils.gunzipFiles(SUBDIR2, SUBDIR3);
177            File file1 = new File(SUBDIR, LARGE_FILE);
178            File file2 = new File(SUBDIR3, LARGE_FILE);
179            assertEquals("Should have same file sizes", file1.length(), file2.length());
180        } finally {
181            IOUtils.closeQuietly(os);
182        }
183    }
184
185    /**
186     * Test reading two large files: One that should unzip OK, one that should fail with wrong CRC checksum.
187     *
188     * @throws IOException
189     */
190    @Test
191    public void failingTestLargeGZIPReadLargeFiles() throws IOException {
192        LargeFileGZIPInputStream largeFileGZIPInputStream = new LargeFileGZIPInputStream(new FileInputStream(new File(
193                BASE_DIR, "data/largeFileWrongCRC/largeFile.gz")));
194        try {
195            byte[] buffer = new byte[Constants.IO_BUFFER_SIZE];
196            while ((largeFileGZIPInputStream.read(buffer)) > 0) {
197                // just carry on.
198            }
199            largeFileGZIPInputStream.close();
200            // Unfortunately this doesn't work. Let's look forward to Java 1.6.0.
201            // fail("Should throw exception on wrong CRC");
202        } catch (IOException e) {
203            // expected... but
204            // Unfortunately this doesn't work. Let's look forward to Java 1.6.0.
205            assertEquals("Must be the right exception, not " + ExceptionUtils.getStackTrace(e), "Corrupt GZIP trailer",
206                    e.getMessage());
207        }
208
209        largeFileGZIPInputStream = new LargeFileGZIPInputStream(new FileInputStream(new File(BASE_DIR,
210                "data/largeFile/largeFile.gz")));
211        byte[] buffer = new byte[Constants.IO_BUFFER_SIZE];
212        while ((largeFileGZIPInputStream.read(buffer)) > 0) {
213            // just carry on.
214        }
215        largeFileGZIPInputStream.close();
216    }
217}