001/* 002 * #%L 003 * Netarchivesuite - common - test 004 * %% 005 * Copyright (C) 2005 - 2018 The Royal Danish Library, 006 * the National Library of France and the Austrian National Library. 007 * %% 008 * This program is free software: you can redistribute it and/or modify 009 * it under the terms of the GNU Lesser General Public License as 010 * published by the Free Software Foundation, either version 2.1 of the 011 * License, or (at your option) any later version. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Lesser Public License for more details. 017 * 018 * You should have received a copy of the GNU General Lesser Public 019 * License along with this program. If not, see 020 * <http://www.gnu.org/licenses/lgpl-2.1.html>. 021 * #L% 022 */ 023 024package dk.netarkivet.common.utils; 025 026import static org.junit.Assert.assertEquals; 027 028import java.io.BufferedInputStream; 029import java.io.BufferedOutputStream; 030import java.io.File; 031import java.io.FileInputStream; 032import java.io.FileOutputStream; 033import java.io.IOException; 034import java.io.InputStream; 035import java.io.OutputStream; 036import java.util.zip.GZIPOutputStream; 037 038import org.apache.commons.io.IOUtils; 039import org.junit.Ignore; 040import org.junit.Test; 041 042import dk.netarkivet.common.CommonSettings; 043import dk.netarkivet.common.Constants; 044import dk.netarkivet.common.distribute.FTPRemoteFile; 045import dk.netarkivet.common.distribute.RemoteFile; 046import dk.netarkivet.common.distribute.RemoteFileFactory; 047import dk.netarkivet.testutils.FileAsserts; 048import dk.netarkivet.testutils.TestFileUtils; 049import dk.netarkivet.testutils.preconfigured.ReloadSettings; 050 051/** 052 * Integrity tests for the package dk.netarkivet.common.utils. 053 */ 054@Ignore("Needs to be run in deploy-test module according to junit 3 test suite") 055public class IntegrityTester { 056 ReloadSettings rs = new ReloadSettings(); 057 private static final File BASE_DIR = new File("tests/dk/netarkivet/common/utils"); 058 private static final File ORIGINALS = new File(BASE_DIR, "fileutils_data"); 059 private static final File WORKING = new File(BASE_DIR, "working"); 060 private static final File SUBDIR = new File(WORKING, "subdir"); 061 private static final File SUBDIR2 = new File(WORKING, "subdir2"); 062 private static final File SUBDIR3 = new File(WORKING, "subdir3"); 063 private static final int BLOCKSIZE = 32768; 064 private static final long LARGE = ((long) Integer.MAX_VALUE) + 1L; 065 public static final String LARGE_FILE = "largeFile"; 066 067 public void setUp() { 068 rs.setUp(); 069 FileUtils.removeRecursively(WORKING); 070 TestFileUtils.copyDirectoryNonCVS(ORIGINALS, WORKING); 071 /** Do not send notifications by email. Print them to STDOUT. */ 072 Settings.set(CommonSettings.NOTIFICATIONS_CLASS, RememberNotifications.class.getName()); 073 } 074 075 public void tearDown() { 076 FileUtils.removeRecursively(WORKING); 077 rs.tearDown(); 078 } 079 080 /** 081 * test that FileUtils.append can append between two remote files using ftp. 082 */ 083 @Test 084 public void failingTestAppendRemoteFiles() throws IOException { 085 Settings.set(CommonSettings.REMOTE_FILE_CLASS, FTPRemoteFile.class.getName()); 086 File in_1 = new File(WORKING, "append_data/file1"); 087 File in_2 = new File(WORKING, "append_data/file2"); 088 File out_file = new File(WORKING, "append_data/output"); 089 RemoteFile rf1 = RemoteFileFactory.getInstance(in_1, true, false, true); 090 RemoteFile rf2 = RemoteFileFactory.getInstance(in_2, true, false, true); 091 OutputStream out = new FileOutputStream(out_file); 092 rf1.appendTo(out); 093 rf2.appendTo(out); 094 out.close(); 095 FileAsserts.assertFileNumberOfLines("File has wrong number of lines", out_file, 2); 096 FileAsserts.assertFileContains("Missing content", "1", out_file); 097 FileAsserts.assertFileContains("Missing content", "2", out_file); 098 } 099 100 /** Test that files larger than 2GB can be copied! */ 101 @Test 102 public void failingTestCopyLargeFiles() throws IOException { 103 byte[] block = new byte[BLOCKSIZE]; 104 SUBDIR.mkdirs(); 105 File largeFile = new File(SUBDIR, LARGE_FILE); 106 OutputStream os = null; 107 try { 108 os = new BufferedOutputStream(new FileOutputStream(largeFile)); 109 System.out.println("Creating file - this will take a long time"); 110 for (long l = 0; l < LARGE / ((long) BLOCKSIZE) + 1L; l++) { 111 os.write(block); 112 } 113 System.out.println("Copying file - this will take a long time"); 114 FileUtils.copyDirectory(SUBDIR, SUBDIR2); 115 File file1 = new File(SUBDIR, LARGE_FILE); 116 File file2 = new File(SUBDIR2, LARGE_FILE); 117 assertEquals("Should have same file sizes", file1.length(), file2.length()); 118 } finally { 119 IOUtils.closeQuietly(os); 120 } 121 } 122 123 /** 124 * This tests that we are actually able to write and read more than 4GB worth of data using GZip. 125 * 126 * @throws IOException 127 */ 128 @Test 129 public void failingTestGzipLargeFile() throws IOException { 130 byte[] block = new byte[BLOCKSIZE]; 131 File largeFile = new File(WORKING, LARGE_FILE); 132 OutputStream os = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(largeFile))); 133 System.out.println("Creating " + 5 * LARGE + " bytes file " + "- this will take a long time"); 134 block[1] = 'a'; 135 block[2] = 'b'; 136 for (long l = 0; l < 5 * LARGE / ((long) BLOCKSIZE) + 1L; l++) { 137 os.write(block); 138 } 139 os.close(); 140 141 InputStream is = new LargeFileGZIPInputStream(new BufferedInputStream(new FileInputStream(largeFile))); 142 System.out.println("Reading " + 5 * LARGE + " bytes file " + "- this will take a long time"); 143 byte[] buf = new byte[BLOCKSIZE]; 144 for (long l = 0; l < 5 * LARGE / ((long) BLOCKSIZE) + 1L; l++) { 145 int totalRead = 0; 146 int read = 0; 147 while (totalRead != block.length && read != -1) { 148 read = is.read(buf, totalRead, buf.length - totalRead); 149 totalRead += read; 150 } 151 assertEquals("Should have read full length of block " + l, block.length, totalRead); 152 for (int i = 0; i < 8; i++) { 153 assertEquals("Read block " + l + " should be equals at " + i, block[i], buf[i]); 154 } 155 } 156 assertEquals("This should be the end of the stream.", -1, is.read()); 157 is.close(); 158 } 159 160 /** Test that files larger than 2GB can be gzipped and gunzipped! */ 161 @Test 162 public void failingTestGZipGUnZipLargeFiles() throws IOException { 163 byte[] block = new byte[BLOCKSIZE]; 164 SUBDIR.mkdirs(); 165 File largeFile = new File(SUBDIR, LARGE_FILE); 166 OutputStream os = null; 167 try { 168 os = new FileOutputStream(largeFile); 169 System.out.println("Creating file - this will take a long time"); 170 for (long l = 0; l < LARGE / ((long) BLOCKSIZE) + 1L; l++) { 171 os.write(block); 172 } 173 System.out.println("Zipping file - this will take a long time"); 174 ZipUtils.gzipFiles(SUBDIR, SUBDIR2); 175 System.out.println("UnZipping file - this will take a long time"); 176 ZipUtils.gunzipFiles(SUBDIR2, SUBDIR3); 177 File file1 = new File(SUBDIR, LARGE_FILE); 178 File file2 = new File(SUBDIR3, LARGE_FILE); 179 assertEquals("Should have same file sizes", file1.length(), file2.length()); 180 } finally { 181 IOUtils.closeQuietly(os); 182 } 183 } 184 185 /** 186 * Test reading two large files: One that should unzip OK, one that should fail with wrong CRC checksum. 187 * 188 * @throws IOException 189 */ 190 @Test 191 public void failingTestLargeGZIPReadLargeFiles() throws IOException { 192 LargeFileGZIPInputStream largeFileGZIPInputStream = new LargeFileGZIPInputStream(new FileInputStream(new File( 193 BASE_DIR, "data/largeFileWrongCRC/largeFile.gz"))); 194 try { 195 byte[] buffer = new byte[Constants.IO_BUFFER_SIZE]; 196 while ((largeFileGZIPInputStream.read(buffer)) > 0) { 197 // just carry on. 198 } 199 largeFileGZIPInputStream.close(); 200 // Unfortunately this doesn't work. Let's look forward to Java 1.6.0. 201 // fail("Should throw exception on wrong CRC"); 202 } catch (IOException e) { 203 // expected... but 204 // Unfortunately this doesn't work. Let's look forward to Java 1.6.0. 205 assertEquals("Must be the right exception, not " + ExceptionUtils.getStackTrace(e), "Corrupt GZIP trailer", 206 e.getMessage()); 207 } 208 209 largeFileGZIPInputStream = new LargeFileGZIPInputStream(new FileInputStream(new File(BASE_DIR, 210 "data/largeFile/largeFile.gz"))); 211 byte[] buffer = new byte[Constants.IO_BUFFER_SIZE]; 212 while ((largeFileGZIPInputStream.read(buffer)) > 0) { 213 // just carry on. 214 } 215 largeFileGZIPInputStream.close(); 216 } 217}