001package dk.netarkivet.common.utils.hadoop; 002 003import java.io.IOException; 004import java.nio.file.Files; 005import java.util.UUID; 006 007import org.apache.hadoop.fs.FileSystem; 008import org.apache.hadoop.fs.Path; 009import org.slf4j.Logger; 010import org.slf4j.LoggerFactory; 011 012import dk.netarkivet.common.CommonSettings; 013import dk.netarkivet.common.utils.Settings; 014 015/** Utilities for file actions related to Hadoop. */ 016public class HadoopFileUtils { 017 private static final Logger log = LoggerFactory.getLogger(HadoopFileUtils.class); 018 019 /** 020 * Creates and returns a unique path under a given directory. 021 * @param fileSystem The used filesystem 022 * @param dir A path to the parent directory to create the Path under 023 * @param uuid The UUID used to name the Path 024 * @return A Hadoop path representing a unique file/directory or null if an error is encountered 025 */ 026 public static Path createUniquePathInDir(FileSystem fileSystem, String dir, UUID uuid) { 027 try { 028 initDir(fileSystem, dir); 029 } catch (IOException e) { 030 log.error("Failed to create output dir '{}'", dir, e); 031 return null; 032 } 033 return new Path(dir, uuid.toString()); 034 } 035 036 /** 037 * Initializes the given directory on the filesystem by deleting any existing file on the direct path 038 * and making all parent dirs in the directory path. 039 * @param fileSystem The filesystem on which the actions are executed. 040 * @param hadoopDir The directory path to initialize. 041 * @throws IOException If any action on the filesystem fails. 042 */ 043 public static void initDir(FileSystem fileSystem, String hadoopDir) throws IOException { 044 Path hadoopDirPath = new Path(hadoopDir); 045 if (fileSystem.exists(hadoopDirPath) && !fileSystem.isDirectory(hadoopDirPath)) { 046 log.warn("'{}' already exists and is a file. Deleting and creating directory.", hadoopDirPath); 047 fileSystem.delete(hadoopDirPath, true); 048 } else { 049 log.info("Creating dir '{}'", hadoopDirPath); 050 } 051 fileSystem.mkdirs(hadoopDirPath); 052 } 053 054 public static java.nio.file.Path makeLocalInputTempFile() { 055 java.nio.file.Path localInputTempFile = null; 056 try { 057 localInputTempFile = Files.createTempFile(null, null); 058 } catch (IOException e) { 059 log.error("Failed writing to/creating file.", e); 060 } 061 return localInputTempFile; 062 } 063}