001package dk.netarkivet.common.utils.hadoop;
002
003import java.io.IOException;
004import java.nio.file.Files;
005import java.util.UUID;
006
007import org.apache.hadoop.fs.FileSystem;
008import org.apache.hadoop.fs.Path;
009import org.slf4j.Logger;
010import org.slf4j.LoggerFactory;
011
012import dk.netarkivet.common.CommonSettings;
013import dk.netarkivet.common.utils.Settings;
014
015/** Utilities for file actions related to Hadoop. */
016public class HadoopFileUtils {
017    private static final Logger log = LoggerFactory.getLogger(HadoopFileUtils.class);
018
019    /**
020     * Creates and returns a unique path under a given directory.
021     * @param fileSystem The used filesystem
022     * @param dir A path to the parent directory to create the Path under
023     * @param uuid The UUID used to name the Path
024     * @return A Hadoop path representing a unique file/directory or null if an error is encountered
025     */
026    public static Path createUniquePathInDir(FileSystem fileSystem, String dir, UUID uuid) {
027        try {
028            initDir(fileSystem, dir);
029        } catch (IOException e) {
030            log.error("Failed to create output dir '{}'", dir, e);
031            return null;
032        }
033        return new Path(dir, uuid.toString());
034    }
035
036    /**
037     * Initializes the given directory on the filesystem by deleting any existing file on the direct path
038     * and making all parent dirs in the directory path.
039     * @param fileSystem The filesystem on which the actions are executed.
040     * @param hadoopDir The directory path to initialize.
041     * @throws IOException If any action on the filesystem fails.
042     */
043    public static void initDir(FileSystem fileSystem, String hadoopDir) throws IOException {
044        Path hadoopDirPath = new Path(hadoopDir);
045        if (fileSystem.exists(hadoopDirPath) && !fileSystem.isDirectory(hadoopDirPath)) {
046            log.warn("'{}' already exists and is a file. Deleting and creating directory.", hadoopDirPath);
047            fileSystem.delete(hadoopDirPath, true);
048        } else {
049            log.info("Creating dir '{}'", hadoopDirPath);
050        }
051        fileSystem.mkdirs(hadoopDirPath);
052    }
053
054    public static java.nio.file.Path makeLocalInputTempFile() {
055        java.nio.file.Path localInputTempFile = null;
056        try {
057            localInputTempFile = Files.createTempFile(null, null);
058        } catch (IOException e) {
059            log.error("Failed writing to/creating file.", e);
060        }
061        return localInputTempFile;
062    }
063}