001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.List;
025import java.util.Map;
026import java.util.Set;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileStatus;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
035import org.apache.hadoop.hbase.testclassification.MasterTests;
036import org.apache.hadoop.hbase.testclassification.SmallTests;
037import org.junit.After;
038import org.junit.AfterClass;
039import org.junit.Before;
040import org.junit.BeforeClass;
041import org.junit.ClassRule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
048
049@Category({ MasterTests.class, SmallTests.class })
050public class TestBackupHFileCleaner {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054    HBaseClassTestRule.forClass(TestBackupHFileCleaner.class);
055
056  private static final Logger LOG = LoggerFactory.getLogger(TestBackupHFileCleaner.class);
057  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
058  private final static Configuration conf = TEST_UTIL.getConfiguration();
059  private final static TableName tableNameWithBackup = TableName.valueOf("backup.hfile.cleaner");
060  private final static TableName tableNameWithoutBackup =
061    TableName.valueOf("backup.hfile.cleaner2");
062
063  private static FileSystem fs = null;
064
065  private Path root;
066
067  @BeforeClass
068  public static void setUpBeforeClass() throws Exception {
069    conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
070    TEST_UTIL.startMiniCluster(1);
071    fs = FileSystem.get(conf);
072  }
073
074  @AfterClass
075  public static void tearDownAfterClass() throws Exception {
076    TEST_UTIL.shutdownMiniCluster();
077  }
078
079  @Before
080  public void setup() throws IOException {
081    root = TEST_UTIL.getDataTestDirOnTestFS();
082  }
083
084  @After
085  public void cleanup() {
086    try {
087      fs.delete(root, true);
088    } catch (IOException e) {
089      LOG.warn("Failed to delete files recursively from path " + root);
090    }
091  }
092
093  @Test
094  public void testGetDeletableFiles() throws IOException {
095    FileStatus file1 = createFile("file1");
096    FileStatus file1Archived = createFile("archived/file1");
097    FileStatus file2 = createFile("file2");
098    FileStatus file3 = createFile("file3");
099
100    BackupHFileCleaner cleaner = new BackupHFileCleaner() {
101      @Override
102      protected Set<TableName> fetchFullyBackedUpTables(BackupSystemTable tbl) {
103        return Set.of(tableNameWithBackup);
104      }
105    };
106    cleaner.setConf(conf);
107
108    Iterable<FileStatus> deletable;
109
110    // The first call will not allow any deletions because of the timestamp mechanism.
111    deletable = cleaner.getDeletableFiles(List.of(file1, file1Archived, file2, file3));
112    assertEquals(Set.of(), Sets.newHashSet(deletable));
113
114    // No bulk loads registered, so all files can be deleted.
115    deletable = cleaner.getDeletableFiles(List.of(file1, file1Archived, file2, file3));
116    assertEquals(Set.of(file1, file1Archived, file2, file3), Sets.newHashSet(deletable));
117
118    // Register some bulk loads.
119    try (BackupSystemTable backupSystem = new BackupSystemTable(TEST_UTIL.getConnection())) {
120      byte[] unused = new byte[] { 0 };
121      backupSystem.registerBulkLoad(tableNameWithBackup, unused,
122        Map.of(unused, List.of(file1.getPath())));
123      backupSystem.registerBulkLoad(tableNameWithoutBackup, unused,
124        Map.of(unused, List.of(file2.getPath())));
125    }
126
127    // File 1 can no longer be deleted, because it is registered as a bulk load.
128    deletable = cleaner.getDeletableFiles(List.of(file1, file1Archived, file2, file3));
129    assertEquals(Set.of(file2, file3), Sets.newHashSet(deletable));
130  }
131
132  private FileStatus createFile(String fileName) throws IOException {
133    Path file = new Path(root, fileName);
134    fs.createNewFile(file);
135    assertTrue("Test file not created!", fs.exists(file));
136    return fs.getFileStatus(file);
137  }
138}