Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertThrows;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.LocatedFileStatus;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.fs.RemoteIterator;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.MetaTableAccessor;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.ResultScanner;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.client.SnapshotType;
043import org.apache.hadoop.hbase.client.Table;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
046import org.apache.hadoop.hbase.io.HFileLink;
047import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
048import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
050import org.apache.hadoop.hbase.mob.MobUtils;
051import org.apache.hadoop.hbase.monitoring.MonitoredTask;
052import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
053import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
054import org.apache.hadoop.hbase.regionserver.HRegion;
055import org.apache.hadoop.hbase.regionserver.HRegionServer;
056import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
057import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.testclassification.RegionServerTests;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.hadoop.hbase.util.CommonFSUtils;
062import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
063import org.apache.hadoop.hbase.util.FSTableDescriptors;
064import org.apache.hadoop.hbase.wal.WALSplitUtil;
065import org.junit.After;
066import org.junit.AfterClass;
067import org.junit.Assert;
068import org.junit.Before;
069import org.junit.BeforeClass;
070import org.junit.ClassRule;
071import org.junit.Test;
072import org.junit.experimental.categories.Category;
073import org.mockito.Mockito;
074import org.slf4j.Logger;
075import org.slf4j.LoggerFactory;
076
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
078
079/**
080 * Test the restore/clone operation from a file-system point of view.
081 */
082@Category({ RegionServerTests.class, MediumTests.class })
083public class TestRestoreSnapshotHelper {
084
085  @ClassRule
086  public static final HBaseClassTestRule CLASS_RULE =
087    HBaseClassTestRule.forClass(TestRestoreSnapshotHelper.class);
088
089  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotHelper.class);
090
091  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
092  protected final static String TEST_HFILE = "abc";
093
094  protected Configuration conf;
095  protected Path archiveDir;
096  protected FileSystem fs;
097  protected Path rootDir;
098
099  protected void setupConf(Configuration conf) {
100  }
101
102  @BeforeClass
103  public static void setupCluster() throws Exception {
104    TEST_UTIL.getConfiguration().setInt(AssignmentManager.ASSIGN_MAX_ATTEMPTS, 3);
105    TEST_UTIL.startMiniCluster();
106  }
107
108  @AfterClass
109  public static void tearDownCluster() throws Exception {
110    TEST_UTIL.shutdownMiniCluster();
111  }
112
113  @Before
114  public void setup() throws Exception {
115    rootDir = TEST_UTIL.getDataTestDir("testRestore");
116    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
117    fs = TEST_UTIL.getTestFileSystem();
118    conf = TEST_UTIL.getConfiguration();
119    setupConf(conf);
120    CommonFSUtils.setRootDir(conf, rootDir);
121    // Turn off balancer so it doesn't cut in and mess up our placements.
122    TEST_UTIL.getAdmin().balancerSwitch(false, true);
123  }
124
125  @After
126  public void tearDown() throws Exception {
127    fs.delete(TEST_UTIL.getDataTestDir(), true);
128  }
129
130  protected SnapshotMock createSnapshotMock() throws IOException {
131    return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
132  }
133
134  @Test
135  public void testRestore() throws IOException {
136    restoreAndVerify("snapshot", "testRestore");
137  }
138
139  @Test
140  public void testRestoreWithNamespace() throws IOException {
141    restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
142  }
143
144  @Test
145  public void testNoHFileLinkInRootDir() throws IOException {
146    rootDir = TEST_UTIL.getDefaultRootDirPath();
147    CommonFSUtils.setRootDir(conf, rootDir);
148    fs = rootDir.getFileSystem(conf);
149
150    TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
151    String snapshotName = tableName.getNameAsString() + "-snapshot";
152    createTableAndSnapshot(tableName, snapshotName);
153
154    Path restoreDir = new Path("/hbase/.tmp-restore");
155    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
156    checkNoHFileLinkInTableDir(tableName);
157  }
158
159  @Test
160  public void testSkipReplayAndUpdateSeqId() throws Exception {
161    rootDir = TEST_UTIL.getDefaultRootDirPath();
162    CommonFSUtils.setRootDir(conf, rootDir);
163    TableName tableName = TableName.valueOf("testSkipReplayAndUpdateSeqId");
164    String snapshotName = "testSkipReplayAndUpdateSeqId";
165    createTableAndSnapshot(tableName, snapshotName);
166    // put some data in the table
167    Table table = TEST_UTIL.getConnection().getTable(tableName);
168    TEST_UTIL.loadTable(table, Bytes.toBytes("A"));
169
170    Configuration conf = TEST_UTIL.getConfiguration();
171    Path rootDir = CommonFSUtils.getRootDir(conf);
172    Path restoreDir = new Path("/hbase/.tmp-restore/testScannerWithRestoreScanner2");
173    // restore snapshot.
174    final RestoreSnapshotHelper.RestoreMetaChanges meta =
175      RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
176    TableDescriptor htd = meta.getTableDescriptor();
177    final List<RegionInfo> restoredRegions = meta.getRegionsToAdd();
178    for (RegionInfo restoredRegion : restoredRegions) {
179      // open restored region
180      HRegion region = HRegion.newHRegion(CommonFSUtils.getTableDir(restoreDir, tableName), null,
181        fs, conf, restoredRegion, htd, null);
182      // set restore flag
183      region.setRestoredRegion(true);
184      region.initialize();
185      Path recoveredEdit =
186        CommonFSUtils.getWALRegionDir(conf, tableName, region.getRegionInfo().getEncodedName());
187      long maxSeqId = WALSplitUtil.getMaxRegionSequenceId(fs, recoveredEdit);
188
189      // open restored region without set restored flag
190      HRegion region2 = HRegion.newHRegion(CommonFSUtils.getTableDir(restoreDir, tableName), null,
191        fs, conf, restoredRegion, htd, null);
192      region2.initialize();
193      long maxSeqId2 = WALSplitUtil.getMaxRegionSequenceId(fs, recoveredEdit);
194      Assert.assertTrue(maxSeqId2 > maxSeqId);
195    }
196  }
197
198  @Test
199  public void testCopyExpiredSnapshotForScanner() throws IOException, InterruptedException {
200    rootDir = TEST_UTIL.getDefaultRootDirPath();
201    CommonFSUtils.setRootDir(conf, rootDir);
202    TableName tableName = TableName.valueOf("testCopyExpiredSnapshotForScanner");
203    String snapshotName = tableName.getNameAsString() + "-snapshot";
204    Path restoreDir = new Path("/hbase/.tmp-expired-snapshot/copySnapshotDest");
205    // create table and put some data into the table
206    byte[] columnFamily = Bytes.toBytes("A");
207    Table table = TEST_UTIL.createTable(tableName, columnFamily);
208    TEST_UTIL.loadTable(table, columnFamily);
209    // create snapshot with ttl = 10 sec
210    Map<String, Object> properties = new HashMap<>();
211    properties.put("TTL", 10);
212    org.apache.hadoop.hbase.client.SnapshotDescription snapshotDesc =
213      new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
214        SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
215    TEST_UTIL.getAdmin().snapshot(snapshotDesc);
216    boolean isExist = TEST_UTIL.getAdmin().listSnapshots().stream()
217      .anyMatch(ele -> snapshotName.equals(ele.getName()));
218    assertTrue(isExist);
219    int retry = 6;
220    while (
221      !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
222        snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
223    ) {
224      retry--;
225      Thread.sleep(10 * 1000);
226    }
227    boolean isExpiredSnapshot = SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
228      snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime());
229    assertTrue(isExpiredSnapshot);
230    assertThrows(SnapshotTTLExpiredException.class, () -> RestoreSnapshotHelper
231      .copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName));
232  }
233
234  /**
235   * Test scenario for HBASE-29346, which addresses the issue where restoring snapshots after region
236   * merge operations could lead to missing store file references, potentially resulting in data
237   * loss.
238   * <p>
239   * This test performs the following steps:
240   * </p>
241   * <ol>
242   * <li>Creates a table with multiple regions.</li>
243   * <li>Inserts data into each region and flushes to create store files.</li>
244   * <li>Takes snapshot of the table and performs restore.</li>
245   * <li>Disable compactions, merge regions, create a new snapshot, and restore that snapshot on the
246   * same restore path.</li>
247   * <li>Verifies data integrity by scanning all data post region re-open.</li>
248   * </ol>
249   */
250  @Test
251  public void testMultiSnapshotRestoreWithMerge() throws IOException, InterruptedException {
252    rootDir = TEST_UTIL.getDefaultRootDirPath();
253    CommonFSUtils.setRootDir(conf, rootDir);
254    TableName tableName = TableName.valueOf("testMultiSnapshotRestoreWithMerge");
255    Path restoreDir = new Path("/hbase/.tmp-snapshot/restore-snapshot-dest");
256
257    byte[] columnFamily = Bytes.toBytes("A");
258    Table table = TEST_UTIL.createTable(tableName, new byte[][] { columnFamily },
259      new byte[][] { new byte[] { 'b' }, new byte[] { 'd' } });
260    Put put1 = new Put(Bytes.toBytes("a")); // Region 1: [-∞, b)
261    put1.addColumn(columnFamily, Bytes.toBytes("q"), Bytes.toBytes("val1"));
262    table.put(put1);
263    Put put2 = new Put(Bytes.toBytes("b")); // Region 2: [b, d)
264    put2.addColumn(columnFamily, Bytes.toBytes("q"), Bytes.toBytes("val2"));
265    table.put(put2);
266    Put put3 = new Put(Bytes.toBytes("d")); // Region 3: [d, +∞)
267    put3.addColumn(columnFamily, Bytes.toBytes("q"), Bytes.toBytes("val3"));
268    table.put(put3);
269
270    TEST_UTIL.getAdmin().flush(tableName);
271
272    String snapshotOne = tableName.getNameAsString() + "-snapshot-one";
273    createAndAssertSnapshot(tableName, snapshotOne);
274    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotOne);
275    flipCompactions(false);
276    mergeRegions(tableName, 2);
277    String snapshotTwo = tableName.getNameAsString() + "-snapshot-two";
278    createAndAssertSnapshot(tableName, snapshotTwo);
279    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotTwo);
280    flipCompactions(true);
281
282    TEST_UTIL.getAdmin().disableTable(tableName);
283    TEST_UTIL.getAdmin().enableTable(tableName);
284    try (ResultScanner scanner = table.getScanner(new Scan())) {
285      assertEquals(3, scanner.next(4).length);
286    }
287    String snapshotThree = tableName.getNameAsString() + "-snapshot-three";
288    createAndAssertSnapshot(tableName, snapshotThree);
289  }
290
291  private void createAndAssertSnapshot(TableName tableName, String snapshotName)
292    throws SnapshotCreationException, IllegalArgumentException, IOException {
293    org.apache.hadoop.hbase.client.SnapshotDescription snapshotDescOne =
294      new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
295        SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1);
296    TEST_UTIL.getAdmin().snapshot(snapshotDescOne);
297    boolean isExist = TEST_UTIL.getAdmin().listSnapshots().stream()
298      .anyMatch(ele -> snapshotName.equals(ele.getName()));
299    assertTrue(isExist);
300
301  }
302
303  private void flipCompactions(boolean isEnable) {
304    int numLiveRegionServers = TEST_UTIL.getHBaseCluster().getNumLiveRegionServers();
305    for (int serverNumber = 0; serverNumber < numLiveRegionServers; serverNumber++) {
306      HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(serverNumber);
307      regionServer.getCompactSplitThread().setCompactionsEnabled(isEnable);
308    }
309
310  }
311
312  private void mergeRegions(TableName tableName, int mergeCount) throws IOException {
313    List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName);
314    int originalRegionCount = ris.size();
315    assertTrue(originalRegionCount > mergeCount);
316    RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo[] {});
317    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
318    MergeTableRegionsProcedure proc =
319      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
320    long procId = procExec.submitProcedure(proc);
321    ProcedureTestingUtility.waitProcedure(procExec, procId);
322    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
323    MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection());
324    assertEquals(originalRegionCount - mergeCount + 1,
325      MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName).size());
326  }
327
328  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
329    return TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
330  }
331
332  protected void createTableAndSnapshot(TableName tableName, String snapshotName)
333    throws IOException {
334    byte[] column = Bytes.toBytes("A");
335    Table table = TEST_UTIL.createTable(tableName, column, 2);
336    TEST_UTIL.loadTable(table, column);
337    TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
338  }
339
340  private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
341    Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
342      CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
343      CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
344    for (Path tableDir : tableDirs) {
345      Assert.assertFalse(hasHFileLink(tableDir));
346    }
347  }
348
349  private boolean hasHFileLink(Path tableDir) throws IOException {
350    if (fs.exists(tableDir)) {
351      RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
352      while (iterator.hasNext()) {
353        LocatedFileStatus fileStatus = iterator.next();
354        if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
355          return true;
356        }
357      }
358    }
359    return false;
360  }
361
362  private void restoreAndVerify(final String snapshotName, final String tableName)
363    throws IOException {
364    // Test Rolling-Upgrade like Snapshot.
365    // half machines writing using v1 and the others using v2 format.
366    SnapshotMock snapshotMock = createSnapshotMock();
367    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName);
368    builder.addRegionV1();
369    builder.addRegionV2();
370    builder.addRegionV2();
371    builder.addRegionV1();
372    Path snapshotDir = builder.commit();
373    TableDescriptor htd = builder.getTableDescriptor();
374    SnapshotDescription desc = builder.getSnapshotDescription();
375
376    // Test clone a snapshot
377    TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
378    testRestore(snapshotDir, desc, htdClone);
379    verifyRestore(rootDir, htd, htdClone);
380
381    // Test clone a clone ("link to link")
382    SnapshotDescription cloneDesc =
383      SnapshotDescription.newBuilder().setName("cloneSnapshot").setTable("testtb-clone").build();
384    Path cloneDir = CommonFSUtils.getTableDir(rootDir, htdClone.getTableName());
385    TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
386    testRestore(cloneDir, cloneDesc, htdClone2);
387    verifyRestore(rootDir, htd, htdClone2);
388  }
389
390  private void verifyRestore(final Path rootDir, final TableDescriptor sourceHtd,
391    final TableDescriptor htdClone) throws IOException {
392    List<String> files = SnapshotTestingUtils.listHFileNames(fs,
393      CommonFSUtils.getTableDir(rootDir, htdClone.getTableName()));
394    assertEquals(12, files.size());
395    for (int i = 0; i < files.size(); i += 2) {
396      String linkFile = files.get(i);
397      String refFile = files.get(i + 1);
398      assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
399      assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
400      assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
401      Path refPath = getReferredToFile(refFile);
402      LOG.debug("get reference name for file " + refFile + " = " + refPath);
403      assertTrue(refPath.getName() + " should be a HFileLink",
404        HFileLink.isHFileLink(refPath.getName()));
405      assertEquals(linkFile, refPath.getName());
406    }
407  }
408
409  /**
410   * Execute the restore operation
411   * @param snapshotDir The snapshot directory to use as "restore source"
412   * @param sd          The snapshot descriptor
413   * @param htdClone    The HTableDescriptor of the table to restore/clone.
414   */
415  private void testRestore(final Path snapshotDir, final SnapshotDescription sd,
416    final TableDescriptor htdClone) throws IOException {
417    LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
418    CommonFSUtils.logFileSystemState(fs, rootDir, LOG);
419
420    new FSTableDescriptors(conf).createTableDescriptor(htdClone);
421    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
422    helper.restoreHdfsRegions();
423
424    LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
425    CommonFSUtils.logFileSystemState(fs, rootDir, LOG);
426  }
427
428  /**
429   * Initialize the restore helper, based on the snapshot and table information provided.
430   */
431  private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
432    final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException {
433    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
434    MonitoredTask status = Mockito.mock(MonitoredTask.class);
435
436    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
437    return new RestoreSnapshotHelper(conf, fs, manifest, htdClone, rootDir, monitor, status);
438  }
439
440  private Path getReferredToFile(final String referenceName) {
441    Path fakeBasePath = new Path(new Path("table", "region"), "cf");
442    return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
443  }
444}