From aa7e329dc1e375c5f5c12720c78853830d41d605 Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <1913250675@qq.com> Date: Wed, 17 Aug 2022 20:26:28 +0800 Subject: [PATCH 1/9] FIX FLAKY TEST CAUSED BY ASYNC WAIT --- .../org/apache/hadoop/hbase/regionserver/StoreScanner.java | 4 ++++ .../apache/hadoop/hbase/regionserver/TestStoreScanner.java | 3 +++ 2 files changed, 7 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 2cf9fe04cf30..62e212737957 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -1240,4 +1240,8 @@ public void shipped() throws IOException { trySwitchToStreamRead(); } } + + public long getOldestUnexpiredTS() { + return this.oldestUnexpiredTS; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index c1843e306f1b..bd06d392604c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -876,6 +876,9 @@ public void testWildCardTtlScan() throws IOException { HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List results = new ArrayList<>(); + if (scanner.getOldestUnexpiredTS() - now >= -200L) { + return; + } assertEquals(true, scanner.next(results)); assertEquals(2, results.size()); assertEquals(kvs[1], results.get(0)); From 2b46bf65c551f55bf4425950f4b2a0837ef8842b Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <1913250675@qq.com> Date: Fri, 26 Aug 2022 19:30:58 +0800 Subject: [PATCH 2/9] fix to avoid flaky tests --- .../hadoop/hbase/regionserver/StoreScanner.java | 15 ++++++++++++--- .../hbase/regionserver/TestStoreScanner.java | 3 --- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 62e212737957..bfcd19ad12bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -357,6 +357,18 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, seekAllScanner(scanInfo, scanners); } + // Almost the same as above except for settting the start time of oldestUnexpiredTS + StoreScanner(Scan scan, ScanInfo scanInfo, NavigableSet columns, + List scanners, Long timeStampOrigin) throws IOException { + // 0 is passed as readpoint because the test bypasses Store + this(null, scan, scanInfo, columns != null ? columns.size() : 0, 0L, + scan.getCacheBlocks(), ScanType.USER_SCAN); + oldestUnexpiredTS = timeStampOrigin - scanInfo.getTtl(); + this.matcher = + UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); + seekAllScanner(scanInfo, scanners); + } + // Used to instantiate a scanner for user scan in test StoreScanner(Scan scan, ScanInfo scanInfo, NavigableSet columns, List scanners) throws IOException { @@ -1241,7 +1253,4 @@ public void shipped() throws IOException { } } - public long getOldestUnexpiredTS() { - return this.oldestUnexpiredTS; - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index bd06d392604c..c1843e306f1b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -876,9 +876,6 @@ public void testWildCardTtlScan() throws IOException { HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List results = new ArrayList<>(); - if (scanner.getOldestUnexpiredTS() - now >= -200L) { - return; - } assertEquals(true, scanner.next(results)); assertEquals(2, results.size()); assertEquals(kvs[1], results.get(0)); From 38123f1bd4921ec232d8c96d7d1ec93c9c0b739e Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <1913250675@qq.com> Date: Fri, 26 Aug 2022 19:33:02 +0800 Subject: [PATCH 3/9] rebase --- .../org/apache/hadoop/hbase/regionserver/TestStoreScanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index c1843e306f1b..056f9fc62d10 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -874,7 +874,7 @@ public void testWildCardTtlScan() throws IOException { scan.readVersions(1); ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); - try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { + try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners, now)) { List results = new ArrayList<>(); assertEquals(true, scanner.next(results)); assertEquals(2, results.size()); From 3c2ea61b9bcb8932d218b1c478c0e4a988fbefb5 Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <73588409+amamiya-yuuko-1225@users.noreply.github.com> Date: Tue, 30 Aug 2022 17:04:12 +0800 Subject: [PATCH 4/9] Update StoreScanner.java --- .../java/org/apache/hadoop/hbase/regionserver/StoreScanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index bfcd19ad12bf..715f8b348fdc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -84,7 +84,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner private boolean parallelSeekEnabled = false; private ExecutorService executor; private final Scan scan; - private final long oldestUnexpiredTS; + private long oldestUnexpiredTS = 0L; private final long now; private final int minVersions; private final long maxRowSize; From 170b3ffca62f024b0aeb965c59f4fe3c0aa26bd2 Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <73588409+amamiya-yuuko-1225@users.noreply.github.com> Date: Tue, 30 Aug 2022 17:04:56 +0800 Subject: [PATCH 5/9] Update StoreScanner.java --- .../java/org/apache/hadoop/hbase/regionserver/StoreScanner.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 715f8b348fdc..927b374fbfcf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -1252,5 +1252,4 @@ public void shipped() throws IOException { trySwitchToStreamRead(); } } - } From e887fde042892233bba1ca3780e8e66fb4580dee Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <73588409+amamiya-yuuko-1225@users.noreply.github.com> Date: Wed, 7 Sep 2022 10:38:45 +0800 Subject: [PATCH 6/9] Update StoreScanner.java --- .../hbase/regionserver/StoreScanner.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 927b374fbfcf..6b7ae708870e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -84,7 +84,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner private boolean parallelSeekEnabled = false; private ExecutorService executor; private final Scan scan; - private long oldestUnexpiredTS = 0L; + private final long oldestUnexpiredTS; private final long now; private final int minVersions; private final long maxRowSize; @@ -161,7 +161,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner /** An internal constructor. */ private StoreScanner(HStore store, Scan scan, ScanInfo scanInfo, int numColumns, long readPt, - boolean cacheBlocks, ScanType scanType) { + boolean cacheBlocks, ScanType scanType, Long timeStampOrigin) { this.readPt = readPt; this.store = store; this.cacheBlocks = cacheBlocks; @@ -170,7 +170,11 @@ private StoreScanner(HStore store, Scan scan, ScanInfo scanInfo, int numColumns, explicitColumnQuery = numColumns > 0; this.scan = scan; this.now = EnvironmentEdgeManager.currentTime(); - this.oldestUnexpiredTS = scan.isRaw() ? 0L : now - scanInfo.getTtl(); + if (timeStampOrigin == null) { + this.oldestUnexpiredTS = scan.isRaw() ? 0L : now - scanInfo.getTtl(); + } else { + this.oldestUnexpiredTS = timeStampOrigin - scanInfo.getTtl(); + } this.minVersions = scanInfo.getMinVersions(); // We look up row-column Bloom filters for multi-column queries as part of @@ -230,7 +234,7 @@ private void addCurrentScanners(List scanners) { public StoreScanner(HStore store, ScanInfo scanInfo, Scan scan, NavigableSet columns, long readPt) throws IOException { this(store, scan, scanInfo, columns != null ? columns.size() : 0, readPt, scan.getCacheBlocks(), - ScanType.USER_SCAN); + ScanType.USER_SCAN, null); if (columns != null && scan.isRaw()) { throw new DoNotRetryIOException("Cannot specify any column for a raw scan"); } @@ -307,7 +311,7 @@ private StoreScanner(HStore store, ScanInfo scanInfo, List s // For mob compaction only as we do not have a Store instance when doing mob compaction. public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners) throws IOException { - this(null, SCAN_FOR_COMPACTION, scanInfo, 0, Long.MAX_VALUE, false, scanType); + this(null, SCAN_FOR_COMPACTION, scanInfo, 0, Long.MAX_VALUE, false, scanType, null); assert scanType != ScanType.USER_SCAN; this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, 0L, oldestUnexpiredTS, now, null, null, null); @@ -346,7 +350,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners, ScanType scanType) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, scan, scanInfo, columns != null ? columns.size() : 0, 0L, scan.getCacheBlocks(), - scanType); + scanType, null); if (scanType == ScanType.USER_SCAN) { this.matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); @@ -362,8 +366,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners, Long timeStampOrigin) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, scan, scanInfo, columns != null ? columns.size() : 0, 0L, - scan.getCacheBlocks(), ScanType.USER_SCAN); - oldestUnexpiredTS = timeStampOrigin - scanInfo.getTtl(); + scan.getCacheBlocks(), ScanType.USER_SCAN, timeStampOrigin); this.matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); seekAllScanner(scanInfo, scanners); @@ -374,7 +377,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, scan, scanInfo, columns != null ? columns.size() : 0, 0L, scan.getCacheBlocks(), - ScanType.USER_SCAN); + ScanType.USER_SCAN, null); this.matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); seekAllScanner(scanInfo, scanners); @@ -385,7 +388,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, maxVersions > 0 ? new Scan().readVersions(maxVersions) : SCAN_FOR_COMPACTION, - scanInfo, 0, 0L, false, scanType); + scanInfo, 0, 0L, false, scanType, null); this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, PrivateConstants.OLDEST_TIMESTAMP, oldestUnexpiredTS, now, null, null, null); seekAllScanner(scanInfo, scanners); From 7d345df8001ec87da8258566ecbb977dfd60f321 Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <73588409+amamiya-yuuko-1225@users.noreply.github.com> Date: Wed, 7 Sep 2022 18:50:21 +0800 Subject: [PATCH 7/9] Update StoreScanner.java --- .../hadoop/hbase/regionserver/StoreScanner.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 6b7ae708870e..0349f8ee943b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -219,6 +219,11 @@ private StoreScanner(HStore store, Scan scan, ScanInfo scanInfo, int numColumns, } } } + + private StoreScanner(HStore store, Scan scan, ScanInfo scanInfo, int numColumns, long readPt, + boolean cacheBlocks, ScanType scanType) { + this(store, scan, scanInfo, numColumns, readPt, cacheBlocks, scanType, null); + } private void addCurrentScanners(List scanners) { this.currentScanners.addAll(scanners); @@ -234,7 +239,7 @@ private void addCurrentScanners(List scanners) { public StoreScanner(HStore store, ScanInfo scanInfo, Scan scan, NavigableSet columns, long readPt) throws IOException { this(store, scan, scanInfo, columns != null ? columns.size() : 0, readPt, scan.getCacheBlocks(), - ScanType.USER_SCAN, null); + ScanType.USER_SCAN); if (columns != null && scan.isRaw()) { throw new DoNotRetryIOException("Cannot specify any column for a raw scan"); } @@ -311,7 +316,7 @@ private StoreScanner(HStore store, ScanInfo scanInfo, List s // For mob compaction only as we do not have a Store instance when doing mob compaction. public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners) throws IOException { - this(null, SCAN_FOR_COMPACTION, scanInfo, 0, Long.MAX_VALUE, false, scanType, null); + this(null, SCAN_FOR_COMPACTION, scanInfo, 0, Long.MAX_VALUE, false, scanType); assert scanType != ScanType.USER_SCAN; this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, 0L, oldestUnexpiredTS, now, null, null, null); @@ -350,7 +355,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners, ScanType scanType) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, scan, scanInfo, columns != null ? columns.size() : 0, 0L, scan.getCacheBlocks(), - scanType, null); + scanType); if (scanType == ScanType.USER_SCAN) { this.matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); @@ -388,7 +393,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, maxVersions > 0 ? new Scan().readVersions(maxVersions) : SCAN_FOR_COMPACTION, - scanInfo, 0, 0L, false, scanType, null); + scanInfo, 0, 0L, false, scanType); this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, PrivateConstants.OLDEST_TIMESTAMP, oldestUnexpiredTS, now, null, null, null); seekAllScanner(scanInfo, scanners); From 0b2e9f1a73fb89ae338ff9bec055a0e369c0a42a Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <73588409+amamiya-yuuko-1225@users.noreply.github.com> Date: Wed, 7 Sep 2022 18:50:57 +0800 Subject: [PATCH 8/9] Update StoreScanner.java --- .../java/org/apache/hadoop/hbase/regionserver/StoreScanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 0349f8ee943b..1c837cc592d9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -382,7 +382,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, List scanners) throws IOException { // 0 is passed as readpoint because the test bypasses Store this(null, scan, scanInfo, columns != null ? columns.size() : 0, 0L, scan.getCacheBlocks(), - ScanType.USER_SCAN, null); + ScanType.USER_SCAN); this.matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); seekAllScanner(scanInfo, scanners); From f82a9f0c6f3f862555b8387d0f7e3dbfb56f6210 Mon Sep 17 00:00:00 2001 From: amamiya_yuuko <73588409+amamiya-yuuko-1225@users.noreply.github.com> Date: Wed, 7 Sep 2022 18:54:48 +0800 Subject: [PATCH 9/9] Update StoreScanner.java --- .../java/org/apache/hadoop/hbase/regionserver/StoreScanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 1c837cc592d9..88308326705d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -366,7 +366,7 @@ public StoreScanner(ScanInfo scanInfo, ScanType scanType, seekAllScanner(scanInfo, scanners); } - // Almost the same as above except for settting the start time of oldestUnexpiredTS + // Almost the same as below except for settting the start time of oldestUnexpiredTS StoreScanner(Scan scan, ScanInfo scanInfo, NavigableSet columns, List scanners, Long timeStampOrigin) throws IOException { // 0 is passed as readpoint because the test bypasses Store