diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java
index 215120aa02bd..1e4321b4611d 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java
@@ -242,7 +242,9 @@ public void start() throws InterruptedException
                       break;
                     }
 
-                    if (isServerViewInitialized) {
+                    // lastFailure != 0L means exceptions happened before and there're some refresh work was not completed.
+                    // so that even ServerView is initialized, we can't let broker complete initialization.
+                    if (isServerViewInitialized && lastFailure == 0L) {
                       // Server view is initialized, but we don't need to do a refresh. Could happen if there are
                       // no segments in the system yet. Just mark us as initialized, then.
                       initialized.countDown();
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaTest.java
index 9b633e8a19e1..0863cd97fb13 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaTest.java
@@ -137,6 +137,7 @@ public static void tearDownClass() throws IOException
 
   private SpecificSegmentsQuerySegmentWalker walker = null;
   private DruidSchema schema = null;
+  private DruidSchema schema2 = null;
   private SegmentManager segmentManager;
   private Set<String> segmentDataSourceNames;
   private Set<String> joinableDataSourceNames;
@@ -269,6 +270,38 @@ protected DruidTable buildDruidTable(String dataSource)
       }
     };
 
+    schema2 = new DruidSchema(
+            CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
+            serverView,
+            segmentManager,
+            new MapJoinableFactory(ImmutableSet.of(globalTableJoinable), ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class)),
+            PLANNER_CONFIG_DEFAULT,
+            new NoopViewManager(),
+            new NoopEscalator()
+    )
+    {
+
+      boolean throwException = true;
+      @Override
+      protected DruidTable buildDruidTable(String dataSource)
+      {
+        DruidTable table = super.buildDruidTable(dataSource);
+        buildTableLatch.countDown();
+        return table;
+      }
+
+      @Override
+      Set<SegmentId> refreshSegments(final Set<SegmentId> segments) throws IOException
+      {
+        if (throwException) {
+          throwException = false;
+          throw new RuntimeException("Query[xxxx] url[http://xxxx:8083/druid/v2/] timed out.");
+        } else {
+          return super.refreshSegments(segments);
+        }
+      }
+    };
+
     schema.start();
     schema.awaitInitialization();
   }
@@ -289,6 +322,19 @@ public void testGetTableMap()
     Assert.assertEquals(ImmutableSet.of("foo", "foo2"), tableMap.keySet());
   }
 
+  @Test
+  public void testSchemaInit() throws InterruptedException
+  {
+    schema2.start();
+    schema2.awaitInitialization();
+    Map<String, Table> tableMap = schema2.getTableMap();
+    Assert.assertEquals(2, tableMap.size());
+    Assert.assertTrue(tableMap.containsKey("foo"));
+    Assert.assertTrue(tableMap.containsKey("foo2"));
+    schema2.stop();
+  }
+
+
   @Test
   public void testGetTableMapFoo()
   {