Add sys.queries table.

docs/configuration/index.md

@@ -1892,6 +1892,7 @@ The Druid SQL server is configured through the following properties on the Broke
 |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true|
 |`druid.sql.planner.maxNumericInFilters`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this property to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100.  Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. If this value is disabled, `maxNumericInFilters` set through query context is ignored.|`-1` (disabled)|
 |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql-aggregations.md). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.<br /><br />Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|`APPROX_COUNT_DISTINCT_BUILTIN`|
+|`druid.sql.planner.enableSysQueriesTable`|**Experimental.** Whether to enable the [`sys.queries` table](../querying/sql-metadata-tables.md#queries-table), which provides information about currently running and recently completed SQL queries. Currently only queries from the Dart (MSQ) engine are shown.|false|
 
 :::info
  Previous versions of Druid had properties named `druid.sql.planner.maxQueryCount` and `druid.sql.planner.maxSemiJoinRowsInMemory`.

docs/querying/sql-metadata-tables.md

@@ -335,4 +335,37 @@ For example, to retrieve properties for a specific server, use the query
 
 ```sql
 SELECT * FROM sys.server_properties WHERE server='192.168.1.1:8081'
-```
+```
+
+### QUERIES table
+
+:::info
+ The `sys.queries` table is an experimental feature. You must enable it by setting the runtime property
+ `druid.sql.planner.enableSysQueriesTable=true` on Broker processes. The main reason this table is experimental
+  is that it only shows queries from the [Dart](dart.md) engine, which is also experimental.
+:::
+
+The queries table provides information about currently running and recently completed SQL queries.
+
+|Column|Type|Notes|
+|------|-----|-----|
+|id|VARCHAR|Execution ID for the query. For Dart queries, this is the `dartQueryId`.|
+|engine|VARCHAR|SQL engine that executed the query, e.g., `msq-dart`|
+|state|VARCHAR|Query status: `ACCEPTED`, `RUNNING`, `SUCCESS`, `FAILED`, or `CANCELED`|
+|info|VARCHAR|JSON-serialized query information including `sqlQueryId`, `sql`, `identity`, `startTime`, and other engine-specific details|
+
+For example, to retrieve all recently completed Dart queries:
+
+```sql
+SELECT *
+FROM sys.queries
+WHERE
+  engine = 'msq-dart'
+  AND state IN ('SUCCESS', 'FAILED', 'CANCELED')
+```
+
+:::info
+ The retention of completed query information is controlled by Dart controller configuration.
+ See `druid.msq.dart.controller.maxRetainedReportCount` and `druid.msq.dart.controller.maxRetainedReportDuration`
+ for details on how long completed queries are retained.
+:::

embedded-tests/pom.xml

@@ -209,7 +209,10 @@
         </exclusion>
       </exclusions>
     </dependency>
-
+    <dependency>
+      <groupId>com.opencsv</groupId>
+      <artifactId>opencsv</artifactId>
+    </dependency>
 
     <!-- Test dependencies -->
     <dependency>

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java

@@ -1006,6 +1006,11 @@ protected void setExpectedSystemSchemaObjects(String dataSource, String taskId)
     );
   }
 
+  protected EmbeddedCoordinator getCoordinator()
+  {
+    return coordinator;
+  }
+
   protected String getCoordinatorUrl()
   {
     return getServerUrl(coordinator);

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/BasicAuthConfigurationTest.java

@@ -196,18 +196,14 @@ private void createUserAndRoleWithPermissions(
       List<ResourceAction> permissions
   )
   {
-    // Setup authentication by creating user and password
-    postAsAdmin(null, "/authentication/db/basic/users/%s", user);
-
-    final BasicAuthenticatorCredentialUpdate credentials
-        = new BasicAuthenticatorCredentialUpdate(password, 5000);
-    postAsAdmin(credentials, "/authentication/db/basic/users/%s/credentials", user);
-
-    // Setup authorization by assigning a role to the user
-    postAsAdmin(null, "/authorization/db/basic/users/%s", user);
-    postAsAdmin(null, "/authorization/db/basic/roles/%s", role);
-    postAsAdmin(null, "/authorization/db/basic/users/%s/roles/%s", user, role);
-    postAsAdmin(permissions, "/authorization/db/basic/roles/%s/permissions", role);
+    EmbeddedBasicAuthResource.createUserWithPermissions(
+        getHttpClient(User.ADMIN),
+        getCoordinator(),
+        user,
+        password,
+        role,
+        permissions
+    );
   }
 
   private void postAsAdmin(

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/EmbeddedBasicAuthResource.java

@@ -20,15 +20,25 @@
 package org.apache.druid.testing.embedded.auth;
 
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.http.client.HttpClient;
 import org.apache.druid.security.basic.BasicSecurityDruidModule;
+import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate;
+import org.apache.druid.server.DruidNode;
+import org.apache.druid.server.security.ResourceAction;
 import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedDruidServer;
 import org.apache.druid.testing.embedded.EmbeddedResource;
+import org.jboss.netty.handler.codec.http.HttpMethod;
+import org.jboss.netty.handler.codec.http.HttpResponseStatus;
+
+import java.util.List;
 
 /**
  * Resource to enable the basic auth extension in embedded tests.
  */
 public class EmbeddedBasicAuthResource implements EmbeddedResource
 {
+  public static final String ADMIN_USER = "admin";
   public static final String ADMIN_PASSWORD = "priest";
   public static final String SYSTEM_PASSWORD = "warlock";
   public static final String SYSTEM_USER = "druid_system";
@@ -65,19 +75,100 @@ public void stop()
   {
     // Do nothing
   }
-  
+
   private String authenticatorProp(String name)
   {
     return StringUtils.format("druid.auth.authenticator.%s.%s", AUTHENTICATOR_NAME, name);
   }
-  
+
   private String authorizerProp(String name)
   {
     return StringUtils.format("druid.auth.authorizer.%s.%s", AUTHORIZER_NAME, name);
   }
-  
+
   private String escalatorProp(String name)
   {
     return StringUtils.format("druid.escalator.%s", name);
   }
+
+  /**
+   * Creates a user with specified permissions using the basic auth security API.
+   *
+   * @param adminClient  HTTP client authenticated as admin
+   * @param coordinator  the coordinator server to make API calls against
+   * @param username     the username to create
+   * @param password     the password for the user
+   * @param roleName     the role name to create and assign
+   * @param permissions  the permissions to grant to the role
+   */
+  public static void createUserWithPermissions(
+      HttpClient adminClient,
+      EmbeddedDruidServer<?> coordinator,
+      String username,
+      String password,
+      String roleName,
+      List<ResourceAction> permissions
+  )
+  {
+    final DruidNode coordinatorDruidNode = coordinator.bindings().selfNode();
+    final String baseUrl = StringUtils.format(
+        "%s://%s/druid-ext/basic-security",
+        coordinatorDruidNode.getServiceScheme(),
+        coordinatorDruidNode.getHostAndPortToUse()
+    );
+
+    // Create user in authentication DB
+    HttpUtil.makeRequest(
+        adminClient,
+        HttpMethod.POST,
+        StringUtils.format("%s/authentication/db/basic/users/%s", baseUrl, username),
+        null,
+        HttpResponseStatus.OK
+    );
+
+    // Set password
+    HttpUtil.makeRequest(
+        adminClient,
+        HttpMethod.POST,
+        StringUtils.format("%s/authentication/db/basic/users/%s/credentials", baseUrl, username),
+        new BasicAuthenticatorCredentialUpdate(password, 5000),
+        HttpResponseStatus.OK
+    );
+
+    // Create user in authorization DB
+    HttpUtil.makeRequest(
+        adminClient,
+        HttpMethod.POST,
+        StringUtils.format("%s/authorization/db/basic/users/%s", baseUrl, username),
+        null,
+        HttpResponseStatus.OK
+    );
+
+    // Create role
+    HttpUtil.makeRequest(
+        adminClient,
+        HttpMethod.POST,
+        StringUtils.format("%s/authorization/db/basic/roles/%s", baseUrl, roleName),
+        null,
+        HttpResponseStatus.OK
+    );
+
+    // Assign role to user
+    HttpUtil.makeRequest(
+        adminClient,
+        HttpMethod.POST,
+        StringUtils.format("%s/authorization/db/basic/users/%s/roles/%s", baseUrl, username, roleName),
+        null,
+        HttpResponseStatus.OK
+    );
+
+    // Grant permissions
+    HttpUtil.makeRequest(
+        adminClient,
+        HttpMethod.POST,
+        StringUtils.format("%s/authorization/db/basic/roles/%s/permissions", baseUrl, roleName),
+        permissions,
+        HttpResponseStatus.OK
+    );
+  }
 }