Skip to content

Ensure that DJORNL queries do not have additional properties #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jayrbolton merged 3 commits into from
Sep 1, 2020
Merged

Ensure that DJORNL queries do not have additional properties #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e32e906
Remove injected ws_ids param unless it is required by the query
ialarmedalien Sep 1, 2020
8f45ed4
Move addition of ws_ids to query to after query param validation
ialarmedalien Sep 1, 2020
884bacb
refactor run_query to prevent unnecessary call to the workspace
ialarmedalien Sep 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 20 additions & 11 deletions relation_engine_server/api_versions/api_v1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,41 +71,50 @@ def run_query():
- public stored queries (these have access controls within them based on params)
"""
json_body = parse_json.get_json_body() or {}
# Don't allow the user to set the special 'ws_ids' field
json_body['ws_ids'] = []
auth_token = auth.get_auth_header()
# Fetch any authorized workspace IDs using a KBase auth token, if present
ws_ids = auth.get_workspace_ids(auth_token)
# fetch number of documents to return
batch_size = int(flask.request.args.get('batch_size', 10000))
full_count = flask.request.args.get('full_count', False)

Copy link
Copy Markdown
Contributor

@jayrbolton jayrbolton Sep 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don't have to revert, but i prefer not having all these additional newlines in the function body

Copy link
Copy Markdown
Collaborator Author

@ialarmedalien ialarmedalien Sep 1, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find adding newlines aids readability -- otherwise it's just a bunch of lines with the same indentation and there's added cognitive load in understanding the code before you can edit it. 😖

if 'query' in json_body:
# Run an adhoc query for a sysadmin
auth.require_auth_token(roles=['RE_ADMIN'])
query_text = _preprocess_stored_query(json_body['query'], json_body)
del json_body['query']
json_body['ws_ids'] = ws_ids
if 'ws_ids' in query_text:
# Fetch any authorized workspace IDs using a KBase auth token, if present
auth_token = auth.get_auth_header()
json_body['ws_ids'] = auth.get_workspace_ids(auth_token)

resp_body = arango_client.run_query(query_text=query_text,
bind_vars=json_body,
batch_size=batch_size,
full_count=full_count)
return flask.jsonify(resp_body)

if 'stored_query' in flask.request.args or 'view' in flask.request.args:
# Run a query from a query name
# Note: we are maintaining backwards compatibility here with the "view" arg.
# "stored_query" is the more accurate name
query_name = flask.request.args.get('stored_query') or flask.request.args.get('view')
stored_query = spec_loader.get_stored_query(query_name)
stored_query_source = _preprocess_stored_query(stored_query['query'], stored_query)

if 'params' in stored_query:
# Validate the user params for the query
run_validator(schema=stored_query['params'], data=json_body)
json_body['ws_ids'] = ws_ids
stored_query_path = spec_loader.get_stored_query(query_name, path_only=True)
run_validator(schema_file=stored_query_path, data=json_body, validate_at='/params')
Comment on lines +103 to +104
Copy link
Copy Markdown
Collaborator Author

@ialarmedalien ialarmedalien Sep 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use the path, so that relative references in the files work.


stored_query_source = _preprocess_stored_query(stored_query['query'], stored_query)
if 'ws_ids' in stored_query_source:
# Fetch any authorized workspace IDs using a KBase auth token, if present
auth_token = auth.get_auth_header()
json_body['ws_ids'] = auth.get_workspace_ids(auth_token)

resp_body = arango_client.run_query(query_text=stored_query_source,
bind_vars=json_body,
batch_size=batch_size,
full_count=full_count)
return flask.jsonify(resp_body)

if 'cursor_id' in flask.request.args:
# Run a query from a cursor ID
cursor_id = flask.request.args['cursor_id']
Expand Down Expand Up @@ -167,9 +176,9 @@ def show_config():

def _preprocess_stored_query(query_text, config):
"""Inject some default code into each stored query."""
ws_id_text = " LET ws_ids = @ws_ids " if 'ws_ids' in query_text else ""
return (
config.get('query_prefix', '') +
" LET ws_ids = @ws_ids " +
# " LET maxint = 9007199254740991 " +
ws_id_text +
query_text
)
7 changes: 7 additions & 0 deletions spec/datasets/distance.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
name: distance
type: integer
title: Traversal Distance
description: How many hops to find neighbors and neighbors-of-neighbors
default: 1
minimum: 0
maximum: 100
2 changes: 1 addition & 1 deletion spec/datasets/djornl/definitions.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ definitions:
_key:
type: string
title: Key
examples: ["AT1G01010"]
examples: ["AT1G01010", "As2"]
clusters:
type: array
title: Clusters
Expand Down
12 changes: 11 additions & 1 deletion spec/stored_queries/djornl/djornl_fetch_all.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,24 @@ name: djornl_fetch_all
description: Fetch all node and edge data from the djornl subgraph
params:
type: object
# additionalProperties: false
additionalProperties: false
properties:
edge_types:
type: array
title: Edge types
description: Permitted edge types
items:
$ref: "../../datasets/djornl/edge_type.yaml"
default: []
examples: [['AraNetv2-HT_high-throughput-ppi', 'AraNetv2-LC_lit-curated-ppi'],['AraGWAS-Phenotype_Associations']]
query: |
LET nodes = (
FOR v IN djornl_node
RETURN v
)
LET edges = (
FOR e IN djornl_edge
FILTER length(@edge_types) > 0 && e.edge_type IN @edge_types || length(@edge_types) == 0
RETURN e
)
RETURN {nodes, edges}
14 changes: 4 additions & 10 deletions spec/stored_queries/djornl/djornl_fetch_clusters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,19 @@ name: djornl_fetch_clusters
description: Fetch all nodes that are members of the specified cluster(s), and the edges and nodes within the specified distance (number of hops) of those nodes.
params:
type: object
additionalProperties: false
required: [cluster_ids]
properties:
cluster_ids:
type: array
title: Cluster IDs
description: Cluster IDs, in the form "clustering_system_name:cluster_id"
items:
type: string
format: regex
pattern: ^\w+:\d+$
examples: [['markov_i2:5', 'markov_i6:2'],['markov_i6:1']]
$ref: "../../datasets/djornl/definitions.yaml#definitions/cluster_id"
minItems: 1
examples: [['markov_i2:5', 'markov_i6:2'],['markov_i6:1']]
distance:
type: integer
title: Traversal Distance
description: How many hops to find neighbors and neighbors-of-neighbors
default: 1
minimum: 0
maximum: 100
$ref: "../../datasets/distance.yaml"
query: |
LET node_ids = (
FOR n IN djornl_node
Expand Down
14 changes: 5 additions & 9 deletions spec/stored_queries/djornl/djornl_fetch_genes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,18 @@ name: djornl_fetch_genes
description: Fetch a gene or list of genes by key, and the edges and nodes within the specified distance (number of hops) of those genes.
params:
type: object
additionalProperties: false
required: [keys]
properties:
distance:
type: integer
title: Traversal Distance
description: How many hops to find neighbors and neighbors-of-neighbors
default: 1
minimum: 0
maximum: 100
keys:
type: array
items:
type: string
title: Gene Keys
items:
$ref: "../../datasets/djornl/definitions.yaml#definitions/djornl_node/_key"
minItems: 1
examples: [["AT1G01010"],["AT1G01020","AT1G01070"]]
distance:
$ref: "../../datasets/distance.yaml"
query: |
LET node_ids = (
FOR n IN djornl_node
Expand Down
14 changes: 5 additions & 9 deletions spec/stored_queries/djornl/djornl_fetch_phenotypes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,18 @@ name: djornl_fetch_phenotypes
description: Fetch a phenotype or list of phenotypes by key, and the edges and nodes within the specified distance (number of hops) of those phenotype nodes.
params:
type: object
additionalProperties: false
required: [keys]
properties:
distance:
type: integer
title: Traversal Distance
description: How many hops to find neighbors and neighbors-of-neighbors
default: 1
minimum: 0
maximum: 100
keys:
type: array
items:
type: string
title: Phenotype Keys
items:
$ref: "../../datasets/djornl/definitions.yaml#definitions/djornl_node/_key"
minItems: 1
examples: [["As2"],["As2", "Na23"]]
distance:
$ref: "../../datasets/distance.yaml"
query: |
LET node_ids = (
FOR n IN djornl_node
Expand Down
10 changes: 3 additions & 7 deletions spec/stored_queries/djornl/djornl_search_nodes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,15 @@ name: djornl_search_nodes
description: Search for nodes using a simple fuzzy search on node metadata; return the matching nodes, and the edges and nodes within the specified distance (number of hops) of those nodes.
params:
type: object
additionalProperties: false
required: [search_text]
properties:
distance:
type: integer
title: Traversal Distance
description: How many hops to find neighbors and neighbors-of-neighbors
default: 1
minimum: 0
maximum: 100
search_text:
type: string
title: Search text
examples: ['GO:0005515', 'organelle machinery']
distance:
$ref: "../../datasets/distance.yaml"
query: |
LET node_ids = (
FOR g IN djornl_node_view
Expand Down
124 changes: 121 additions & 3 deletions spec/test/djornl/results.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,109 @@
{
"params": {"musical": "Mary Poppins"},
"error": {
"message": "ArangoDB server error.",
"arango_message": "AQL: bind parameter 'musical' was not declared in the query (while parsing)"
"failed_validator": "additionalProperties",
"message": "Additional properties are not allowed ('musical' was unexpected)",
"path": [],
"value": {"musical": "Mary Poppins"}
}
},
{
"params": {"edge_types": ["straight", "curved"]},
"error": {
"failed_validator": "oneOf",
"message": "'straight' is not valid under any of the given schemas",
"path": ["edge_types", 0],
"value": "straight"
}
},
{
"params": {"edge_types": []},
"results": {
"nodes": [
"As2",
"As75",
"AT1G01010",
"AT1G01020",
"AT1G01030",
"AT1G01040",
"AT1G01050",
"AT1G01060",
"AT1G01070",
"AT1G01080",
"AT1G01090",
"AT1G01100",
"Na23",
"SDV"
],
"edges": [
"As2__AT1G01020__AraGWAS-Phenotype_Associations__8.4",
"As2__AT1G01040__AraGWAS-Phenotype_Associations__5.4",
"As75__AT1G01020__AraGWAS-Phenotype_Associations__39.9",
"AT1G01010__AT1G01020__AraNetv2-HT_high-throughput-ppi__2.3",
"AT1G01010__AT1G01030__AraNetv2-HT_high-throughput-ppi__2.4",
"AT1G01010__AT1G01040__AraNetv2-DC_domain-co-occurrence__2.5",
"AT1G01010__AT1G01040__AraNetv2-LC_lit-curated-ppi__170.5",
"AT1G01030__AT1G01050__AraNetv2-CX_pairwise-gene-coexpression__2.6",
"AT1G01050__AT1G01060__AraNetv2-LC_lit-curated-ppi__2.7",
"AT1G01080__AT1G01090__AraNetv2-LC_lit-curated-ppi__2.8"
]
}
},
{
"params": {"edge_types": ["AraGWAS-Phenotype_Associations"]},
"results": {
"nodes": [
"As2",
"As75",
"AT1G01010",
"AT1G01020",
"AT1G01030",
"AT1G01040",
"AT1G01050",
"AT1G01060",
"AT1G01070",
"AT1G01080",
"AT1G01090",
"AT1G01100",
"Na23",
"SDV"
],
"edges": [
"As2__AT1G01020__AraGWAS-Phenotype_Associations__8.4",
"As2__AT1G01040__AraGWAS-Phenotype_Associations__5.4",
"As75__AT1G01020__AraGWAS-Phenotype_Associations__39.9"
]
}
},
{
"params": {"edge_types": ["AraGWAS-Phenotype_Associations", "AraNetv2-HT_high-throughput-ppi", "AraNetv2-LC_lit-curated-ppi"]},
"results": {
"nodes": [
"As2",
"As75",
"AT1G01010",
"AT1G01020",
"AT1G01030",
"AT1G01040",
"AT1G01050",
"AT1G01060",
"AT1G01070",
"AT1G01080",
"AT1G01090",
"AT1G01100",
"Na23",
"SDV"
],
"edges": [
"As2__AT1G01020__AraGWAS-Phenotype_Associations__8.4",
"As2__AT1G01040__AraGWAS-Phenotype_Associations__5.4",
"As75__AT1G01020__AraGWAS-Phenotype_Associations__39.9",
"AT1G01010__AT1G01020__AraNetv2-HT_high-throughput-ppi__2.3",
"AT1G01010__AT1G01030__AraNetv2-HT_high-throughput-ppi__2.4",
"AT1G01010__AT1G01040__AraNetv2-LC_lit-curated-ppi__170.5",
"AT1G01050__AT1G01060__AraNetv2-LC_lit-curated-ppi__2.7",
"AT1G01080__AT1G01090__AraNetv2-LC_lit-curated-ppi__2.8"
]
}
}
],
Expand All @@ -116,7 +217,7 @@
"failed_validator": "required",
"message": "'keys' is a required property",
"path": [],
"value": {"ws_ids": []}
"value": {}
}
},
{
Expand Down Expand Up @@ -181,6 +282,23 @@
]
}
},
{
"params": { "keys": ["AT1G01010"], "distance": 5 },
"results": {
"nodes": ["As2", "As75", "AT1G01010", "AT1G01020", "AT1G01030", "AT1G01040", "AT1G01050", "AT1G01060"],
"edges": [
"As2__AT1G01020__AraGWAS-Phenotype_Associations__8.4",
"As2__AT1G01040__AraGWAS-Phenotype_Associations__5.4",
"As75__AT1G01020__AraGWAS-Phenotype_Associations__39.9",
"AT1G01010__AT1G01020__AraNetv2-HT_high-throughput-ppi__2.3",
"AT1G01010__AT1G01030__AraNetv2-HT_high-throughput-ppi__2.4",
"AT1G01010__AT1G01040__AraNetv2-DC_domain-co-occurrence__2.5",
"AT1G01010__AT1G01040__AraNetv2-LC_lit-curated-ppi__170.5",
"AT1G01030__AT1G01050__AraNetv2-CX_pairwise-gene-coexpression__2.6",
"AT1G01050__AT1G01060__AraNetv2-LC_lit-curated-ppi__2.7"
]
}
},
{
"params": {"keys": ["AT1G01020", "AT1G01070"], "distance": 0 },
"results": {
Expand Down