diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index be990f35..ac33bd14 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -648,10 +648,15 @@ async def upload_score_set_variant_data( assert_permission(user_data, item, Action.UPDATE) assert_permission(user_data, item, Action.SET_SCORES) - scores_df = csv_data_to_df(scores_file.file) - counts_df = None - if counts_file and counts_file.filename: - counts_df = csv_data_to_df(counts_file.file) + try: + scores_df = csv_data_to_df(scores_file.file) + counts_df = None + if counts_file and counts_file.filename: + counts_df = csv_data_to_df(counts_file.file) + # Handle non-utf8 file problem. + except UnicodeDecodeError as e: + raise HTTPException(status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values.") + if scores_file: # Although this is also updated within the variant creation job, update it here diff --git a/tests/routers/counts.csv b/tests/routers/counts.csv deleted file mode 100644 index 2242e957..00000000 --- a/tests/routers/counts.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,value,c_0.5,c_1,c_2,c_4,c_8,c_16,c_32,c_64,c_128,c_256,c_512,c_1024,c_total,ambler -p.Met1Ala,0.0,22.0,12,3,4,4,0,0,0,0,0,0,0,51.0,3.0 -p.Met1Cys,0.0,8.0,17,4,0,0,0,0,1,1,0,2,0,33.0,3.0 -p.Met1Asp,0.0,7.0,5,0,0,0,0,0,0,0,0,0,0,12.0,3.0 -p.Met1Glu,0.0,15.0,16,0,0,0,0,0,0,0,0,0,0,31.0,3.0 -p.Met1Phe,0.0,0.0,2,7,0,0,0,0,0,0,0,1,0,10.0,3.0 -p.Met1Gly,29.0,18.0,5,0,0,0,0,1,0,0,0,0,0,53.0,3.0 -p.Met1His,2.0,9.0,2,0,0,0,0,0,0,0,0,0,0,13.0,3.0 -p.Met1Ile,0.0,3.0,4,0,0,0,2,0,29,68,5,3,0,114.0,3.0 -p.Met1Lys,0.0,23.0,26,5,0,0,0,0,0,0,0,0,0,54.0,3.0 -p.Met1Leu,0.0,4.0,0,0,0,1,0,1,30,22,12,5,0,75.0,3.0 -p.Met1Met,0.0,0.0,0,0,0,0,0,0,0,0,0,0,0,0.0,3.0 \ No newline at end of file diff --git a/tests/routers/counts_with_different_variants.csv b/tests/routers/counts_with_different_variants.csv deleted file mode 100644 index 9a21eaf0..00000000 --- a/tests/routers/counts_with_different_variants.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,value,c_0.5,c_1,c_2,c_4,c_8,c_16,c_32,c_64,c_128,c_256,c_512,c_1024,c_total,ambler -p.Ser2Leu,0.0,22.0,12,3,4,4,0,0,0,0,0,0,0,51.0,3.0 -p.Ser2Met,0.0,8.0,17,4,0,0,0,0,1,1,0,2,0,33.0,3.0 -p.Met1Asp,0.0,7.0,5,0,0,0,0,0,0,0,0,0,0,12.0,3.0 -p.Met1Glu,0.0,15.0,16,0,0,0,0,0,0,0,0,0,0,31.0,3.0 -p.Met1Phe,0.0,0.0,2,7,0,0,0,0,0,0,0,1,0,10.0,3.0 -p.Met1Gly,29.0,18.0,5,0,0,0,0,1,0,0,0,0,0,53.0,3.0 -p.Met1His,2.0,9.0,2,0,0,0,0,0,0,0,0,0,0,13.0,3.0 -p.Met1Ile,0.0,3.0,4,0,0,0,2,0,29,68,5,3,0,114.0,3.0 -p.Met1Lys,0.0,23.0,26,5,0,0,0,0,0,0,0,0,0,54.0,3.0 -p.Met1Leu,0.0,4.0,0,0,0,1,0,1,30,22,12,5,0,75.0,3.0 -p.Met1Met,0.0,0.0,0,0,0,0,0,0,0,0,0,0,0,0.0,3.0 \ No newline at end of file diff --git a/tests/routers/counts_with_hgvs_nt_and_pro.csv b/tests/routers/counts_with_hgvs_nt_and_pro.csv deleted file mode 100644 index a89ca9d9..00000000 --- a/tests/routers/counts_with_hgvs_nt_and_pro.csv +++ /dev/null @@ -1,3 +0,0 @@ -hgvs_nt,hgvs_pro,value,c_0.5,c_1,c_2,c_4,c_8,c_16,c_32,c_64,c_128,c_256,c_512,c_1024,c_total,ambler -c.1A>T,p.Met1Leu,0.0,22.0,12,3,4,4,0,0,0,0,0,0,0,51.0,3.0 -c.3G>A,p.Met1Ile,0.0,8.0,17,4,0,0,0,0,1,1,0,2,0,33.0,3.0 \ No newline at end of file diff --git a/tests/routers/counts_with_score.csv b/tests/routers/counts_with_score.csv deleted file mode 100644 index ae4c8b82..00000000 --- a/tests/routers/counts_with_score.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score,c_0.5,c_1,c_2,c_4,c_8,c_16,c_32,c_64,c_128,c_256,c_512,c_1024,c_total,ambler -p.Met1Ala,0.0,22.0,12,3,4,4,0,0,0,0,0,0,0,51.0,3.0 -p.Met1Cys,0.0,8.0,17,4,0,0,0,0,1,1,0,2,0,33.0,3.0 -p.Met1Asp,0.0,7.0,5,0,0,0,0,0,0,0,0,0,0,12.0,3.0 -p.Met1Glu,0.0,15.0,16,0,0,0,0,0,0,0,0,0,0,31.0,3.0 -p.Met1Phe,0.0,0.0,2,7,0,0,0,0,0,0,0,1,0,10.0,3.0 -p.Met1Gly,29.0,18.0,5,0,0,0,0,1,0,0,0,0,0,53.0,3.0 -p.Met1His,2.0,9.0,2,0,0,0,0,0,0,0,0,0,0,13.0,3.0 -p.Met1Ile,0.0,3.0,4,0,0,0,2,0,29,68,5,3,0,114.0,3.0 -p.Met1Lys,0.0,23.0,26,5,0,0,0,0,0,0,0,0,0,54.0,3.0 -p.Met1Leu,0.0,4.0,0,0,0,1,0,1,30,22,12,5,0,75.0,3.0 -p.Met1Met,0.0,0.0,0,0,0,0,0,0,0,0,0,0,0,0.0,3.0 \ No newline at end of file diff --git a/tests/routers/data/scores_non_utf8_encoded.csv b/tests/routers/data/scores_non_utf8_encoded.csv new file mode 100755 index 00000000..d47850d7 Binary files /dev/null and b/tests/routers/data/scores_non_utf8_encoded.csv differ diff --git a/tests/routers/scores.csv b/tests/routers/scores.csv deleted file mode 100644 index edf2b364..00000000 --- a/tests/routers/scores.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score,err,ambler -p.Met1Ala,0.00201791436463821,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Asp,0.00221984947284538,0.0005666107418204864,3.0 -p.Met1Glu,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_hgvs_nt_not_match_pro.csv b/tests/routers/scores_hgvs_nt_not_match_pro.csv deleted file mode 100644 index b20d21e5..00000000 --- a/tests/routers/scores_hgvs_nt_not_match_pro.csv +++ /dev/null @@ -1,3 +0,0 @@ -hgvs_nt,hgvs_pro,score,err,ambler -c.1A>C,p.Met1Leu,0.00201791436463821,0.000294291714898736,3.0 -c.3G>A,p.Met1Ile,0.0030227552110695657,0.000521690342044468,3.0 diff --git a/tests/routers/scores_hgvs_pro_has_same_values.csv b/tests/routers/scores_hgvs_pro_has_same_values.csv deleted file mode 100644 index c35273e0..00000000 --- a/tests/routers/scores_hgvs_pro_has_same_values.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score,err,ambler -p.Met1Ala,0.00201791436463821,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Cys,0.00221984947284538,0.0005666107418204864,3.0 -p.Met1Glu,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_with_duplicate_columns.csv b/tests/routers/scores_with_duplicate_columns.csv deleted file mode 100644 index 1f7ad984..00000000 --- a/tests/routers/scores_with_duplicate_columns.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score,score,ambler -p.Met1Ala,0.00201791436463821,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Asp,0.00221984947284538,0.0005666107418204864,3.0 -p.Met1Glu,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_with_hgvs_nt_and_pro.csv b/tests/routers/scores_with_hgvs_nt_and_pro.csv deleted file mode 100644 index e49cac44..00000000 --- a/tests/routers/scores_with_hgvs_nt_and_pro.csv +++ /dev/null @@ -1,3 +0,0 @@ -hgvs_nt,hgvs_pro,score,err,ambler -c.1A>T,p.Met1Leu,0.00201791436463821,0.000294291714898736,3.0 -c.3G>A,p.Met1Ile,0.0030227552110695657,0.000521690342044468,3.0 diff --git a/tests/routers/scores_with_invalid_hgvs_nt_prefix.csv b/tests/routers/scores_with_invalid_hgvs_nt_prefix.csv deleted file mode 100644 index e518cedc..00000000 --- a/tests/routers/scores_with_invalid_hgvs_nt_prefix.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_nt,score,err,ambler -p.Met1Ala,0.00201791436463821,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Asp,0.00221984947284538,0.0005666107418204864,3.0 -c.2C>T,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_with_invalid_hgvs_pro_prefix.csv b/tests/routers/scores_with_invalid_hgvs_pro_prefix.csv deleted file mode 100644 index be2aaddf..00000000 --- a/tests/routers/scores_with_invalid_hgvs_pro_prefix.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score,err,ambler -p.Met1Ala,0.00201791436463821,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Asp,0.00221984947284538,0.0005666107418204864,3.0 -c.2C>T,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_with_nan_column_name.csv b/tests/routers/scores_with_nan_column_name.csv deleted file mode 100644 index 9d50cc85..00000000 --- a/tests/routers/scores_with_nan_column_name.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score, ,ambler -p.Met1Ala,0.00201791436463821,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Asp,0.00221984947284538,0.0005666107418204864,3.0 -p.Met1Glu,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_with_string.csv b/tests/routers/scores_with_string.csv deleted file mode 100644 index b4b56e37..00000000 --- a/tests/routers/scores_with_string.csv +++ /dev/null @@ -1,12 +0,0 @@ -hgvs_pro,score,err,ambler -p.Met1Ala, 0.0030227552110695611 ,0.000294291714898736,3.0 -p.Met1Cys,0.0030227552110695657,0.000521690342044468,3.0 -p.Met1Asp, abc,0.0005666107418204864,3.0 -p.Met1Glu,0.00237828918977636,0.00042051167207546845,3.0 -p.Met1Phe,0.005702414661461012,0.0015618903633585822,3.0 -p.Met1Gly,0.0012076958364638844,0.0001735283520462529,3.0 -p.Met1His,0.0016630080489589568,0.0004115364012988436,3.0 -p.Met1Ile,0.37803241907317636,0.04914421447951293,3.0 -p.Met1Lys,0.002639860787342766,0.0003765768113237344,3.0 -p.Met1Leu,0.3928550107047532,0.05107115139161792,3.0 -p.Met1Met,NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_without_hgvs_column.csv b/tests/routers/scores_without_hgvs_column.csv deleted file mode 100644 index b45245ef..00000000 --- a/tests/routers/scores_without_hgvs_column.csv +++ /dev/null @@ -1,5 +0,0 @@ -score,err,ambler -0.00201791436463821,0.000294291714898736,3.0 -0.0030227552110695657,0.000521690342044468,3.0 -0.00221984947284538,0.0005666107418204864,3.0 -NA,NA,3.0 \ No newline at end of file diff --git a/tests/routers/scores_without_score_column.csv b/tests/routers/scores_without_score_column.csv deleted file mode 100644 index 707dcaab..00000000 --- a/tests/routers/scores_without_score_column.csv +++ /dev/null @@ -1,3 +0,0 @@ -hgvs_pro,err -p.Met1Ala,0.1 -p.Met1Cys,0.2 diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 8a38952e..738143f7 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -2089,6 +2089,26 @@ def test_score_set_not_found_for_non_existent_score_set_when_adding_score_calibr assert "score_calibrations" not in response_data +######################################################################################################################## +# Score set upload files +######################################################################################################################## + +# Not sure why scores_non_utf8_encoded.csv file has a wrong encoding problem, but it's good for this test. +def test_upload_a_non_utf8_file(session, client, setup_router_db, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + scores_csv_path = data_files / "scores_non_utf8_encoded.csv" + with open(scores_csv_path, "rb") as scores_file: + response = client.post( + f"/api/v1/score-sets/{score_set['urn']}/variants/data", + files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")}, + ) + assert response.status_code == 400 + response_data = response.json() + assert "Error decoding file: 'utf-8' codec can't decode byte 0xdd in position 10: invalid continuation byte. " \ + "Ensure the file has correct values." in response_data["detail"] + + ######################################################################################################################## # Score set download files ########################################################################################################################