From 5bbd8b7b5b03709fee84cd3d4c5ee795f8c4b70e Mon Sep 17 00:00:00 2001 From: MaribelleHGomez Date: Mon, 20 Mar 2023 12:35:47 -0500 Subject: [PATCH 1/2] add utilities file for dsv file operations and manipulation --- gen3/tools/file_utils.py | 430 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 gen3/tools/file_utils.py diff --git a/gen3/tools/file_utils.py b/gen3/tools/file_utils.py new file mode 100644 index 000000000..f21f6bb7a --- /dev/null +++ b/gen3/tools/file_utils.py @@ -0,0 +1,430 @@ +import os +import csv + +from typing import Union +from cdislogging import get_logger + +logging = get_logger("__name__") + + +def _get_directory_file_paths(directory: str): + """ + List all file paths in a directory. + Args: + directory(str): + Path of the directory containing the input manifests. All of the manifests contained in directory are assumed to be in a delimiter-separated values (DSV) format, and that there are no other non-DSV files in directory. + Returns: + List of files in directory + """ + + logging.info(f"Iterating over files in {directory} directory") + + files = [] + if os.path.exists(directory): + directory_list = os.listdir(directory) + else: + raise NotADirectoryError() + if directory_list: + for file in sorted(directory_list): + files.append(os.path.join(directory, file)) + + files.sort() + else: + raise RuntimeError(f"Directory {directory} is empty") + return files + + +def _delimiter(file_name: str): + """ + Determine delimiter of DSV file from filename, non-DSV files not accepted. + Args: + file_name(str): + Filename or file path + Returns: + DSV file delimiter + """ + + ext = file_name.lower()[-3:] + delimiter = "" + if ext == "tsv": + delimiter = "\t" + elif "csv": + delimiter = "," + elif "txt": + delimiter = "." + else: + raise ValueError( + "Please check you are providing file with an appropriate extension - TSV, CSV, or TXT. Filenames without an extension will error." + ) + + return delimiter + + +def get_headers(file_name: str): + """ + Get headers from DSV file + Args: + file_name(str): + Filename or file path + Returns: + List of headers + """ + + logging.info(f"Collecting headers from {file_name}...") + + delimiter = _delimiter(file_name) + headers = [] + if delimiter == "txt": + file = open(file_name, "r") + lines = file.readlines() + headers.append(lines[0]) + else: + with open(file_name) as process_file: + file_reader = csv.reader(process_file, delimiter=delimiter) + headers = next(file_reader) + + return headers + + +def file_to_list(file_name: str, has_headers=False): + """ + Collect records from DSV file + Args: + file_name(str): + Filename or file path + has_headers(bool): + True when headers present + Returns: + List of lists (rows) from input file + """ + + logging.info(f"Collecting records from {file_name}...") + + delimiter = _delimiter(file_name) + records = [] + if delimiter == "txt": + file = open(file_name, "r") + lines = file.readlines() + if has_headers: + lines.pop(0) + for line in lines: + records.append(line) + else: + with open(file_name) as process_file: + file_reader = csv.reader(process_file, delimiter=delimiter) + if has_headers: + next(file_reader) + for row in file_reader: + records.append(row) + + return records + + +def move_columns( + reordered_headers: list, + file_name: str, +): + """ + Move columns in list to match header order + Args: + reordered_headers(list): + "" + file_name(str): + Filename or file path + Returns: + List of lists (rows) from input file, with modified order based on given headers + """ + + original_headers = get_headers(file_name) + if reordered_headers != original_headers and sorted(reordered_headers) == sorted( + original_headers + ): + relational_indices = [] + for header in reordered_headers: + relational_indices.append(original_headers.index(header)) + + original_file_list = file_to_list(file_name, True) + reordered_file_list = [] + for row in original_file_list: + reordered_row = [] + for column in relational_indices: + reordered_row.append(row[column]) + reordered_file_list.append(reordered_row) + + elif reordered_headers != original_headers: + raise AssertionError("Columns do not have matching headers") + + +def write(records: list, output_file_name: str): + """ + Write records to DSV file + Args: + records(list): + List of rows, each row a list itself (list of lists) + output_file_name(str): + Filename or file path, expect file extension in the filename to determine file type + Returns: + None + """ + + logging.info(f"Writing to {output_file_name}...") + + delimiter = _delimiter(output_file_name) + with open(output_file_name, "w") as output: + output_writer = csv.writer(output, delimiter=delimiter) + for record in records: + output_writer.writerow(record) + + +def convert_type(file_name: str, new_type: str, has_headers=False): + """ + Convert file to different extension type + Args: + file_name(str): + Filename or file path + new_type(str): + Name of DSV type file is being converted to + has_headers(bool): + True when headers present + Returns: + None + """ + + logging.info(f"Converting {file_name} to {new_type}...") + + records = file_to_list(file_name, has_headers) + if has_headers: + headers = get_headers(file_name) + records.insert(0, headers) + new_file_name = f"{file_name[:-3:]}{new_type.lower()}" + write(records, new_file_name) + + +def merge_files(files: Union[str, list], output_file_name: str, has_headers=False): + """ + Merge multiple files into one file + Args: + files(str | list): + List of filepaths, or string directory path + output_file_name(str): + Name of file to be output + has_headers(bool): + True when headers present + Returns: + None + """ + + logging.info(f"Merging files: {files}...") + + if type(files) is str: + files = _get_directory_file_paths(files) + + all_lists = [] + for file in files: + all_lists.append(file_to_list(file, True)) + all_records = [item for list_ in all_lists for item in list_] + if has_headers: + headers = get_headers(files[0]) + all_records.insert(0, headers) + + write(all_records, output_file_name) + + +def split_by_headers( + file_name: str, +): + """ + Split records by header and write into separate files + Args: + file_name(str): + Filename or file path + Returns: + None + """ + logging.info(f"Splitting {file_name}...") + + records = file_to_list(file_name, has_headers=True) + headers = get_headers(file_name) + all_chunks = [] + for i in range(len(headers)): + chunk = [] + chunk.append(headers[i]) + for record in records: + chunk.append(record[i]) + all_chunks.append(chunk) + + for i in range(len(all_chunks)): + ext = file_name[-3::] + output_file_name = f"{file_name[:-4:]}_{headers[i]}.{ext}" + write(all_chunks[i], output_file_name) + + +def chunk( + file_name: str, + chunk_size: int, + has_headers=False, +): + """ + Chunk records into files of input size + Args: + file_name(str): + Filename or file path + chunk_size(int): + Size of desired segment + has_headers(bool): + True when headers present + Returns: + None + """ + logging.info(f"Chunking records into {chunk_size} size...") + + records = file_to_list(file_name, has_headers) + all_chunks = [] + chunk = [] + marker = 0 + for record in records: + if marker != chunk_size: + chunk.append(record) + marker += 1 + else: + all_chunks.append(chunk) + chunk = [] + chunk.append(record) + marker = 1 + if len(chunk) > 0: + all_chunks.append(chunk) + + if has_headers: + headers = get_headers(file_name) + for i in range(len(all_chunks)): + if has_headers: + all_chunks[i].insert(0, headers) + ext = file_name[-3::] + output_file_name = f"{file_name[:-4:]}_{i}.{ext}" + write(all_chunks[i], output_file_name) + + +def file_difference(file_one: str, file_two: str, has_headers=False, strict=False): + """ + Take difference between two files, files must have matching column headers + Args: + file_one(str): + Filename or file path + file_two(str): + Filename or file path + has_headers(bool): + True when headers present + strict(bool): + When true, will look for exact match; else, will match lower and uppercase strings and match array of different order + Returns: + List of lists, returns rows that contain at least one difference + """ + + logging.info(f"Taking differences between {file_one} & {file_two}...") + + list_one = file_to_list(file_one, has_headers) + + if has_headers: + file_one_headers = get_headers(file_one) + file_two_headers = get_headers(file_two) + if file_one_headers != file_two_headers and sorted(file_one_headers) == sorted( + file_two_headers + ): + list_two = move_columns(file_one_headers, file_two) + elif file_one_headers != file_two_headers: + raise AssertionError("Columns do not have matching headers") + else: + list_two = file_to_list(file_two, has_headers) + else: + list_two = file_to_list(file_two, has_headers) + + if not strict: + for row in list_one: + for column in row: + column.lower() + str(sorted(eval(column))) + for row in list_two: + for column in row: + column.lower() + str(sorted(eval(column))) + + return [row for row in list_one if row not in list_two] + [ + row for row in list_two if row not in list_one + ] + + +def file_intersection(file_one: str, file_two: str, has_headers=False, strict=False): + """ + Take intersection between two files, files must have matching column headers + Args: + file_one(str): + Filename or file path + file_two(str): + Filename or file path + has_headers(bool): + True when headers present + strict(bool): + When true, will look for exact match; else, will match lower and uppercase strings and match array of different order + Returns: + List of lists, returns rows that contain matches between all columns + """ + + logging.info(f"Taking intersection between {file_one} & {file_two}...") + + list_one = file_to_list(file_one, has_headers) + + if has_headers: + file_one_headers = get_headers(file_one) + file_two_headers = get_headers(file_two) + if file_one_headers != file_two_headers and sorted(file_one_headers) == sorted( + file_two_headers + ): + list_two = move_columns(file_one_headers, file_two) + elif file_one_headers != file_two_headers: + raise AssertionError("Columns do not have matching headers") + else: + list_two = file_to_list(file_two, has_headers) + else: + list_two = file_to_list(file_two, has_headers) + + if not strict: + for row in list_one: + for column in row: + column.lower() + str(sorted(eval(column))) + for row in list_two: + for column in row: + column.lower() + str(sorted(eval(column))) + + return [row for row in list_one if row in list_two] + + +# TODO +# def file_subset( +# file_name: str, +# criteria: str, +# column: str, +# has_headers=False +# ): +# """ +# Find all records that match a given criteria within a specified column +# Args: +# file_name(str): +# Filename or file path +# criteria(str): +# Expression to match (for example: >5, ==3, ==this_guid) +# column(str | int): +# If has_headers, str name of column to search for criteria; else, int index of column +# has_headers(bool): +# True when headers present +# Returns: +# List of lists, returns rows that match criteria +# """ +# logging.info(f"Finding subset of records that match {criteria} for {column}...") + +# records = file_to_list(file_name, has_headers) +# subset = [] +# # do stuff here + +# return subset From f1fa109d67f0d61367b650022d13bc7faf2fd2aa Mon Sep 17 00:00:00 2001 From: MaribelleHGomez Date: Mon, 20 Mar 2023 17:44:22 +0000 Subject: [PATCH 2/2] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 1552862 -> 1552862 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 99440 -> 99440 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 35609 -> 35609 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index d3c502ed245ef7ab72dff33e10fb4ffc0561789c..cf2e7c8a18dd2e311fd1996c37646f949489a8aa 100644 GIT binary patch delta 10676 zcma)Cd0>r4*Z17q{odSrpP5`)tg$78NURZC5P}dP_BBC-TN0H}EsE+EQlUwmz9?F{ z&{|4$RK2L3%0}#3`94}&lvdHAd}ro)=F-0JAKyPYzq8KFnVB;)_sRE{M}L2L^p@y) zMXTzC%$<^%moYvgFQd4?=*tll|&TT#^dwk~1N!eM&1=gp~IAPM{ zOq4uzH9bE)bJ~l=1?HTwzRYoX#Rc}<>0_r%%FUgWlZ}Bhrsqv8E-;A*NJZ%4CDX?i zSY~WHSg%RI+_Q#i)sniLGdH?$Yk9Koq~tCdTJ`jud&+I9>pS7+f)>R2%KTi&yo0_< zKNtCJ7vE_=*DS^5JLTtM2VC);@pFq;XZy}4`q{!8RlX8Gw|1S^SK;S&wYlmm^>cgo zkMouLxr63yMOz!SSI!pw(kLvB51_C(vcMf3-8?HZyJd5pu;{5T7B3O~=PHj>iKJ+{ zvZ92DFr~6+L_}@nbkR!@H6wVh|7I=ntXcChaGf_g8oG?PQ5Qx(uggMFmhiG8*od-{ zqCX>Im5QQo8jmkZXdG4aL1bdl_$FT0)86svDXQBfq$sFyXwi?6(Jp`IrsqwXoLjWG zaSI0@IB%^nM!sF?iBO7*Vk4i&K*37+w%JWc2s}zQdAF_Y`o9LqO);t4mUaKV3+&op z_HJAK#*7*oYO+?b!>D+RcU!{7adu6%gB1kBYhA70ZNth&8{k2rRo!-^;fY%vC?eieB@ld zmI^GH*?H3@W#$%-DvnI?@ul~Aaf(=Uobm<6DPLNg@D7?5sW zrA*>9^nGPJEjoA43!xhn3JHA-Va&5#X<>k-O%#=-F{M>36l5!2w(^=Xp|gNst7p(n z6C({jhiJk{2~s$mPYj3CG}tgLASGUdZ@v{kTpHZGBH$0L2W;0D0l%At6l^;K^dA29b-=53&2-VB zfIrYW)8W1ObNf(EUPdJo)j#uUv9K1r0q)!mI#bf@rv#yNTiwv8}ut)VsW(hJ7DQ5x*G z*4VeUQU+>TKoH#j5<{dqO=0Z&Z;Y`9rHpwF7z>D0e^KIBpEgEIFBX2+XbIMBSVyPn znng14K_;JBGGB$6N*$6}*~rw(DVYZ?Ooq^+sEuCO-`Ru%n}Z$b)W_7?Q1n`NmA#r` zazqO4PZOoKVuoo<10nmeNXn+IFx`saGB#ts=_{dSiPub@*A+6m&0M>-Kpi8@I~r-S z6my$cfd%)Y&kg}OzMJh$b6 zK=5U4iymZ49m__|b!|H-TODs1?G@o#_LZ{Esg}bo5v9f$%Q!8{;VDwPH`~%MNN69t zVp;AKXnmo@rJ>_1EPsRu*|{$)&{Lq-{<7o;a|8o}ta@Kkz1G&vVC6SPFI(Q#nr;C1 z0i((s-L2tXSb5s$g%^fd>9T%S8^lE5_RJn`HHCwDkH^d2oMjzjP)@OHbF39q6O}Ht z)^ece#!BmrdYb6w9&1>#25UNOpLPdFj_PF>Q*Gl7aQV7NW&P7^(a(ltxgJe_@3#H6TfprP9Sf^#!sWq^D4Ujo({&w#j2i6F&~Z;6 zOj4YqM!07C!Z1gz`t5j)_Vuq>t(vI#{CwvXo1g5Q@}mtaw{MhYAkKfG^p&!b*^Z(X zN(sx@>^NXkO2ocq@n1PS(UKW+)sgFGu6mup)pZSAt%GORmU+%FmeACB#VsA$b#S(b zlFYm;=MSAE^U_C7SGr{0GY2Xzoq=;SP!ClV6nG3fq;+IZQsAJ@l4<-#;D8#ES@J>P zRj*_Ub_7=Wd$prfhT2pXc(RtXS#~pUfhw8kCxL2^&cINcOKaOu`3UAmxYF5W>Kaa^ zI=_*wW^*H4uML8QJ8WJSI>$A1g@AC>?WQnqk~@rT3vu^0Kth^LWv&|TgLcjIT0{5o zx*9y%!##FUP`j2bTeWB%7u_Z%Hnvqv+vv$TSrM>wnz;r{nc&H1mruAWP2T@z)EM5k zC}D8Bg?kw5e9momyZ>PYpTFR#4YjVhoBpTfn!EEqh3v^S_kKm|+L@o-@w5B}?OMgP zX&cv?)t%spHUEb+eX^&kkp4t6ZNQdbJUNm zn%Bh_R4-A^!qT_YNa*{v8UigAstMqFORW#(tJGSsm9wM}*|*+SqM_R&wFgXFh-|Gd z)Y@=t5o%$vW^9H{chq3$U4XJ*^HH|3K+OPq0iqfSqzOHCL{KKxv_Bh7(-&8*g&ntE4UrDnwT0O=h=tRp%x>UuIei0AIK9A_ zmoqN6cusMo@9Z)oZ3_>Us@otnh^TD$+v44Ofs<*G@8 zkbR3-iauWLg3hg}7uvn6s%+*;^(O^B>r8U^OlHEhZbStPPuZ~5YPQ08!p^NxztSLd zZcZ%h(+|{Nb=9VI>LXpX`Iu^f?dw!G)CpAFaAh`j^tX@HeL@||NPgKX(7zUViVP06Zl8Mk**4wn zXb^P%6`MP9K;5mGFN?seu0EvxrhD^=VjGUAcQl@1YVYbQgu$%jxLT>Jww3a<@;-z8 zC$$F5-4Nh}o#Qa~KbQMQv-^blmu|#E1BUiM`JMpE?pLZMy2f)I_ho8(>VO9?qr2^l zx>s-G!ddlq-RSWZY@@tAwQ6m6FQ^BFMo;LLYXJ@@zY}1Exx0D3zf-@~9mT?V^u79n z&KEvZE$l&+dR^y5=hJH%ovij{wNw)_Y)wxI&k$gSJW$or7|0r3RZr+P=%{y&;RtT& z;lyAz4KX?Z^RKeRo6n|N3sn9h28z3-mg_D&vw%Vi+RWVm7O7dEE?$cr)nA`G#7({SEi#DfMCy#abS_r{f#HFGDF>DSOqp(7M<1;Ei zRdq=Dz#PbasZB6j+z4mB7NjOpGrLunV4V9 zg^g}W9_sdJgsVWtHr|Bv?n<6PX~xX zGt3Fb*G5%MOz-}V1Z#D>J6PiIsLX=Zcm4C8K&C33CongdT3GY01PeCj*@JE*+yGAk zXbwM6C%`-p)$3wV62ZENJYnzjE*>*p6}cW^6l|ViGeN-^99Ys2f^}DDu&jo| z?O8SlOB%*!n+sX=2)<0DXE;o~Vso+FQRIy7gyF(r?`p4`VL06hzI~Ys>)8Y0{Bg{~ zwhV$h^=S;$xEa>TlX@H2ut9G_c-Eb2cWH(2P8MGc!qbyY8bZ|yyNfA4a#8oZ^&D|) zo(%O?JqY$W9I6i6E%=1Ao7vE*X0S2DJn)UQ*G;2QAh^c?clk@6)vPC)1 z-hG9_f#VCDcGi43!RLr@=CuljY$lOx=SuRS z9&z_N5(B&6BX;)bDpID0ELg*5O|}>Vp0AxYcIg9xm6;0}I$@y@dW5Z+^JBiq1dqLm zg1v*N3DQqG16j;^f~UzpG}(h_AnW%j!2?TZaKOSR2#gmnD+@LfJR`Y=;dt??CX5iq zF!-D=aiN;G&97?wEf(D^7o35R5*TP^`=#FBUMh3(fcG{39!f1B>=klHuU#?tr7A+W@K#Hn`_M^hhShT*MxT^?@btp~_< zJ%%Wa^A7QPDB8f>q2-pqU|4+yEl(dIUG#qAh+I1CRYIyi8~K*aWr9wB1O~G$W#q_z z?;YBB*uN|M!+g$#3EzB@Px``C+zRpuV1`Sla4S5XBCoi~rwLU!PuSEm4Y@t0`HiBS>N$fj6?rkaI+zLuhNu_yF>2i zDn5@4w=K@A)&MRGHs?M+fJGGCLRiv;D#VQU(mprcA?)TuvOtFBJ2L+-K8M1HrOjBk zo?xa>ij{KHHasTf^DYb4l*^*_z)N!#&J%`bC1zg0KA{w^Y#ah!9W5|89FM8nk32Ru zr3S_QBa8;rqIka)M#=SQD4Se|R_LmJ;q+%6qDqH=(omM(kYc$IUgFZ562bMie^f)Y zR?;J=8Tw7Y4Vm4TdJH4^$BSgMc!4Hc#q%@SAl{zIb;QFn*(P3{$u-53GdV!KH)(&DdX&wRxA|9rzhr%Vx@yNXw=c)*sz zb4_VY@pQ&-Nus|?MCYG!Jo;0PKc1g*`~m%xeXCn9h~=|C=N4 z9S_5emWUiY<@AqUnr?bQVAkZEv19P1oi{BLzwJmn-#Ip<8C4AUtH-eBG+a3evzpT= ztCl(63E0$}uEO8u`^L~1r4n9_p@l)b?>sd@@8S#oy?^m+or3UKT2na#onq+>7w^re zV&ANaYSl7)C9p4+9#SeGzXhG4lrC#Yqm*)J+LA8l#}`1pNMXJhUcQ+9d@=d?V(#(P z(cVZ##nB!}*2M|kk8#2>svYf!!b$D0*%Pp_9ky@+ZfGPbp0+|RJs#UT0q@5PzsvE$ zGNL_gkHT^7vGXTjZF|}QNmY9hvtbA98Txgg?T{?*Alxr>5GElVMSFug3L$i)?HyvR z_vwd|v=-j)h(G=CXQI{DYPeD@5@*)kNqF?_B)V462@_KWUw5M2ktm(9i85%fk?9)Q z+!>oMgP%37K?03KZd3wogA1cL0sC7Cl?m9-Eqn)LFs|hQVD%}2(GXPrneF*dSIF=!IdPU^hpx#Zzs_f%4s;5gk3rf=AJm! zr=V?5Ovovi(Gv%D3O4q{D5v0dPi)}~MD@awjr?zK z93MI)YYxf6a#yktn)=YbUh$LJhJ83Y?e^m~$Sa;-T+jl3eU7xG`V4h)i+E`h}1bOhS287@5k94^B37(s`iaMcLf7pvlp5p)=m z9wP-=GZIZI;O0nL8;NriO-C|i6t-9aJ4WHkuYgAyNf?dmvH}*27Q(|Cag4$3R{^PG za7ruSUt>g+9b-gHQ#y{R0tTgHMk`=LI%c~9?xka2Dj+38SWd~nDXV~E8JOQvs5_Qs zAbDjh%|vo-ELxUA!Z@@ng)!skXe7JGp+hOu7?0bi98$*9wn$zdj}@#O_Kl~Vo#IE{ zfz9ye-VdYUnEr*d4Xxond6*+Oh z!~tTHXbW=iNM6Lm61;@#^{KXAHEc}NRzUd#S_=pCi$MmE%i-;2bBHfD!OKy7nk9R^ zyKC1GUvN&nhZ2GMzAsVbM;x)pR^I8^bH#aL*@zN>j`)#dvy|=JDpA>XiR?_GuXjjv zX{SVo{w>k+y%L?qPCAYVl%R;H%?V5*cw*7*Fdj$p7eHM;S;EVnvHs8l9Z7Hp(hVrew&)HYo_x||)$@;Bvuf6u(>zsX0zP&L1 z+Y94&#y0C~npQA5Z(do6>2GuLW<8CxsIYMQ?6MN8kWVYfFDtRno-=XAwAr(#6&940 z7^h5|o>x|4E}ZDgn^c5+?wq2jr9GNO*sU}2igG9C7UhS$x2vvxWu@ z(tFiB%nFU@cH4K_&$U^(#COuq#WXDSRrtA{vxfQ3`MKW9e7+h#m)&5W?~I=tJ!iMC z+|P|$w$@ka=bml6#dj*z&o1t;*;ni5R(0*_tMha3E(`Wm`MGV8SA5le?xThArMp^n zQ_hzD+$uDYPhe$Pbcs7QwtaqHLC5yIDzP)4E?Xt?=PHj>i9%|+bEPR!p-N5Zn5b~& zOzAUGO`>?#f3gC;N-CKJ9uK{Lfn$+EE``+vY``yC&PqN`hhC`T+0l(Wk~6l+#=GRuX7;ANi9PoD@zb7`?lHzI}!Y89QfJ zHV1)oy}`>?G*#ZPE2mkzHp;vpO}nqBa=N|HLf#~WmeyYgW#t7*UGyTpl}apm1x2%_ z<;^Y|R~DV=<2!FzS*F-?obvs}Dc@O~@;${V-%XtI72%XupHp6OPWe_?R+d-doK`St z`kcvme8t5OVgwQ1*MbKZkwsV$Qv?*zL?~Z1%Xm$>6`-7Cy*4N_3`louRHktn@`kd9 zf}>FNLdg3Hg_MDYQ0CdAbTGh$pFAqdW=f}cp=TQ|D^rp+Q(H5GZkifx_^}b39E$$a zQlxNZFGJr5&FRZr!{uNN9)H@PYij||7N zHP|vcATvpWFTET;P~$YP6Y`FMfYTY!utW=Yf+icVB8a^nRfCRU(mBHk2KEHlNl3doTbHr z7M+cH2JS>-rsf6jrb^+azQ!54Q{*t?LOp@qV~o2RYXdy9CwK*jVD+pX;F^PklI59mX<+6d+#~RI;TMI zt+co_^woOHZ;gbk_G1h56X=ECEyckc!LT5!o=cY3+PPh6?QJhx*V~$7fVOQEl{xxa zBfU^kpm^b_(N?;4u+;`}QK%?*!fJ{H^Ffc7y)@rC!Jw40%L}cQR1=l2wuU)SbY+9} zN;6G#^`JF0LxW8mwh#J(aiPo0&S%*s8{qd)x5|cO+h$QsS~JDA%&EcIvu)EfDco3K zqfmd$7Rth3v`vrK^qcnDZnV+hI~Q#2i3WeVWeaYk!C;p?KTLzAG4`@34R+7APt)PP zXY45vn$*4CzFr^q;5+s(RY;)`v&(TmwJQ!yn15IbbC22=*);!KC+!JZ1~BzIsV}Iv z&-Q5g+t=;a-2(2p?^xbY6Rrz(#MrbFoQrS_H)^nZOUG?}GU>JXgK?Bj``4gQtY5pwdlt0C>S^LIm2I7e?l)fr~ z+fN0a36nN!uLdqvB@_E7Pz}-<7-e&5V;jysfF)6`9Cm@ao}f})+)7uoMNzI7hAVYi zY9R||v<8$}Vj$w{(9MA(=1I+!I#>C3HyW+OcC| zY=?xnj&U72#!fHHkAl^+%#C5j6i+d`aN2#=F!e5(?KlxJR?(TDRTp{)-iS z^pqzY!Y;ep{-@@$JNd6d_UN+vh$54#{?VN@-*3{TQ+&txgm@M)#S?G-4{6SHPjAOx zrF``4{}f)v`m~zo>HJp>^Ud>gGQy{;@d=SQ)kD|z^62fPeu1Y|Z=qm~4|+zlf#rK` zUijjIheA#Z6$|}?C(ZyV**2Bk{K;bvg!9X6UPkYFTG%x>FhG5&p$2~lQRfi>fyJnL zfGus+@v5e8*i&Uen!fo^sds0qTXg;L$!e+|HKI^`$Etaqe@gWd;*RwDcIt4b{NS)ryu=_+KyK32ow%N3|~y{s9VVfzg=7zUJ} z?3ZGcZ7Wf8Vb*d)jZ;Wd$a)zSLELwF8VQF=&aUB+Q(6%dJj_Db$X8HiFF~1Wwu3Q<>k z#C65@UT39&_Fnf1NBrH=cZ4JHmRG?Me~I*c!x4Xt^!>;Yf06Xv;fTLV`ixx=@s~-T z%8}?>;T-Xv<%{8ncP(EcN4#(O`gRFH$lo`8Ls2-V#8^}~X?EGdvS{DgwMLo%_g1UB zAuEWe?7*w)5`{y;hLowhbm(2HUeO_aO;rq2+8e; zT42vs)eTJp6*pX5fQkP4wt85oL)gi8)vJF90vN;v4^tE;yZXLbrH4EFq1xO4>hmNF zQhMXzuQ!pd+MZgoL(PPW5~5`XmYwQOe0ScvQ~gchVgiSEsaN2B5G~}JZVIG?QGFus ze5@8KoF^>$6ZNPLA2Ri!4qf|HJ037@&}P3%bvXEdYSSQW$|DvC{z>&f`z{7MtV*O# z28Yx$nm?d_7KWAt5<*0c`=QgCPpU< zzo3?DLWZO1#o@IAtd$4qI~oI7t4r!>-3A?R>JaC0O%ErgwSA<~0k{$>OTGHXqHBRF zp2k3l*VGE#h1VERXo2f58!ZeM>z)E1CKGWUM6;Rzm-}Fi!f*03hmf$3Zt-m-;EQ+o z2ILTMZeF;4&FEks-BrKR^(e$MDfJ<+Ihk76w%_?e2q7;Jti6v_d+QNjXB4Vzxk7LY za6MZcKtc>~c(s}caXZum-Yp{`$xQk|a+y4-Z=1<1h4Z+7tppcRzy~&RM}vTe)WV!j za$D!czStB>n!uqcCJXE7B6ZTT326>9-NePlcu1`-{CODe&_RR*7(`^)&W*0uo7^zg zgW0|nL~v`1G3PWSc&G$?p%K9j#GzsSJyoO>RfN&?Dih;0_dHKG~T>!m2o|z?1R*GGTA9u<;$qecc|7?&|i~3@mJ6 z0>M=m{@Y`CeB|9!Rjm3#TOshos)C}K}>b}A{?9L_FsUF8bjhkV|Jfn|+ z0~_=)glEJ|yGv_{H}d&z5T2d_(h};|+g(iYk@LFewI_*N^JJ*E>OnB)NT@$lV4_`KCdYMq)CBAHiL1#9-5%S4g`F=Y*#3oo)(#T0wv3qB)itC> z_pAou(9KKkl0cCTw}G#gI_<3eIx;}l;-;{`$Q>k_?b|@!(jy+&O5)(a>%`7J*ho(4 zAxk#%Rg)vef#)-)jeYkf!OqNu44tr12tC5i%=r%AWP-J%{ zUo-{7X&@W?0l^2B(BOoHPY@Wt#;PpcM(`QQH4NvAFKEIDVH|^x_!bwc#k>8g*8jq$ zyYp*jAY=vxT3GoWlBPMaR(p9*5&pQdEYR)%Y02XD@jd=0VHdYCx%L~%CFA#OCjOK&a0qrB@V*`vo0t#R=&-VenXSUa@b z85j(ks?qY?3DQf?8)xLwVgHk);U6Pkv$;&r^S8iYw(}G@@!zvU8xQ+OrGJ`Fy0G9o z&hSOQxfq+3eDOEKcU5>4jw~T9SjIU*6%JKaSWR|m5ZrivSyC-ObV8N(H9y({PWguS z2M!^w9kKGW;AX47Blzwi^lScywAW3mE|491z{~@x6|e?od=+&yVxRq+p8yeL#x>Fk za`wAS?7$_y0sKO?^9OQJ54!j#g6~U@RU5AIhq@RK{V(aNZCh0|sH-O4kXeaT z8K#QsszoWn2`;?DAHX8Yio2wRp7o*o*cUXbZB5Xs_;uFf(+eYe9I8*`0@UpgMiYp`d&z7Up zbqmGg#d&76)0R5y>7>|EgcOIzJt5#~kCXx*5cW^?G=;%?Ji$z%6#L`yEF4;1=+?_fHoe*x3E>LGt}!q3eq3Cy2f zIB^2rtczyl;fEYKxbGAj*^Vj({FP&Ldm5>nhWYJjj8&_guL8EWryG@XFer}3DK+qH z99ZUsiaXG` zO4ZtqG)Ae0wjJrx!F&T0ixL)#=@pCBFBXemEY`kQEPb(9`C_r~#bU0-VqJ>GjElvh z7vq{g>`2GrJjQgQ9g(c;M3a%6>O}h?Y1EnaK{B>8jm0^x=uDH49P3O|kvO_g{K5tU zy3iD^>!Rtp(B8<|5^&-bkd%Oy6|g9Q_CoT{1e%3}B+>y$#wF4;BwG`O?)yYx8Pk>a zK;g8mIBW%M>xv^(z!i-W3@L5mV7m1RLL!5(d8kwV!9mzQSIrvfITBOiKqRq=yw{8NM{+~sl6zx@HL$ohE=vu3(OXys_7Rq&`v~s! zKA2ezeBB3gse#B;IuJ=gst|sdDon1Y3SF1JLbtRpW>y0S`r?w+fRcvGQUe3i1h+B` z%Uc7LX;`KjaHWeV1Ji~3tLe0Zat1z6$CS>1xgRcdIVAMMf|SGDemJpm*wzoDl*9FY zI6@`F^v9`G!W@kp?2ijy3FZMb9ZBW@p?ggu|2qKZhYlH%fpm~p{6zNt zVO*WANAMUFl`SbN=>WexNjg$}iG)ReRQkpq1ITttr*N!4j*f>+5RLbMjq2jjT3iEi4&~^{$2T!E};SbjFouQTq@NTQ=>$Uz3TRUM;+og@>^g{*6R4 zZFC+@FMb2B4A)GpP@m)s^xLl=f?J>#E)AhWkt7YJBao~fN{1o&ekfM47UHsLHj?MF z>2M@<*}}5jFdpABp9w zh5Z_FjuPD1QDV05jlv4o!u?Sq*s#&KyJ}(6Xqv6fYtbE?SL$z2KZ3S_xPJr&$x7Ek z>JxMf+HQVAc;0kIatv; zcs~cLT?e;wFqb;W%oUb1a&gJ(;LBXBZxuvLq`63*n@IDJT%L%QRgf|XEvsO{Bsw0+ zfl2641&t@;F{*~l$ut4UvdP%Ns^Rctn(P!m>JDp%KkvR33(w}!2xyl_?Wj%6!}C-P zv-4(d%^Kif znzjxqrqD2)(9ari<|9|d+T_zEV_WlQk8hI={|L0+AyI-;qB8=qvq$Ki7CrcX!1(sL z@sB`mRiXt%qWe^$0SzTO5W*3&{y@_T7Kw)3s~JiJY7!;UooI>jVav!;`TSud=6=_YLWfG(g5eyozl{p|rdhi9(oiu9B?5WINOW|pM196d z^qwCr9WQ0$Cve2p%n$OdTqM3$T%Cjxfp$!h=-4!gUYRbDy-=cYf0JlTkwh)$Ni=`H zL1ArUE&=!zfJydq`WOC|EH z(a{`VP;P?;_&;k7_PfK|$L0`UYJ%6H`V7nPdJlv*6<=(sUPp;QgWizn?Asi%=uX}_ z1&hS<#Ig+~0-f+9#||mmw@ac^dnB?mi9Xva(RceKI`$8V)*X_l_OL{&_+w9uw*Kfa z36nmTsM3$D$E7UlghcUQNc7d05;Z+3(G#aQVl%gRha6cXI>3r25N4ACdHj1RfdMWJti@5RVy1pt1I9kl=e delta 119 zcmey+!SJti@5RVy1ptDl9mD_t diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 7f3613f246dd20955844ec235cf6558226a1a51c..df3acba95692f3692b445605f9864e5d8943f31f 100644 GIT binary patch delta 115 zcmbO^jcMjIrVUA&{FcVXCPt@X66>An+r8vSkVP17i!vU(y!yd5brhj#SnXG N?u0J3*~;R65CFi`9oYZ? delta 115 zcmbO^jcMjIrVUA&{N_gHmd3^wdS)hu#)jsGn+r8vSkVP17i!vU(y!yd5brhj#SnXG N?u0J3*~;R65CE>x9eMx& diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 562f6a08b..f2cf73e58 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [0, 1, 3, 5, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "upload": [1, 2, 3, 8, 13], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": [0, 1], "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_id": [1, 2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [2, 9, 10], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [0, 3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [0, 3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "client": [0, 3], "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "exist": [1, 3, 5, 6, 9, 12], "": [1, 4, 8, 10, 11], "dispatch": 4, "job_nam": 4, "job_input": 4, "kwarg": [3, 4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "uniqu": [1, 5], "singl": [1, 5, 8, 10], "altern": [5, 11], "merg": 5, "AND": 5, "descript": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": [1, 5], "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "allow": [0, 6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "bucket": [1, 6], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [0, 7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "should": [0, 8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "obtain": [0, 10], "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": [0, 11], "header": 11, "space": [0, 11], "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "ha": [0, 11], "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": 11, "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1664913404": [], "4600763": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "7884498": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "upload_file_to_guid": [1, 2], "1664995746": [], "2094085": [], "6912842": [], "1665361686": [], "0196362": [], "4282863": [], "1665415592": [], "0841575": [], "369792": [], "download_singl": [1, 2], "1667237446": [], "310436": [], "461374": [], "1667237580": [], "6713352": [], "1667237581": [], "0283232": [], "1668452999": [], "8464384": [], "1668453000": [], "219631": [], "1668455015": [], "5573874": [], "9250507": [], "client_credenti": 0, "client_scop": 0, "client_id": 0, "client_secret": 0, "oidc": 0, "grant": 0, "openid": 0, "secret": 0, "1670019997": [], "6497164": [], "9551775": [], "1670272579": [], "3670187": [], "6539462": [], "never": 0, "hardcod": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "1671036553": [], "0874994": [], "3574216": [], "datacommon": 0, "1671044831": [], "8660681": [], "1671044832": [], "1868503": [], "1671140235": [], "0801866": [], "4101439": [], "1671140559": [], "279068": [], "6550183": [], "1673541949": [], "824122": [], "1673541950": [], "2653491": [], "1674497076": [], "8602023": [], "1674497077": [], "2165172": [], "python_subprocess_command": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "1676929147": [], "3869426": [], "758984": [], "1676929361": [], "016488": [], "398911": [], "1676930607": [], "061559": [], "4525151": [], "1677279338": [11, 12], "2703023": 11, "6413171": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_drs_object"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}}) \ No newline at end of file +Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [0, 1, 3, 5, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "upload": [1, 2, 3, 8, 13], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": [0, 1], "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_id": [1, 2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [2, 9, 10], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [0, 3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [0, 3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "client": [0, 3], "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "exist": [1, 3, 5, 6, 9, 12], "": [1, 4, 8, 10, 11], "dispatch": 4, "job_nam": 4, "job_input": 4, "kwarg": [3, 4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "uniqu": [1, 5], "singl": [1, 5, 8, 10], "altern": [5, 11], "merg": 5, "AND": 5, "descript": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": [1, 5], "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "allow": [0, 6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "bucket": [1, 6], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [0, 7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "should": [0, 8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "obtain": [0, 10], "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": [0, 11], "header": 11, "space": [0, 11], "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "ha": [0, 11], "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": 11, "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1664913404": [], "4600763": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "7884498": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "upload_file_to_guid": [1, 2], "1664995746": [], "2094085": [], "6912842": [], "1665361686": [], "0196362": [], "4282863": [], "1665415592": [], "0841575": [], "369792": [], "download_singl": [1, 2], "1667237446": [], "310436": [], "461374": [], "1667237580": [], "6713352": [], "1667237581": [], "0283232": [], "1668452999": [], "8464384": [], "1668453000": [], "219631": [], "1668455015": [], "5573874": [], "9250507": [], "client_credenti": 0, "client_scop": 0, "client_id": 0, "client_secret": 0, "oidc": 0, "grant": 0, "openid": 0, "secret": 0, "1670019997": [], "6497164": [], "9551775": [], "1670272579": [], "3670187": [], "6539462": [], "never": 0, "hardcod": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "1671036553": [], "0874994": [], "3574216": [], "datacommon": 0, "1671044831": [], "8660681": [], "1671044832": [], "1868503": [], "1671140235": [], "0801866": [], "4101439": [], "1671140559": [], "279068": [], "6550183": [], "1673541949": [], "824122": [], "1673541950": [], "2653491": [], "1674497076": [], "8602023": [], "1674497077": [], "2165172": [], "python_subprocess_command": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "1676929147": [], "3869426": [], "758984": [], "1676929361": [], "016488": [], "398911": [], "1676930607": [], "061559": [], "4525151": [], "1677279338": [], "2703023": [], "6413171": [], "1679334258": [11, 12], "6011407": 11, "9486785": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_drs_object"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index 57515b283..74d996659 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -381,7 +381,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1677279338.2703023.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1679334258.6011407.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index c1bbe9e00..e5cf31575 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1677279338.6413171.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1679334258.9486785.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters: