From 4774f51ac41ab11e5ebc9ab94cc35181d83e6549 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:38:18 +0100 Subject: [PATCH 01/19] bashlib: implement Processing Worker args --- ocrd/ocrd/lib.bash | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/lib.bash b/ocrd/ocrd/lib.bash index a34263637c..11417c5706 100644 --- a/ocrd/ocrd/lib.bash +++ b/ocrd/ocrd/lib.bash @@ -143,34 +143,45 @@ ocrd__parse_argv () { --profile) ocrd__argv[profile]=true ;; --profile-file) ocrd__argv[profile_file]=$(realpath "$2") ; shift ;; -V|--version) ocrd ocrd-tool "$OCRD_TOOL_JSON" version; exit ;; + --queue) ocrd__worker_queue="$2" ; shift ;; + --database) ocrd__worker_database="$2" ; shift ;; *) ocrd__raise "Unknown option '$1'" ;; esac shift done - if [[ ! -e "${ocrd__argv[mets_file]}" ]];then + if [ -v ocrd__worker_queue -a -v ocrd__worker_database ]; then + ocrd processing-worker $OCRD_TOOL_NAME --queue "${ocrd__worker_queue}" --database "${ocrd__worker_database}" + exit + elif [ -v ocrd__worker_queue ]; then + ocrd__raise "Processing Worker also requires a --database argument" + elif [ -v ocrd__worker_database ]; then + ocrd__raise "Processing Worker also requires a --queue argument" + fi + + if [[ ! -e "${ocrd__argv[mets_file]}" ]]; then ocrd__raise "METS file '${ocrd__argv[mets_file]}' not found" fi - if [[ ! -d "${ocrd__argv[working_dir]:=$(dirname "${ocrd__argv[mets_file]}")}" ]];then + if [[ ! -d "${ocrd__argv[working_dir]:=$(dirname "${ocrd__argv[mets_file]}")}" ]]; then ocrd__raise "workdir '${ocrd__argv[working_dir]}' not a directory. Use -w/--working-dir to set correctly" fi - if [[ ! "${ocrd__argv[log_level]:=INFO}" =~ OFF|ERROR|WARN|INFO|DEBUG|TRACE ]];then + if [[ ! "${ocrd__argv[log_level]:=INFO}" =~ OFF|ERROR|WARN|INFO|DEBUG|TRACE ]]; then ocrd__raise "log level '${ocrd__argv[log_level]}' is invalid" fi - if [[ -z "${ocrd__argv[input_file_grp]:=}" ]];then + if [[ -z "${ocrd__argv[input_file_grp]:=}" ]]; then ocrd__raise "Provide --input-file-grp/-I explicitly!" fi - if [[ -z "${ocrd__argv[output_file_grp]:=}" ]];then + if [[ -z "${ocrd__argv[output_file_grp]:=}" ]]; then ocrd__raise "Provide --output-file-grp/-O explicitly!" fi # enable profiling (to be extended/acted upon by caller) - if [[ ${ocrd__argv[profile]} = true ]];then - if [[ -n "${ocrd__argv[profile_file]}" ]];then + if [[ ${ocrd__argv[profile]} = true ]]; then + if [[ -n "${ocrd__argv[profile_file]}" ]]; then exec 3> "${ocrd__argv[profile_file]}" else exec 3>&2 From 6741d55b66f13db685172379f06ef958b2a41eab Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:10:05 +0100 Subject: [PATCH 02/19] improve generate_processor_help: - for docstring extraction, add newlines after each block - remove useless "wiring" of fileGrps - add default values in brackets where applicable - re-order logically (mets before grps) - group processing and non-processing options for clarity - fix the Processing Worker queue format - improve formulations --- ocrd/ocrd/processor/helpers.py | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index bee6a087ca..96ded17815 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -240,11 +240,11 @@ def wrap(s): if processor_instance: module = inspect.getmodule(processor_instance) if module and module.__doc__: - doc_help += '\n' + inspect.cleandoc(module.__doc__) + doc_help += '\n' + inspect.cleandoc(module.__doc__) + '\n' if processor_instance.__doc__: - doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + '\n' if processor_instance.process.__doc__: - doc_help += '\n' + inspect.cleandoc(processor_instance.process.__doc__) + doc_help += '\n' + inspect.cleandoc(processor_instance.process.__doc__) + '\n' if doc_help: doc_help = '\n\n' + wrap_text(doc_help, width=72, initial_indent=' > ', @@ -255,43 +255,47 @@ def wrap(s): %s%s -Options: +Options for processing: + -m, --mets URL-PATH URL or file path of METS to process [./mets.xml] + -w, --working-dir PATH Working directory of local workspace [dirname(URL-PATH)] -I, --input-file-grp USE File group(s) used as input -O, --output-file-grp USE File group(s) used as output - -g, --page-id ID Physical page ID(s) to process + -g, --page-id ID Physical page ID(s) to process [all] --overwrite Remove existing output pages/images - (with --page-id, remove only those) - --queue The RabbitMQ server address in format: {host}:{port}/{vhost}" - --database The MongoDB address in format: mongodb://{host}:{port}" + (with "--page-id", remove only those) --profile Enable profiling - --profile-file Write cProfile stats to this file. Implies --profile + --profile-file PROF-PATH Write cProfile stats to PROF-PATH. Implies "--profile" -p, --parameter JSON-PATH Parameters, either verbatim JSON string or JSON file path -P, --param-override KEY VAL Override a single JSON object key-value pair, - taking precedence over --parameter - -m, --mets URL-PATH URL or file path of METS to process - -w, --working-dir PATH Working directory of local workspace + taking precedence over "--parameter" -l, --log-level [OFF|ERROR|WARN|INFO|DEBUG|TRACE] - Log level + Override log level globally [INFO] + +Options for Processing Worker server: + --queue The RabbitMQ server address in format + "amqp://{user}:{pass}@{host}:{port}/{vhost}" + [amqp://admin:admin@localhost:5672] + --database The MongoDB server address in format + "mongodb://{host}:{port}" + [mongodb://localhost:27018] + +Options for information: -C, --show-resource RESNAME Dump the content of processor resource RESNAME -L, --list-resources List names of processor resources - -J, --dump-json Dump tool description as JSON and exit - -D, --dump-module-dir Output the 'module' directory with resources for this processor - -h, --help This help message + -J, --dump-json Dump tool description as JSON + -D, --dump-module-dir Show the 'module' resource location path for this processor + -h, --help Show this message -V, --version Show version Parameters: %s -Default Wiring: - %s -> %s ''' % ( ocrd_tool['executable'], ocrd_tool['description'], doc_help, parameter_help, - ocrd_tool.get('input_file_grp', 'NONE'), - ocrd_tool.get('output_file_grp', 'NONE') ) From 0ccc9633fb0007e0fab1046005a3573ae926a5e0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:22:50 +0100 Subject: [PATCH 03/19] ocrd_cli_options: remove redundant help kwarg and add `type=click.Path` for `--profile-file`. --- ocrd/ocrd/decorators/ocrd_cli_options.py | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 6245f092bd..2ba4bf8ae1 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -1,4 +1,4 @@ -from click import option +from click import option, Path from .parameter_option import parameter_option, parameter_override_option from .loglevel_option import loglevel_option from ocrd_network import QueueServerParamType, DatabaseParamType @@ -19,28 +19,28 @@ def cli(mets_url): """ # XXX Note that the `--help` output is statically generate_processor_help params = [ - option('-m', '--mets', help="METS to process", default="mets.xml"), - option('-w', '--working-dir', help="Working Directory"), + option('-m', '--mets', default="mets.xml"), + option('-w', '--working-dir'), # TODO OCR-D/core#274 - # option('-I', '--input-file-grp', help='File group(s) used as input. **required**'), - # option('-O', '--output-file-grp', help='File group(s) used as output. **required**'), - option('-I', '--input-file-grp', help='File group(s) used as input.', default='INPUT'), - option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'), - option('-g', '--page-id', help="ID(s) of the pages to process"), - option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False), - option('--queue', help="The URL of the Queue Server, format: username:password@host:port/vhost", type=QueueServerParamType()), - option('--database', help="The URL of the MongoDB, format: mongodb://host:port", type=DatabaseParamType()), - option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'), - option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'), + # option('-I', '--input-file-grp', required=True), + # option('-O', '--output-file-grp', required=True), + option('-I', '--input-file-grp', default='INPUT'), + option('-O', '--output-file-grp', default='OUTPUT'), + option('-g', '--page-id'), + option('--overwrite', is_flag=True, default=False), + option('--profile', is_flag=True, default=False), + option('--profile-file', type=Path(dir_okay=False, writable=True)), parameter_option, parameter_override_option, - option('-J', '--dump-json', help="Dump tool description as JSON and exit", is_flag=True, default=False), - option('-D', '--dump-module-dir', help="Print processor's 'moduledir' of resourcess", is_flag=True, default=False), loglevel_option, - option('-V', '--version', help="Show version", is_flag=True, default=False), - option('-h', '--help', help="This help message", is_flag=True, default=False), - option('--profile', help="Enable profiling", is_flag=True, default=False), - option('--profile-file', help="Write cProfile stats to this file. Implies --profile"), + option('--queue', type=QueueServerParamType()), + option('--database', type=DatabaseParamType()), + option('-C', '--show-resource'), + option('-L', '--list-resources', is_flag=True, default=False), + option('-J', '--dump-json', is_flag=True, default=False), + option('-D', '--dump-module-dir', is_flag=True, default=False), + option('-h', '--help', is_flag=True, default=False), + option('-V', '--version', is_flag=True, default=False), ] for param in params: param(f) From e2e6e69fe22da43b4fed999f4c397b61e16e2925 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:25:43 +0100 Subject: [PATCH 04/19] start native cli: no need for is_bashlib_processor --- ocrd_network/ocrd_network/deployer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 50046a1e1d..27ac323fad 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -22,7 +22,6 @@ CustomDockerClient, DeployType, HostData, - is_bashlib_processor, ) from .rabbitmq_utils import RMQPublisher @@ -281,11 +280,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u self.log.info(f'Starting native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - if is_bashlib_processor(processor_name): - cmd = f'ocrd processing-worker {processor_name} --database {database_url} ' \ - f'--queue {queue_url}' - else: - cmd = f'{processor_name} --database {database_url} --queue {queue_url}' + cmd = f'{processor_name} --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is # printed with `echo $!` but it is printed inbetween other output. Because of that I added From 4ab2657bfb42a81e416859740a876f7bfc38bcca Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:26:10 +0100 Subject: [PATCH 05/19] rm is_bashlib_processor --- ocrd_network/ocrd_network/deployment_utils.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 8b4f2356da..6b943127b4 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -38,22 +38,6 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) -def is_bashlib_processor(processor_name): - """ Determine if a processor is a bashlib processor - - Returns True if processor_name is available as a program and does not contain a python hashbang - in line 1 """ - if not processor_name.startswith("ocrd"): - return False - program = which(processor_name) - if not program: - return False - with open(program) as fin: - line = fin.readline().strip() - if re.fullmatch('[#][!].*/python[0-9.]*', line): - return False - return True - class HostData: """class to store runtime information for a host From a4b9c96ff0488d4ffd12da61271713d43e75bb65 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:29:40 +0100 Subject: [PATCH 06/19] =?UTF-8?q?rm=20Py36=20test=20=E2=80=93=20no=20longe?= =?UTF-8?q?r=20supported?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 556d52388f..eea93f8fb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,17 +19,6 @@ jobs: - run: make install - run: make deps-test test benchmark - test-python36: - docker: - - image: python:3.6.15 - working_directory: ~/ocrd-core - steps: - - checkout - - run: apt-get -y update - - run: pip install -U pip - - run: make deps-ubuntu install - - run: make deps-test test benchmark - test-python37: docker: - image: python:3.7.16 @@ -104,7 +93,6 @@ workflows: only: master test-pull-request: jobs: - - test-python36 - test-python37 - test-python38 - test-python39 From 3f13747f9b0978c08d95c83ba110de2f10aa7d68 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:32:18 +0100 Subject: [PATCH 07/19] CI: use Python images from CircleCI instead of DH (because they are cached/faster) --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eea93f8fb5..181e9d17aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: test-python37: docker: - - image: python:3.7.16 + - image: cimg/python:3.7.16 working_directory: ~/ocrd-core steps: - checkout @@ -31,7 +31,7 @@ jobs: test-python38: docker: - - image: python:3.8.16 + - image: cimg/python:3.8.16 working_directory: ~/ocrd-core steps: - checkout @@ -41,7 +41,7 @@ jobs: test-python39: docker: - - image: python:3.9.16 + - image: cimg/python:3.9.16 working_directory: ~/ocrd-core steps: - checkout @@ -51,7 +51,7 @@ jobs: test-python310: docker: - - image: python:3.10.10 + - image: cimg/python:3.10.10 working_directory: ~/ocrd-core steps: - checkout @@ -61,7 +61,7 @@ jobs: test-python311: docker: - - image: python:3.11.2 + - image: cimg/python:3.11.2 working_directory: ~/ocrd-core steps: - checkout From 6ead599dce67bca8bee0792e2c5ca6241f2be5da Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:33:24 +0100 Subject: [PATCH 08/19] require Py37+ --- ocrd_utils/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_utils/setup.py b/ocrd_utils/setup.py index 88d319573c..d3e43b033b 100644 --- a/ocrd_utils/setup.py +++ b/ocrd_utils/setup.py @@ -14,6 +14,7 @@ url='https://github.com/OCR-D/core', license='Apache License 2.0', packages=['ocrd_utils'], + python_requires=">=3.7", install_requires=install_requires, package_data={'': ['*.json', '*.yml', '*.xml']}, keywords=['OCR', 'OCR-D'] From 89570a79bab5f30fe8ed6ae352c40d19a8f8a5d4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:33:59 +0100 Subject: [PATCH 09/19] require Py37+ --- ocrd/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd/setup.py b/ocrd/setup.py index 67536620f6..be28ba0d6b 100644 --- a/ocrd/setup.py +++ b/ocrd/setup.py @@ -22,6 +22,7 @@ license='Apache License 2.0', packages=find_packages(exclude=('tests', 'docs')), include_package_data=True, + python_requires=">=3.7", install_requires=install_requires, entry_points={ 'console_scripts': [ From e1f6ed8c572952cccd766767fed32c7be777351d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:34:33 +0100 Subject: [PATCH 10/19] require Py37+ --- ocrd_network/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py index 63678f191e..f79081fa09 100644 --- a/ocrd_network/setup.py +++ b/ocrd_network/setup.py @@ -15,6 +15,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=[ 'ocrd_network', From fed4128a89f9cf2dbab6b1492691cfcbfda863e1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:34:56 +0100 Subject: [PATCH 11/19] require Py37+ --- ocrd_modelfactory/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_modelfactory/setup.py b/ocrd_modelfactory/setup.py index 16bf4c6ccb..a251460738 100644 --- a/ocrd_modelfactory/setup.py +++ b/ocrd_modelfactory/setup.py @@ -17,6 +17,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=['ocrd_modelfactory'], package_data={'': ['*.json', '*.yml', '*.xml']}, From af08d1eaa85efcfda0312a8ef4b89a9ea03f9ef3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:35:17 +0100 Subject: [PATCH 12/19] require Py37+ --- ocrd_models/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_models/setup.py b/ocrd_models/setup.py index 3d37f4a335..b34212603c 100644 --- a/ocrd_models/setup.py +++ b/ocrd_models/setup.py @@ -16,6 +16,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=['ocrd_models'], package_data={'': ['*.json', '*.yml', '*.xml']}, From 21c46c1de30bc2d0409b8d6afb7d70bda8b25d34 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:35:33 +0100 Subject: [PATCH 13/19] require Py37+ --- ocrd_validators/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_validators/setup.py b/ocrd_validators/setup.py index 121071e91f..fc8c23e376 100644 --- a/ocrd_validators/setup.py +++ b/ocrd_validators/setup.py @@ -18,6 +18,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=['ocrd_validators'], package_data={ From fe3303e7edbe9a43307f9fd59c547a2e2cbcb40f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 01:09:19 +0100 Subject: [PATCH 14/19] fix CI: - cimg needs sudo - don't specify Python subminors, so it always picks latest - activate docker_layer_caching for deploy to speed up --- .circleci/config.yml | 50 +++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 181e9d17aa..1d1be701c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,58 +16,63 @@ jobs: steps: - checkout - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos - - run: make install - - run: make deps-test test benchmark + - run: make install deps-test + - run: make test benchmark test-python37: docker: - - image: cimg/python:3.7.16 + - image: cimg/python:3.7 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python38: docker: - - image: cimg/python:3.8.16 + - image: cimg/python:3.8 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python39: docker: - - image: cimg/python:3.9.16 + - image: cimg/python:3.9 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python310: docker: - - image: cimg/python:3.10.10 + - image: cimg/python:3.10 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python311: docker: - - image: cimg/python:3.11.2 + - image: cimg/python:3.11 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark deploy: docker: @@ -75,6 +80,7 @@ jobs: steps: - checkout - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: make docker - run: make docker-cuda - run: From df7bfee37ecacca2817a3323a797bfbe6ca68fd3 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:28:06 +0200 Subject: [PATCH 15/19] --help: clearer description of --page-id Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd/ocrd/processor/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 96ded17815..bc3cce6374 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -260,7 +260,7 @@ def wrap(s): -w, --working-dir PATH Working directory of local workspace [dirname(URL-PATH)] -I, --input-file-grp USE File group(s) used as input -O, --output-file-grp USE File group(s) used as output - -g, --page-id ID Physical page ID(s) to process [all] + -g, --page-id ID Physical page ID(s) to process instead of full document [] --overwrite Remove existing output pages/images (with "--page-id", remove only those) --profile Enable profiling From 378400e17e0776f1e24eb967312c91aadced8203 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:30:54 +0200 Subject: [PATCH 16/19] .circleci/config.yml: fix syntax --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d1be701c8..e018e389fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,8 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker docker_layer_caching: true - run: make docker - run: make docker-cuda From dbae7c42b6de64a027d86e0f2dfd4cf7dc8ca36e Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:30:54 +0200 Subject: [PATCH 17/19] .circleci/config.yml: fix syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d1be701c8..ba90d83f33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker docker_layer_caching: true - run: make docker - run: make docker-cuda From 25cc6fcf58b6975f75711c2f76e0a26c66472ec1 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:33:38 +0200 Subject: [PATCH 18/19] .circleci/config.yml: fix syntax --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e018e389fb..ba90d83f33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,6 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - # https://circleci.com/docs/2.0/building-docker-images/ - setup_remote_docker docker_layer_caching: true - run: make docker From 176cf73da04e32b3b5e3ce0d539accc10812f69c Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:35:10 +0200 Subject: [PATCH 19/19] .circleci/config.yml: fix syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba90d83f33..3490bd4670 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ docker_layer_caching: true - run: make docker - run: make docker-cuda