From 368f14c6fded3fa9d245e80c26a9702b24d43659 Mon Sep 17 00:00:00 2001 From: Ignotus Date: Wed, 16 Nov 2022 16:49:36 +0100 Subject: [PATCH 1/2] Update scripts for new po4a version and better markdown support --- .github/workflows/add-lang.yml | 2 +- .github/workflows/jekyll.yml | 2 +- .github/workflows/main.yml | 2 +- _po4a-tools/po4a-cache.sh | 6 +++--- _po4a-tools/po4a-create-all-targets.sh | 19 +++++++++++++++++-- _po4a-tools/po4a-update-templates.sh | 7 ++++++- 6 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/add-lang.yml b/.github/workflows/add-lang.yml index 9f6b86f5b..0a9a12f11 100644 --- a/.github/workflows/add-lang.yml +++ b/.github/workflows/add-lang.yml @@ -48,7 +48,7 @@ jobs: id: cache-po4a with: path: "~/po4a" - key: ${{ runner.os }}-po4a + key: ${{ runner.os }}-po4a-0.68 - name: Install/retrieve po4a from cache env: CACHE_HIT: ${{ steps.cache-po4a.outputs.cache-hit }} diff --git a/.github/workflows/jekyll.yml b/.github/workflows/jekyll.yml index b180adb16..d0ac3092e 100644 --- a/.github/workflows/jekyll.yml +++ b/.github/workflows/jekyll.yml @@ -29,7 +29,7 @@ jobs: id: cache-po4a with: path: "~/po4a" - key: ${{ runner.os }}-po4a + key: ${{ runner.os }}-po4a-0.68 - name: Install or retrieve po4a from cache env: CACHE_HIT: ${{steps.cache-po4a.outputs.cache-hit}} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a7bac976..529de20ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: id: cache-po4a with: path: "~/po4a" - key: ${{ runner.os }}-po4a + key: ${{ runner.os }}-po4a-0.68 - name: Install or retrieve po4a from cache env: CACHE_HIT: ${{steps.cache-po4a.outputs.cache-hit}} diff --git a/_po4a-tools/po4a-cache.sh b/_po4a-tools/po4a-cache.sh index 174d4aa59..735efafc0 100755 --- a/_po4a-tools/po4a-cache.sh +++ b/_po4a-tools/po4a-cache.sh @@ -6,8 +6,8 @@ if [[ "$CACHE_HIT" == 'true' ]] ; then sudo cp --force --recursive ~/po4a/* / else - sudo apt install -yq gettext libsgmls-perl libyaml-tiny-perl opensp - wget -O po4a.deb https://github.com/jamulussoftware/assets/raw/main/po4a/po4a_0.66.deb + sudo apt install -yq gettext libsgmls-perl libyaml-tiny-perl opensp libsyntax-keyword-try-perl + wget -O po4a.deb https://github.com/jamulussoftware/assets/raw/main/po4a/po4a_0.68.deb sudo dpkg -i po4a.deb if [ -f po4a.deb ] ; then @@ -16,7 +16,7 @@ fi mkdir -p ~/po4a -for dep in po4a libcroco3 libosp5 sgml-base gettext libsgmls-perl libyaml-tiny-perl opensp; do +for dep in po4a libcroco3 libosp5 sgml-base gettext libsgmls-perl libyaml-tiny-perl opensp libsyntax-keyword-try-perl; do dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/po4a/ done fi diff --git a/_po4a-tools/po4a-create-all-targets.sh b/_po4a-tools/po4a-create-all-targets.sh index e1e10b1a6..9391a9071 100755 --- a/_po4a-tools/po4a-create-all-targets.sh +++ b/_po4a-tools/po4a-create-all-targets.sh @@ -26,6 +26,7 @@ PO_DIR="../_translator-files/po" # Directories where the translated files will be WIKI_DIR="../wiki" DATA_DIR="../_data" +# INCLUDES_DIR="../_includes/wiki" # CHECK FOR PO4A INSTALLATION @@ -62,6 +63,11 @@ for lang in $(ls "$PO_DIR") ; do rm -rf "$DATA_DIR/$lang" echo "$lang" folder deleted from ./_data fi + +# if [ -d "$INCLUDES_DIR/$lang" ] ; then +# rm -rf "$INCLUDES_DIR/$lang" +# echo "$lang" folder deleted from ./_includes +# fi done # FUNCTION TO CREATE TARGET FILES FROM .po FILES USING PO4A @@ -81,6 +87,9 @@ process_with_po4a () { if [[ $filename == 'general' || $filename == 'navigation' ]] ; then TARG_DIR="$DATA_DIR" +# elif [[ $filename == 'footertext' ]] ; then +# TARG_DIR="$INCLUDES_DIR" + else TARG_DIR="$WIKI_DIR" fi @@ -91,6 +100,7 @@ process_with_po4a () { if [[ "$filename" == 'Include-'* || \ "$filename" == *'-index' || \ +# "$filename" == 'footertext' || \ "$filename" == 'general' || \ "$filename" == 'navigation' ]] ; then @@ -102,10 +112,13 @@ process_with_po4a () { # Determine file format to be used if [ $ext == yml ] ; then FILE_FORMAT=yaml + OPTION="skip_array" elif [ $ext == html ] ; then FILE_FORMAT=xml + OPTION="ontagerror=warn" elif [ $ext == md ] ; then - FILE_FORMAT=asciidoc + FILE_FORMAT=text + OPTION="markdown" fi # Run po4a-translate and create target files @@ -116,6 +129,8 @@ process_with_po4a () { --po "$PO_DIR/$lang/${filename}.po" \ --localized "$targ_doc" \ --localized-charset "UTF-8" \ + --no-deprecation \ + --option "$OPTION" \ --keep "$THRESHOLD" # Display message if translated file is created @@ -136,4 +151,4 @@ while IFS= read -r -d '' dir ; do done < <(find "$PO_DIR" -mindepth 1 -maxdepth 1 -type d -print0) # Produce a file with translation status of all .po files -source ./po4a-stats.sh +# source ./po4a-stats.sh diff --git a/_po4a-tools/po4a-update-templates.sh b/_po4a-tools/po4a-update-templates.sh index 69dd1f0e2..97b96fe8f 100755 --- a/_po4a-tools/po4a-update-templates.sh +++ b/_po4a-tools/po4a-update-templates.sh @@ -63,10 +63,13 @@ while IFS= read -r -d '' doc ; do # Determine file format to be used if [ $ext == yml ] ; then FILE_FORMAT=yaml + OPTION="skip_array" elif [ $ext == html ] ; then FILE_FORMAT=xml + OPTION="ontagerror=warn" elif [ $ext == md ] ; then - FILE_FORMAT=asciidoc + FILE_FORMAT=text + OPTION="markdown" fi # Update/create .po files @@ -76,6 +79,8 @@ while IFS= read -r -d '' doc ; do --master-charset "UTF-8" \ --msgmerge-opt --no-wrap \ --wrap-po newlines \ + --no-deprecation \ + --option "$OPTION" \ --po "$po_file" ; then echo '' echo Error updating "$lang" PO file for: "$filename".$ext From 2260164d247a92b6d86972e0abb38d28eebec44e Mon Sep 17 00:00:00 2001 From: Ignotus Date: Wed, 16 Nov 2022 17:04:03 +0100 Subject: [PATCH 2/2] Remove unneeded lines and stats script --- _po4a-tools/po4a-create-all-targets.sh | 13 ------- _po4a-tools/po4a-stats.sh | 48 -------------------------- 2 files changed, 61 deletions(-) delete mode 100755 _po4a-tools/po4a-stats.sh diff --git a/_po4a-tools/po4a-create-all-targets.sh b/_po4a-tools/po4a-create-all-targets.sh index 9391a9071..6b85c8b37 100755 --- a/_po4a-tools/po4a-create-all-targets.sh +++ b/_po4a-tools/po4a-create-all-targets.sh @@ -26,7 +26,6 @@ PO_DIR="../_translator-files/po" # Directories where the translated files will be WIKI_DIR="../wiki" DATA_DIR="../_data" -# INCLUDES_DIR="../_includes/wiki" # CHECK FOR PO4A INSTALLATION @@ -63,11 +62,6 @@ for lang in $(ls "$PO_DIR") ; do rm -rf "$DATA_DIR/$lang" echo "$lang" folder deleted from ./_data fi - -# if [ -d "$INCLUDES_DIR/$lang" ] ; then -# rm -rf "$INCLUDES_DIR/$lang" -# echo "$lang" folder deleted from ./_includes -# fi done # FUNCTION TO CREATE TARGET FILES FROM .po FILES USING PO4A @@ -87,9 +81,6 @@ process_with_po4a () { if [[ $filename == 'general' || $filename == 'navigation' ]] ; then TARG_DIR="$DATA_DIR" -# elif [[ $filename == 'footertext' ]] ; then -# TARG_DIR="$INCLUDES_DIR" - else TARG_DIR="$WIKI_DIR" fi @@ -100,7 +91,6 @@ process_with_po4a () { if [[ "$filename" == 'Include-'* || \ "$filename" == *'-index' || \ -# "$filename" == 'footertext' || \ "$filename" == 'general' || \ "$filename" == 'navigation' ]] ; then @@ -149,6 +139,3 @@ while IFS= read -r -d '' dir ; do echo "$lang": process_with_po4a "$lang" done < <(find "$PO_DIR" -mindepth 1 -maxdepth 1 -type d -print0) - -# Produce a file with translation status of all .po files -# source ./po4a-stats.sh diff --git a/_po4a-tools/po4a-stats.sh b/_po4a-tools/po4a-stats.sh deleted file mode 100755 index 61197641a..000000000 --- a/_po4a-tools/po4a-stats.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Creates a statistics file with information on the translation status of each file for every language. - -# Stats file location -STATS_FILE="../wiki/Statistics.md" - -# Print yaml front matter and table title/header -echo '--- -layout: wiki -title: "Statistics" -lang: "en" -permalink: "/wiki/Statistics" ---- -# Current status of website translations - -| Language | Document | Translation status | -|----------|----------|--------------------|' >> "$STATS_FILE" - -produce_stats () { -# Determine file names - while IFS= read -r -d '' doc ; do - - # Get file extension - ext=$(echo "$doc" | sed 's/.*\.//') - - filename=$(basename "$doc" .$ext) - - # Stats printed to Statistics.md - echo -n "|**"$lang"**| **"$filename".po**|" >> "$STATS_FILE" - msgfmt --statistics "$PO_DIR/$lang/$filename".po &>> "$STATS_FILE" - - done < <(find -L "$SRC_DIR" -name "*.*" -print0) - - # Separator between languages - echo '|**-----**|**--------------------**|**--------------------**|' >> "$STATS_FILE" -} - -# Run produce_stats on each language folder -while IFS= read -r -d '' dir ; do - lang=$(basename "$dir") - produce_stats "$lang" -done < <(find "$PO_DIR" -mindepth 1 -maxdepth 1 -type d -print0) - -# Remove unwanted messages.mo file created by msgfmt -rm -f *.mo - -echo Statistics file created