From c97ced46dd947a3c6fc18ab9d9a4890bc0b1929e Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 20 Jan 2022 22:44:34 -0500 Subject: [PATCH 1/8] Add HTTP Last-Modified header for TEI resources --- modules/app.xqm | 4 ++++ modules/config.xqm | 45 +++++++++++++++++++++++++++++++++++++++++++++ modules/pages.xqm | 23 +++++++++++++++++++++-- modules/view.xql | 10 +++++++++- 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/modules/app.xqm b/modules/app.xqm index da32bd4e0..0efdd2265 100644 --- a/modules/app.xqm +++ b/modules/app.xqm @@ -151,6 +151,10 @@ function app:handle-error($node as node(), $model as map(*), $code as xs:int?) { () }; +declare function app:set-last-modified($last-modified as xs:dateTime) { + response:set-header("Last-Modified", format-dateTime($last-modified, "[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]")) +}; + declare function app:uri($node as node(), $model as map(*)) { {request:get-attribute("hsg-shell.path")} }; diff --git a/modules/config.xqm b/modules/config.xqm index 04d41d8ed..12bb6a3b4 100644 --- a/modules/config.xqm +++ b/modules/config.xqm @@ -115,6 +115,19 @@ declare variable $config:PUBLICATIONS := map { "frus": map { "collection": $config:FRUS_VOLUMES_COL, + "document-last-modified": function($document-id) { + ( + xmldb:last-modified($config:FRUS_VOLUMES_COL, $document-id || '.xml'), + (: for volumes that we do not have as TEI yet, fall back on volume metadata :) + xmldb:last-modified($config:FRUS_METADATA_COL, $document-id || '.xml') + )[1] + }, + "section-last-modified": function($document-id, $section-id) { + ( + xmldb:last-modified($config:FRUS_VOLUMES_COL, $document-id || '.xml'), + xmldb:last-modified($config:FRUS_METADATA_COL, $document-id || '.xml') + )[1] + }, "select-document": function($document-id) { doc($config:FRUS_VOLUMES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { let $node := doc($config:FRUS_VOLUMES_COL || '/' || $document-id || '.xml')/id($section-id) @@ -155,6 +168,8 @@ declare variable $config:PUBLICATIONS := }, "buildings": map { "collection": $config:BUILDINGS_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:BUILDINGS_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:BUILDINGS_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:BUILDINGS_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:BUILDINGS_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, $section-id), '/') }, @@ -171,6 +186,8 @@ declare variable $config:PUBLICATIONS := }, "conferences": map { "collection": $config:CONFERENCES_ARTICLES_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:CONFERENCES_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:CONFERENCES_ARTICLES_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/conferences/" || string-join(($document-id, $section-id), '/') }, @@ -193,6 +210,8 @@ declare variable $config:PUBLICATIONS := }, "countries": map { "collection": $config:COUNTRIES_ARTICLES_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:COUNTRIES_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:COUNTRIES_ARTICLES_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/countries/" || string-join(($document-id, $section-id), '/') }, @@ -203,6 +222,8 @@ declare variable $config:PUBLICATIONS := }, "countries-issues": map { "collection": $config:COUNTRIES_ISSUES_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:COUNTRIES_ISSUES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:COUNTRIES_ISSUES_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/countries/issues/" || string-join(($document-id, $section-id), '/') }, @@ -213,6 +234,8 @@ declare variable $config:PUBLICATIONS := }, "archives": map { "collection": $config:ARCHIVES_ARTICLES_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:ARCHIVES_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:ARCHIVES_ARTICLES_COL || '/' || $document-id || '.xml')//tei:body }, "html-href": function($document-id, $section-id) { "$app/countries/" || string-join(($document-id, $section-id), '/') }, @@ -222,6 +245,8 @@ declare variable $config:PUBLICATIONS := }, "articles": map { "collection": $config:FRUS_HISTORY_ARTICLES_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:FRUS_HISTORY_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:FRUS_HISTORY_ARTICLES_COL || '/' || $document-id || '.xml')//tei:body }, "html-href": function($document-id, $section-id) { "$app/frus-history/" || string-join(($document-id, $section-id), '/') }, @@ -240,6 +265,8 @@ declare variable $config:PUBLICATIONS := }, "people": map { "collection": $config:SECRETARY_BIOS_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:SECRETARY_BIOS_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:SECRETARY_BIOS_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:SECRETARY_BIOS_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:SECRETARY_BIOS_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/people/" || string-join(($document-id, $section-id), '/') }, @@ -285,6 +312,8 @@ declare variable $config:PUBLICATIONS := }, "milestones": map { "collection": $config:MILESTONES_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:MILESTONES_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:MILESTONES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:MILESTONES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:MILESTONES_COL|| '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/milestones/" || string-join(($document-id, $section-id), '/') }, @@ -295,6 +324,8 @@ declare variable $config:PUBLICATIONS := }, "short-history": map { "collection": $config:SHORT_HISTORY_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:SHORT_HISTORY_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:SHORT_HISTORY_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:SHORT_HISTORY_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:SHORT_HISTORY_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, $section-id), '/') }, @@ -305,6 +336,8 @@ declare variable $config:PUBLICATIONS := }, "timeline": map { "collection": $config:ADMINISTRATIVE_TIMELINE_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:ADMINISTRATIVE_TIMELINE_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:ADMINISTRATIVE_TIMELINE_COL || '/' || $document-id || '.xml')/id('chapter_' || $section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, substring-after($section-id, 'chapter_')), '/') }, @@ -316,6 +349,8 @@ declare variable $config:PUBLICATIONS := }, "faq": map { "collection": $config:FAQ_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:FAQ_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FAQ_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:FAQ_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:FAQ_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/about/" || string-join(($document-id, $section-id), '/') }, @@ -325,6 +360,8 @@ declare variable $config:PUBLICATIONS := }, "hac": map { "collection": $config:HAC_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:HAC_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:HAC_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:HAC_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:HAC_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/about/" || string-join(($document-id, $section-id), '/') }, @@ -334,6 +371,8 @@ declare variable $config:PUBLICATIONS := }, "education": map { "collection": $config:EDUCATION_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:EDUCATION_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:EDUCATION_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:EDUCATION_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:EDUCATION_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/education/modules/" || string-join(($document-id, $section-id), '#') }, @@ -346,6 +385,8 @@ declare variable $config:PUBLICATIONS := }, "frus-history-monograph": map { "collection": $config:FRUS_HISTORY_MONOGRAPH_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:FRUS_HISTORY_MONOGRAPH_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { let $target-section-id := @@ -392,6 +433,8 @@ declare variable $config:PUBLICATIONS := }, "vietnam-guide": map { "collection": $config:VIETNAM_GUIDE_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:VIETNAM_GUIDE_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:VIETNAM_GUIDE_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:VIETNAM_GUIDE_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:VIETNAM_GUIDE_COL || '/' || $document-id || '.xml') }, "html-href": function($document-id, $section-id) { "$app/historicaldocuments/" || string-join(($document-id, $section-id), '/') }, @@ -401,6 +444,8 @@ declare variable $config:PUBLICATIONS := }, "views-from-the-embassy": map { "collection": $config:VIEWS_FROM_EMBASSY_COL, + "document-last-modified": function($document-id) { xmldb:last-modified($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') }, + "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:VIEWS_FROM_EMBASSY_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:VIEWS_FROM_EMBASSY_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/wwi" }, diff --git a/modules/pages.xqm b/modules/pages.xqm index 45182aec7..44716687d 100644 --- a/modules/pages.xqm +++ b/modules/pages.xqm @@ -44,6 +44,11 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str $section-id as xs:string?, $view as xs:string, $ignore as xs:boolean) { let $log := console:log("loading publication-id: " || $publication-id || " document-id: " || $document-id || " section-id: " || $section-id ) + let $last-modified := + if (exists($publication-id) and exists($document-id)) then + pages:last-modified($publication-id, $document-id, $section-id) + else + () let $content := map { "data": if (exists($publication-id) and exists($document-id)) then @@ -54,7 +59,8 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str "section-id": $section-id, "view": $view, "base-path": - (: allow for pages that don't have $config:PUBLICATIONS?select-document defined :) + (: allow for pages that do not have $config:PUBLICATIONS?select-document defined :) + (: ... TODO: I do not see any such cases in config:PUBLICATIONS! Check if OK to remove this entry? - JW :) if (exists($publication-id) and map:contains(map:get($config:PUBLICATIONS, $publication-id), 'base-path')) then map:get($config:PUBLICATIONS, $publication-id)?base-path($document-id, $section-id) else (), @@ -62,7 +68,20 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str } return - templates:process($node/*, map:merge(($model, $content))) + ( + if (exists($last-modified)) then + request:set-attribute("hsgshell.last-modified", $last-modified) + else + (), + templates:process($node/*, map:merge(($model, $content))) + ) +}; + +declare function pages:last-modified($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?) { + if ($section-id) then + map:get($config:PUBLICATIONS, $publication-id)?section-last-modified($document-id, $section-id) + else + map:get($config:PUBLICATIONS, $publication-id)?document-last-modified($document-id) }; declare function pages:load-xml($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?, $view as xs:string) { diff --git a/modules/view.xql b/modules/view.xql index b9619292a..533822448 100644 --- a/modules/view.xql +++ b/modules/view.xql @@ -56,4 +56,12 @@ let $lookup := function($functionName as xs:string, $arity as xs:int) { :) let $content := request:get-data() return - templates:apply($content, $lookup, (), $config) + ( + templates:apply($content, $lookup, (), $config), + let $last-modified := request:get-attribute("hsgshell.last-modified") + return + if (exists($last-modified)) then + app:set-last-modified($last-modified) + else + () + ) \ No newline at end of file From e789d8d05d5d75fbf060d288f9a33c24f73b5f3b Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 21 Jan 2022 00:49:24 -0500 Subject: [PATCH 2/8] Ensure Last-Modified is GMT (+0000) --- modules/app.xqm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/app.xqm b/modules/app.xqm index 0efdd2265..f342ac7c5 100644 --- a/modules/app.xqm +++ b/modules/app.xqm @@ -152,7 +152,7 @@ function app:handle-error($node as node(), $model as map(*), $code as xs:int?) { }; declare function app:set-last-modified($last-modified as xs:dateTime) { - response:set-header("Last-Modified", format-dateTime($last-modified, "[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]")) + response:set-header("Last-Modified", format-dateTime(adjust-dateTime-to-timezone($last-modified, xs:dayTimeDuration("PT0H")), "[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]", "en", (), ())) }; declare function app:uri($node as node(), $model as map(*)) { From 23b47594084cc5ece1d02b9110aeb5191f6f0eeb Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 21 Jan 2022 00:50:22 -0500 Subject: [PATCH 3/8] Work around header blindness outside controller --- controller.xql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/controller.xql b/controller.xql index e206cad0a..abf2138bc 100644 --- a/controller.xql +++ b/controller.xql @@ -40,6 +40,13 @@ console:log('exist:path: ' || $exist:path) , :) +let $if-modified-since := request:get-header("If-Modified-Since") +return + if ($if-modified-since) then + request:set-attribute("if-modified-since", $if-modified-since) + else + (), + if ($exist:path eq '') then From 59008ca443c15b145bf548f3f9c41687572c7e86 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 21 Jan 2022 00:50:54 -0500 Subject: [PATCH 4/8] Set 304 based on "If-Modified-Since" header --- modules/pages.xqm | 66 +++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/modules/pages.xqm b/modules/pages.xqm index 44716687d..4a876370b 100644 --- a/modules/pages.xqm +++ b/modules/pages.xqm @@ -49,32 +49,48 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str pages:last-modified($publication-id, $document-id, $section-id) else () - let $content := map { - "data": - if (exists($publication-id) and exists($document-id)) then - pages:load-xml($publication-id, $document-id, $section-id, $view, $ignore) - else (), - "publication-id": $publication-id, - "document-id": $document-id, - "section-id": $section-id, - "view": $view, - "base-path": - (: allow for pages that do not have $config:PUBLICATIONS?select-document defined :) - (: ... TODO: I do not see any such cases in config:PUBLICATIONS! Check if OK to remove this entry? - JW :) - if (exists($publication-id) and map:contains(map:get($config:PUBLICATIONS, $publication-id), 'base-path')) then - map:get($config:PUBLICATIONS, $publication-id)?base-path($document-id, $section-id) - else (), - "odd": if (exists($publication-id)) then map:get($config:PUBLICATIONS, $publication-id)?transform else $config:odd-transform-default - } - + let $if-modified-since := try { request:get-attribute("if-modified-since") => parse-ietf-date() } catch * { () } + let $should-return-304 := + if (exists($last-modified) and exists($if-modified-since)) then + $if-modified-since gt $last-modified + else + () return - ( - if (exists($last-modified)) then - request:set-attribute("hsgshell.last-modified", $last-modified) - else - (), - templates:process($node/*, map:merge(($model, $content))) - ) + (: if the "If-Modified-Since" header in the client request is later than the + : last-modified date, then halt further processing of the templates and simply + : return a 304 response. :) + if ($should-return-304) then + ( + response:set-status-code(304), + app:set-last-modified($last-modified) + ) + else + let $content := map { + "data": + if (exists($publication-id) and exists($document-id)) then + pages:load-xml($publication-id, $document-id, $section-id, $view, $ignore) + else (), + "publication-id": $publication-id, + "document-id": $document-id, + "section-id": $section-id, + "view": $view, + "base-path": + (: allow for pages that do not have $config:PUBLICATIONS?select-document defined :) + (: ... TODO: I do not see any such cases in config:PUBLICATIONS! Check if OK to remove this entry? - JW :) + if (exists($publication-id) and map:contains(map:get($config:PUBLICATIONS, $publication-id), 'base-path')) then + map:get($config:PUBLICATIONS, $publication-id)?base-path($document-id, $section-id) + else (), + "odd": if (exists($publication-id)) then map:get($config:PUBLICATIONS, $publication-id)?transform else $config:odd-transform-default + } + + return + ( + if (exists($last-modified)) then + request:set-attribute("hsgshell.last-modified", $last-modified) + else + (), + templates:process($node/*, map:merge(($model, $content))) + ) }; declare function pages:last-modified($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?) { From 47f5b8579130b2a4816d88000319ecbb840559c8 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 21 Jan 2022 08:42:35 -0500 Subject: [PATCH 5/8] Clients request previously received Last-Modified Address @agh2342's comment: > As per rfc2616 clients SHOULD set If-Modified-Since to a previously received Last-Modified. So using ge (greater or equal) instead of gr (greater) here seems desirable. Otherwise most requests will still get a 200 instead of a 304. --- modules/pages.xqm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pages.xqm b/modules/pages.xqm index 4a876370b..549cecac3 100644 --- a/modules/pages.xqm +++ b/modules/pages.xqm @@ -52,7 +52,7 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str let $if-modified-since := try { request:get-attribute("if-modified-since") => parse-ietf-date() } catch * { () } let $should-return-304 := if (exists($last-modified) and exists($if-modified-since)) then - $if-modified-since gt $last-modified + $if-modified-since ge $last-modified else () return From 7f3a694b71412eaf6aa2ca6f5bb8873524e18cb3 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 3 Feb 2022 12:28:39 -0500 Subject: [PATCH 6/8] Truncate milliseconds from last-modified --- modules/pages.xqm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/pages.xqm b/modules/pages.xqm index 549cecac3..56d371945 100644 --- a/modules/pages.xqm +++ b/modules/pages.xqm @@ -47,6 +47,12 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str let $last-modified := if (exists($publication-id) and exists($document-id)) then pages:last-modified($publication-id, $document-id, $section-id) + (: For the purpose of comparing the resource's last modified date with the If-Modified-Since + : header supplied by the client, we must truncate any milliseconds from the last modified date. + : This is because HTTP-date is only specific to the second. + : @see https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1 :) + => format-dateTime("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]") + => xs:dateTime() else () let $if-modified-since := try { request:get-attribute("if-modified-since") => parse-ietf-date() } catch * { () } From 8695ad3daee8dd2a985ea54ac9f126a5b7bcf2e7 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 3 Feb 2022 15:27:27 -0500 Subject: [PATCH 7/8] Add HTTP Created header for TEI publications --- modules/app.xqm | 14 ++++++++++++-- modules/config.xqm | 45 +++++++++++++++++++++++++++++++++++++++++++++ modules/pages.xqm | 20 ++++++++++++++++++-- modules/view.xql | 8 ++++++-- 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/modules/app.xqm b/modules/app.xqm index f342ac7c5..f441b1f10 100644 --- a/modules/app.xqm +++ b/modules/app.xqm @@ -1,4 +1,4 @@ -xquery version "3.0"; +xquery version "3.1"; module namespace app="http://history.state.gov/ns/site/hsg/templates"; @@ -151,8 +151,18 @@ function app:handle-error($node as node(), $model as map(*), $code as xs:int?) { () }; +declare function app:format-http-date($dateTime as xs:dateTime) as xs:string { + $dateTime + => adjust-dateTime-to-timezone(xs:dayTimeDuration("PT0H")) + => format-dateTime("[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]", "en", (), ()) +}; + declare function app:set-last-modified($last-modified as xs:dateTime) { - response:set-header("Last-Modified", format-dateTime(adjust-dateTime-to-timezone($last-modified, xs:dayTimeDuration("PT0H")), "[FNn,*-3], [D01] [MNn,*-3] [Y0001] [H01]:[m01]:[s01] [Z0000]", "en", (), ())) + response:set-header("Last-Modified", app:format-http-date($last-modified)) +}; + +declare function app:set-created($created as xs:dateTime) { + response:set-header("Created", app:format-http-date($created)) }; declare function app:uri($node as node(), $model as map(*)) { diff --git a/modules/config.xqm b/modules/config.xqm index 12bb6a3b4..bf46f68e5 100644 --- a/modules/config.xqm +++ b/modules/config.xqm @@ -128,6 +128,19 @@ declare variable $config:PUBLICATIONS := xmldb:last-modified($config:FRUS_METADATA_COL, $document-id || '.xml') )[1] }, + "document-created": function($document-id) { + ( + xmldb:created($config:FRUS_VOLUMES_COL, $document-id || '.xml'), + (: for volumes that we do not have as TEI yet, fall back on volume metadata :) + xmldb:created($config:FRUS_METADATA_COL, $document-id || '.xml') + )[1] + }, + "section-created": function($document-id, $section-id) { + ( + xmldb:created($config:FRUS_VOLUMES_COL, $document-id || '.xml'), + xmldb:created($config:FRUS_METADATA_COL, $document-id || '.xml') + )[1] + }, "select-document": function($document-id) { doc($config:FRUS_VOLUMES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { let $node := doc($config:FRUS_VOLUMES_COL || '/' || $document-id || '.xml')/id($section-id) @@ -170,6 +183,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:BUILDINGS_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:BUILDINGS_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:BUILDINGS_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:BUILDINGS_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:BUILDINGS_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:BUILDINGS_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:BUILDINGS_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, $section-id), '/') }, @@ -188,6 +203,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:CONFERENCES_ARTICLES_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:CONFERENCES_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:CONFERENCES_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:CONFERENCES_ARTICLES_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/conferences/" || string-join(($document-id, $section-id), '/') }, @@ -213,6 +230,8 @@ declare variable $config:PUBLICATIONS := "document-last-modified": function($document-id) { xmldb:last-modified($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:COUNTRIES_ARTICLES_COL || '/' || $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:COUNTRIES_ARTICLES_COL, $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:COUNTRIES_ARTICLES_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/countries/" || string-join(($document-id, $section-id), '/') }, "odd": "frus.odd", @@ -224,6 +243,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:COUNTRIES_ISSUES_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:COUNTRIES_ISSUES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:COUNTRIES_ISSUES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:COUNTRIES_ISSUES_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/countries/issues/" || string-join(($document-id, $section-id), '/') }, @@ -236,6 +257,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:ARCHIVES_ARTICLES_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:ARCHIVES_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:ARCHIVES_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:ARCHIVES_ARTICLES_COL || '/' || $document-id || '.xml')//tei:body }, "html-href": function($document-id, $section-id) { "$app/countries/" || string-join(($document-id, $section-id), '/') }, @@ -247,6 +270,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:FRUS_HISTORY_ARTICLES_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:FRUS_HISTORY_ARTICLES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:FRUS_HISTORY_ARTICLES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:FRUS_HISTORY_ARTICLES_COL || '/' || $document-id || '.xml')//tei:body }, "html-href": function($document-id, $section-id) { "$app/frus-history/" || string-join(($document-id, $section-id), '/') }, @@ -267,6 +292,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:SECRETARY_BIOS_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:SECRETARY_BIOS_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:SECRETARY_BIOS_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:SECRETARY_BIOS_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:SECRETARY_BIOS_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:SECRETARY_BIOS_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:SECRETARY_BIOS_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/people/" || string-join(($document-id, $section-id), '/') }, @@ -314,6 +341,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:MILESTONES_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:MILESTONES_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:MILESTONES_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:MILESTONES_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:MILESTONES_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:MILESTONES_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:MILESTONES_COL|| '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/milestones/" || string-join(($document-id, $section-id), '/') }, @@ -326,6 +355,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:SHORT_HISTORY_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:SHORT_HISTORY_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:SHORT_HISTORY_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:SHORT_HISTORY_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:SHORT_HISTORY_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:SHORT_HISTORY_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:SHORT_HISTORY_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, $section-id), '/') }, @@ -338,6 +369,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:ADMINISTRATIVE_TIMELINE_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:ADMINISTRATIVE_TIMELINE_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:ADMINISTRATIVE_TIMELINE_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:ADMINISTRATIVE_TIMELINE_COL || '/' || $document-id || '.xml')/id('chapter_' || $section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/" || string-join(($document-id, substring-after($section-id, 'chapter_')), '/') }, @@ -351,6 +384,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:FAQ_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:FAQ_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FAQ_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:FAQ_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:FAQ_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:FAQ_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:FAQ_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/about/" || string-join(($document-id, $section-id), '/') }, @@ -362,6 +397,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:HAC_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:HAC_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:HAC_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:HAC_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:HAC_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:HAC_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:HAC_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/about/" || string-join(($document-id, $section-id), '/') }, @@ -373,6 +410,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:EDUCATION_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:EDUCATION_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:EDUCATION_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:EDUCATION_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:EDUCATION_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:EDUCATION_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:EDUCATION_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/education/modules/" || string-join(($document-id, $section-id), '#') }, @@ -387,6 +426,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:FRUS_HISTORY_MONOGRAPH_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:FRUS_HISTORY_MONOGRAPH_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:FRUS_HISTORY_MONOGRAPH_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { let $target-section-id := @@ -435,6 +476,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:VIETNAM_GUIDE_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:VIETNAM_GUIDE_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:VIETNAM_GUIDE_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:VIETNAM_GUIDE_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:VIETNAM_GUIDE_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:VIETNAM_GUIDE_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:VIETNAM_GUIDE_COL || '/' || $document-id || '.xml') }, "html-href": function($document-id, $section-id) { "$app/historicaldocuments/" || string-join(($document-id, $section-id), '/') }, @@ -446,6 +489,8 @@ declare variable $config:PUBLICATIONS := "collection": $config:VIEWS_FROM_EMBASSY_COL, "document-last-modified": function($document-id) { xmldb:last-modified($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') }, "section-last-modified": function($document-id, $section-id) { xmldb:last-modified($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') }, + "document-created": function($document-id) { xmldb:created($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') }, + "section-created": function($document-id, $section-id) {xmldb:created($config:VIEWS_FROM_EMBASSY_COL, $document-id || '.xml') }, "select-document": function($document-id) { doc($config:VIEWS_FROM_EMBASSY_COL || '/' || $document-id || '.xml') }, "select-section": function($document-id, $section-id) { doc($config:VIEWS_FROM_EMBASSY_COL || '/' || $document-id || '.xml')/id($section-id) }, "html-href": function($document-id, $section-id) { "$app/departmenthistory/wwi" }, diff --git a/modules/pages.xqm b/modules/pages.xqm index 56d371945..30be2c2e8 100644 --- a/modules/pages.xqm +++ b/modules/pages.xqm @@ -61,6 +61,12 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str $if-modified-since ge $last-modified else () + let $created := + if (exists($publication-id) and exists($document-id)) then + (: No need to truncate creation date; it'll be serialized in view.xql :) + pages:created($publication-id, $document-id, $section-id) + else + () return (: if the "If-Modified-Since" header in the client request is later than the : last-modified date, then halt further processing of the templates and simply @@ -91,8 +97,11 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str return ( - if (exists($last-modified)) then - request:set-attribute("hsgshell.last-modified", $last-modified) + if (exists($last-modified) and exists($created)) then + ( + request:set-attribute("hsgshell.last-modified", $last-modified), + request:set-attribute("hsgshell.created", $created) + ) else (), templates:process($node/*, map:merge(($model, $content))) @@ -106,6 +115,13 @@ declare function pages:last-modified($publication-id as xs:string, $document-id map:get($config:PUBLICATIONS, $publication-id)?document-last-modified($document-id) }; +declare function pages:created($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?) { + if ($section-id) then + map:get($config:PUBLICATIONS, $publication-id)?section-created($document-id, $section-id) + else + map:get($config:PUBLICATIONS, $publication-id)?document-created($document-id) +}; + declare function pages:load-xml($publication-id as xs:string, $document-id as xs:string, $section-id as xs:string?, $view as xs:string) { pages:load-xml($publication-id, $document-id, $section-id, $view, false()) }; diff --git a/modules/view.xql b/modules/view.xql index 533822448..a4ec2abd6 100644 --- a/modules/view.xql +++ b/modules/view.xql @@ -59,9 +59,13 @@ return ( templates:apply($content, $lookup, (), $config), let $last-modified := request:get-attribute("hsgshell.last-modified") + let $created := request:get-attribute("hsgshell.created") return - if (exists($last-modified)) then - app:set-last-modified($last-modified) + if (exists($last-modified) and exists($created)) then + ( + app:set-last-modified($last-modified), + app:set-created($created) + ) else () ) \ No newline at end of file From 043c955251175f80d9eac1bd3de548ebfd3fd883 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 4 Feb 2022 10:20:41 -0500 Subject: [PATCH 8/8] Ensure last modified is present before truncating --- modules/pages.xqm | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/pages.xqm b/modules/pages.xqm index 30be2c2e8..630286ac3 100644 --- a/modules/pages.xqm +++ b/modules/pages.xqm @@ -47,18 +47,19 @@ function pages:load($node as node(), $model as map(*), $publication-id as xs:str let $last-modified := if (exists($publication-id) and exists($document-id)) then pages:last-modified($publication-id, $document-id, $section-id) - (: For the purpose of comparing the resource's last modified date with the If-Modified-Since - : header supplied by the client, we must truncate any milliseconds from the last modified date. - : This is because HTTP-date is only specific to the second. - : @see https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1 :) - => format-dateTime("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]") - => xs:dateTime() else () let $if-modified-since := try { request:get-attribute("if-modified-since") => parse-ietf-date() } catch * { () } let $should-return-304 := if (exists($last-modified) and exists($if-modified-since)) then - $if-modified-since ge $last-modified + $if-modified-since ge + $last-modified + (: For the purpose of comparing the resource's last modified date with the If-Modified-Since + : header supplied by the client, we must truncate any milliseconds from the last modified date. + : This is because HTTP-date is only specific to the second. + : @see https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1 :) + => format-dateTime("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]") + => xs:dateTime() else () let $created :=