Various updates

modules/app.xqm

@@ -2,6 +2,7 @@ xquery version "3.1" encoding "UTF-8";
 
 module namespace app="http://xquery.weber-gesamtausgabe.de/modules/app";
 
+declare namespace err="http://www.w3.org/2005/xqt-errors";
 declare namespace tei="http://www.tei-c.org/ns/1.0";
 declare namespace mei="http://www.music-encoding.org/ns/mei";
 declare namespace util="http://exist-db.org/xquery/util";
@@ -820,7 +821,7 @@ declare
                     switch($provider)
                     case 'osm' return 'https://www.openstreetmap.org/?mlat=' || $latLon[1] || '&mlon=' || $latLon[2] || '&zoom=11'
                     case 'google' return 'https://www.google.com/maps/@?api=1&map_action=map&zoom=12&basemap=terrain&center=' || string-join($latLon, ',')
-                    case 'geoNames' return 'http://geonames.org/' || $model?geonames-id
+                    case 'geoNames' return 'https://geonames.org/' || $model?geonames-id
                     default return ''
                 },
                 switch($provider)
@@ -1130,9 +1131,9 @@ declare
                 'Der Text unter der Überschrift „Wikipedia“ entstammt dem Artikel „',
                 <a xmlns="http://www.w3.org/1999/xhtml" href='{$model('wikiUrl')}' title='Wikipedia Artikel zu "{$model('wikiName')}"'>{$model('wikiName')}</a>,
                 '“ aus der freien Enzyklopädie ',
-                <a xmlns="http://www.w3.org/1999/xhtml" href="http://de.wikipedia.org" title="Wikipedia Hauptseite">Wikipedia</a>, 
+                <a xmlns="http://www.w3.org/1999/xhtml" href="https://de.wikipedia.org" title="Wikipedia Hauptseite">Wikipedia</a>,
                 ' und steht unter der ',
-                <a xmlns="http://www.w3.org/1999/xhtml" href="http://creativecommons.org/licenses/by-sa/3.0/deed.de">CC-BY-SA-Lizenz</a>,
+                <a xmlns="http://www.w3.org/1999/xhtml" href="https://creativecommons.org/licenses/by-sa/3.0/deed.de">CC-BY-SA-Lizenz</a>,
                 '. In der Wikipedia findet sich auch die ',
                 <a xmlns="http://www.w3.org/1999/xhtml" href="{concat(replace($model('wikiUrl'), 'wiki/', 'w/index.php?title='), '&amp;action=history')}" title='Autoren und Versionsgeschichte des Wikipedia Artikels zu "{$model('wikiName')}"'>Versionsgeschichte mitsamt Autorennamen</a>,
                 ' für diesen Artikel.'
@@ -1141,9 +1142,9 @@ declare
                 'The text under the headline “Wikipedia” is taken from the article “',
                 <a xmlns="http://www.w3.org/1999/xhtml" href='{$model('wikiUrl')}' title='Wikipedia article for {$model('wikiName')}'>{$model('wikiName')}</a>,
                 '” from ',
-                <a xmlns="http://www.w3.org/1999/xhtml" href="http://en.wikipedia.org">Wikipedia</a>,
+                <a xmlns="http://www.w3.org/1999/xhtml" href="https://en.wikipedia.org">Wikipedia</a>,
                 ' the free encyclopedia, and is released under a ',
-                <a xmlns="http://www.w3.org/1999/xhtml" href="http://creativecommons.org/licenses/by-sa/3.0/deed.en">CC-BY-SA-license</a>,
+                <a xmlns="http://www.w3.org/1999/xhtml" href="https://creativecommons.org/licenses/by-sa/3.0/deed.en">CC-BY-SA-license</a>,
                 '. You will find the ',
                 <a xmlns="http://www.w3.org/1999/xhtml" href="{concat(replace($model('wikiUrl'), 'wiki/', 'w/index.php?title='), '&amp;action=history')}" title="Authors and revision history of the Wikipedia Article for {$model('wikiName')}">revision history along with the authors</a>,
                 ' of this article in Wikipedia.'

modules/beacon.xql

@@ -40,13 +40,13 @@ declare function beacon:new($type as xs:string) as xs:string {
         default return ()
     let $feed := 
         switch($type)
-        case 'pnd' return '#FEED: http://weber-gesamtausgabe.de/pnd_beacon.txt'
-        case 'gkd' return '#FEED: http://weber-gesamtausgabe.de/gkd_beacon.txt'
-        case 'works' return '#FEED: http://weber-gesamtausgabe.de/works_beacon.txt'
+        case 'pnd' return '#FEED: https://weber-gesamtausgabe.de/pnd_beacon.txt'
+        case 'gkd' return '#FEED: https://weber-gesamtausgabe.de/gkd_beacon.txt'
+        case 'works' return '#FEED: https://weber-gesamtausgabe.de/works_beacon.txt'
         default return ()
     let $header := (
         '#FORMAT: BEACON',
-        '#PREFIX: http://d-nb.info/gnd/',
+        '#PREFIX: https://d-nb.info/gnd/',
         '#VERSION: 0.1',
         '#TARGET: https://weber-gesamtausgabe.de/de/gnd/{ID}',
         $feed,

modules/external-requests.xqm

@@ -5,6 +5,7 @@ xquery version "3.1" encoding "UTF-8";
  :)
 module namespace er="http://xquery.weber-gesamtausgabe.de/modules/external-requests";
 
+declare namespace err="http://www.w3.org/2005/xqt-errors";
 declare namespace tei="http://www.tei-c.org/ns/1.0";
 declare namespace mei="http://www.music-encoding.org/ns/mei";
 declare namespace wega="http://www.weber-gesamtausgabe.de";
@@ -49,8 +50,8 @@ declare function er:grabExternalResource($resource as xs:string, $id as xs:strin
         case 'wikipedia' return (er:grab-external-resource-wikidata($id, 'gnd')//sr:binding[@name=('article' || upper-case($lang))]/sr:uri/data(.))[1]
         case 'dnb' return concat('https://d-nb.info/gnd/', $id, '/about/rdf')
         case 'viaf' return concat('https://viaf.org/viaf/', $id, '.rdf')
-        case 'geonames' return concat('http://sws.geonames.org/', $id, '/about.rdf') (: $id is actually the geonames ID :)
-        case 'dbpedia' return concat('http://www.wikidata.org/entity/', $id, '.rdf') (: $id is actually the dbpedia(wikidata?) ID :)
+        case 'geonames' return concat('https://sws.geonames.org/', $id, '/about.rdf') (: $id is actually the geonames ID :)
+        case 'dbpedia' return concat('https://www.wikidata.org/entity/', $id, '.rdf') (: $id is actually the dbpedia(wikidata?) ID :)
         case 'deutsche-biographie' return 'https://www.deutsche-biographie.de/gnd' || $id || '.html'
         default return config:get-option($resource) || $id
     let $fileName := string-join(($id, $lang, 'xml'), '.')
@@ -80,7 +81,7 @@ declare function er:grab-external-resource-via-beacon($beaconProvider as xs:stri
  :)
 declare function er:grab-external-resource-wikidata($id as xs:string, $authority-provider as xs:string) as element(er:response)? {
     let $uri := 
-        if($authority-provider eq 'wikidata') then xs:anyURI('http://www.wikidata.org/entity/' || $id || '.rdf')
+        if($authority-provider eq 'wikidata') then xs:anyURI('https://www.wikidata.org/entity/' || $id || '.rdf')
         else er:wikidata-url($id, $authority-provider)
     let $fileName := util:hash($uri, 'md5') || '.xml'
     return
@@ -126,7 +127,7 @@ declare function er:lookup-gnd-from-beaconURI($beaconURI as xs:anyURI, $gnd as x
  :      an `@rdf:resource` attribute which indicates the resource to fetch
  :  @return an er:response element if successful, the empty sequence otherwise. For a description of the `er:response` element
  :      see http://expath.org/modules/http-client/
-~:)
+ :)
 declare function er:resolve-rdf-resource($elem as element()) as element(er:response)? {
     let $uri := 
         if(starts-with($elem/@rdf:resource, 'https://d-nb.info/gnd')) then ($elem/@rdf:resource || '/about/lds.rdf')
@@ -140,9 +141,12 @@ declare function er:resolve-rdf-resource($elem as element()) as element(er:respo
 };
 
 (:~
- : Helper function for wega:grabExternalResource()
+ : Fetch an external resource via HTTP GET request and return the response wrapped in a wega:externalResource element.
+ : The function constructs an HTTP GET request for the given URL, sends the request through the EXPath http-client module,
+ : and captures the response.
+ : The response is then wrapped in a wega:externalResource element, which includes the date of retrieval.
+ : If the request fails (e.g., due to a timeout), an appropriate log message is recorded.
  :
- : @author Peter Stadler 
  : @param $url the URL as xs:anyURI
  : @return element wega:externalResource, a wrapper around er:response
  :)
@@ -173,12 +177,9 @@ declare function er:wikimedia-iiif($wikiFilename as xs:string) as map(*)* {
     (: zu IIIF@Wikipedia: siehe https://commons.wikimedia.org/wiki/Commons:International_Image_Interoperability_Framework :)
     let $escapedWikiFilename := replace($wikiFilename, ' ', '_')
     let $url := 'https://tools.wmflabs.org/zoomviewer/proxy.php?iiif=' || $escapedWikiFilename || '/info.json'
-    let $lease := function($currentDateTimeOfFile as xs:dateTime?) as xs:boolean { wega-util:check-if-update-necessary($currentDateTimeOfFile, ()) }
     let $fileName := util:hash($escapedWikiFilename, 'md5') || '.xml'
-    let $onFailureFunc := function($errCode, $errDesc) {
-        wega-util:log-to-file('warn', string-join(($errCode, $errDesc), ' ;; '))
-    }
-    let $response := mycache:doc(str:join-path-elements(($config:tmp-collection-path, 'iiif', $fileName)), er:http-get#1, xs:anyURI($url), $lease, $onFailureFunc)
+    let $localFilePath := str:join-path-elements(($config:tmp-collection-path, 'iiif', $fileName))
+    let $response := er:cached-external-request(xs:anyURI($url), $localFilePath)
     return 
         if($response//er:response/@statusCode eq '200') then 
             try { parse-json(util:binary-to-string($response//er:body)) }
@@ -205,24 +206,34 @@ declare function er:cached-external-request($uri as xs:anyURI, $localFilepath as
 };
 
 (:~
- : Make a (locally) cached request to an external URI
- : This is the full fledged 4-arity version
+ : Make a (locally) cached request to an external URI.
+ : This is the full fledged 4-arity version.
+ : The function makes use of `er:http-get#1` to retrieve the external data
+ : but will only cache responses with status codes 2xx, or 4xx.
  :
  : @param $uri the external URI to fetch
  : @param $localFilepath the filepath to store the cached document
- : @param $lease a function to determine wether the cache should be updated. Must return a boolean value
+ : @param $lease a function to determine whether the cache should be updated. Must return a boolean value
  : @param $onFailureFunc an on-error function that's passed on to the underlying mycache:doc() function 
  : @return a er:response element with the response stored within er:body if successful, the empty sequence otherwise
  :)
 declare function er:cached-external-request($uri as xs:anyURI, $localFilepath as xs:string, $lease as function() as xs:boolean, $onFailureFunc as function() as item()*) as element(er:response)? {
-    mycache:doc($localFilepath, er:http-get#1, $uri, $lease, $onFailureFunc)//er:response[@statusCode = '200']
+    let $http-get := function($url as xs:anyURI) as element(wega:externalResource)? {
+        (: locally modify `er:http-get#1` to not cache failed requests (e.g. timeouts) :)
+        er:http-get($url)//er:response[matches(@statusCode, '^[24]\d+')]/parent::wega:externalResource
+    }
+    return
+        try {
+            mycache:doc($localFilepath, $http-get, $uri, $lease, $onFailureFunc)//er:response[@statusCode = '200']
+        }
+        catch * {()}
 };
 
 
 (:~
  : construct wikidata query URL
  : Helper function for `er:grab-external-resource-wikidata()`
-~:)
+ :)
 declare %private function er:wikidata-url($id as xs:string, $authority-provider as xs:string) as xs:anyURI {
     (:  
     see https://query.wikidata.org/ 
@@ -298,7 +309,7 @@ declare %private function er:parse-beacon($beaconURI as xs:anyURI) as element(er
  :
  :  @param $gnd a GND identifier
  :  @return the corresponding VIAF identifier(s) as string(s)
-~:)
+ :)
 declare function er:gnd2viaf($gnd as xs:string) as xs:string* {
     er:translate-authority-id(<tei:idno type="gnd">{$gnd}</tei:idno>, 'viaf')
 };

modules/img.xqm

@@ -621,7 +621,8 @@ declare function img:iiif-canvas($graphic as element(tei:graphic)) as map(*) {
     let $canvas-id := replace($manifest-id, 'manifest.json', 'canvas/') || encode-for-uri($page-label)
     let $image-info :=
         try {
-            er:http-get(xs:anyURI($image-id || '/info.json'))//*:response => util:base64-decode() => parse-json() (: why is this not cached? – the wrapper request to the manifest.json is cached! :)
+            (: this request does not need to be cached since the wrapper request to the manifest.json is already cached at `view-json.xql`! :)
+            er:http-get(xs:anyURI($image-id || '/info.json'))//er:response => util:base64-decode() => parse-json()
         }
         catch * {
             wega-util:log-to-file('error', 'failed to fetch image info for ' || $image-id)

modules/wdt.xqm

@@ -46,7 +46,7 @@ declare function wdt:orgs($item as item()*) as map(*) {
             crud:data-collection('orgs')[descendant::tei:org][descendant-or-self::tei:orgName]
         },
         'init-sortIndex' : function() as item()* {
-            sort:create-index-callback('orgs', wdt:orgs(())('init-collection')(), function($node) { wdt:orgs($node)('title')('txt') }, ())
+            sort:create-index-callback('orgs', wdt:orgs(())('init-collection')(), function($node) { wdt:orgs($node)('title')('txt') || $node/root()/tei:org/@xml:id }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
             for $this.item in $item
@@ -191,7 +191,7 @@ declare function wdt:letters($item as item()*) as map(*) {
                 let $normDate := query:get-normalized-date($node)
                 let $n :=  functx:pad-integer-to-length(($node//tei:correspAction[@type='sent']/tei:date)[1]/data(@n), 4)
                 return
-                    (if(exists($normDate)) then $normDate else 'xxxx-xx-xx') || $n
+                    (if(exists($normDate)) then $normDate else 'xxxx-xx-xx') || $n || $node/root()/tei:TEI/@xml:id
             }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
@@ -431,7 +431,7 @@ declare function wdt:diaries($item as item()*) as map(*) {
             crud:data-collection('diaries')[tei:ab/@where]
         },
         'init-sortIndex' : function() as item()* {
-            sort:create-index-callback('diaries', wdt:diaries(())('init-collection')(), function($node) { query:get-normalized-date($node) }, ())
+            sort:create-index-callback('diaries', wdt:diaries(())('init-collection')(), function($node) { query:get-normalized-date($node) || $node/root()/tei:ab/@xml:id }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
             let $lang := config:guess-language(())
@@ -499,7 +499,7 @@ declare function wdt:news($item as item()*) as map(*) {
             crud:data-collection('news')[descendant::tei:text]
         },
         'init-sortIndex' : function() as item()* {
-            sort:create-index-callback('news', wdt:news(())('init-collection')(), function($node) { $node//tei:date[parent::tei:publicationStmt]/xs:dateTime(@when) }, ())
+            sort:create-index-callback('news', wdt:news(())('init-collection')(), function($node) { $node//tei:date[parent::tei:publicationStmt]/xs:dateTime(@when) || $node/root()/tei:TEI/@xml:id }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
             for $this.item in $item
@@ -646,7 +646,8 @@ declare function wdt:biblio($item as item()*) as map(*) {
                 let $date := query:get-normalized-date($node)
                 return
                     (if(exists($date)) then $date else '0000') ||
-                    tokenize(($node//tei:author)[1], '\s+')[last()]
+                    tokenize(($node//tei:author)[1], '\s+')[last()] ||
+                    $node/root()/tei:biblStruct/@xml:id
                 }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
@@ -697,7 +698,7 @@ declare function wdt:places($item as item()*) as map(*) {
             crud:data-collection('places')[descendant::tei:placeName]
         },
         'init-sortIndex' : function() as item()* {
-            sort:create-index-callback('places', wdt:places(())('init-collection')(), function($node) { str:normalize-space($node//tei:placeName[@type='reg']) }, ())
+            sort:create-index-callback('places', wdt:places(())('init-collection')(), function($node) { str:normalize-space($node//tei:placeName[@type='reg']) || $node/root()/tei:place/@xml:id }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
             for $this.item in $item
@@ -759,7 +760,8 @@ declare function wdt:sources($item as item()*) as map(*) {
                 let $date := query:get-normalized-date($node)
                 return
                     (if(exists($date)) then $date else 'xxxx-xx-xx') ||
-                    $node//*:title[1]
+                    $node//*:title[1] ||
+                    $node/root()/mei:manifestation/@xml:id
                 }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
@@ -863,7 +865,7 @@ declare function wdt:documents($item as item()*) as map(*) {
                 let $normDate := query:get-normalized-date($node)
                 let $title := replace(str:normalize-space(($node//tei:fileDesc/tei:titleStmt/tei:title[@level = 'a'])[1] ), '^(Der|Die|Das|Eine?)\s', '')
                 return 
-                    (if(exists($normDate)) then $normDate else 'xxxx-xx-xx') || $title
+                    (if(exists($normDate)) then $normDate else 'xxxx-xx-xx') || $title || $node/root()/tei:TEI/@xml:id
             }, ())
         },
         'title' : function($serialization as xs:string) as item()* {
@@ -1113,7 +1115,7 @@ declare %private function wdt:sort-key-person($node as node()) as xs:string? {
         else str:normalize-space($node//tei:persName[@type='reg']/tei:surname[1])
     let $name := str:normalize-space($node//tei:persName[@type='reg'])
     return 
-        lower-case(replace(str:strip-diacritics($sortName || $name), "'", ""))
+        lower-case(replace(str:strip-diacritics($sortName || $name), "'", "")) || $node/root()/tei:persName/@xml:id
 };
 
 (:~

testing/expected-results/places/A130002.html

@@ -300,7 +300,7 @@ <h2>Basisdaten</h2>
                                 <i class="fa fa-globe"></i>
                             </div>
                             <div class="media-body"> 
-                                <span>GeoNames ID:</span> <a href="http://geonames.org/2950159">2950159</a> 
+                                <span>GeoNames ID:</span> <a href="https://geonames.org/2950159">2950159</a>
                             </div>
                         </li>
                         <li class="media">

testing/expected-results/places/A130005.html

@@ -300,7 +300,7 @@ <h2>Basisdaten</h2>
                                 <i class="fa fa-globe"></i>
                             </div>
                             <div class="media-body"> 
-                                <span>GeoNames ID:</span> <a href="http://geonames.org/2879139">2879139</a> 
+                                <span>GeoNames ID:</span> <a href="https://geonames.org/2879139">2879139</a>
                             </div>
                         </li>
                         <li class="media">

testing/expected-results/places/A130008.html

@@ -300,7 +300,7 @@ <h2>Basisdaten</h2>
                                 <i class="fa fa-globe"></i>
                             </div>
                             <div class="media-body"> 
-                                <span>GeoNames ID:</span> <a href="http://geonames.org/2988507">2988507</a> 
+                                <span>GeoNames ID:</span> <a href="https://geonames.org/2988507">2988507</a>
                             </div>
                         </li>
                         <li class="media">

testing/expected-results/places/A130010.html

@@ -308,7 +308,7 @@ <h2>Basisdaten</h2>
                                 <i class="fa fa-globe"></i>
                             </div>
                             <div class="media-body"> 
-                                <span>GeoNames ID:</span> <a href="http://geonames.org/2873891">2873891</a> 
+                                <span>GeoNames ID:</span> <a href="https://geonames.org/2873891">2873891</a>
                             </div>
                         </li>
                         <li class="media">