feat: new hash removal techniques with translate() + sample output

After removing the remote hashing call, it was time to replace the
URL-encoding. However, applying a consistent and unified technique to
both `rr:template` and `rml:reference` proved to be challenging, in
order to realize the following substitution requirements:

- `Q\{[^}]+\}` with nothing
- `\[(\d+)\]` with `_\1`
- `/` with `.`
- `encode-for-uri()` function with `replace()`
- adding periods (`.`) in the concat functions (revisiting the need for it, first)
- checking any double-encoding problems (`rr:template` already does URI-encoding)

String substitution techniques like these rely on "regular expressions",
aka regex. In regex, certain characters have special meaning, so
"escaping" them (typically with a backslash `\`) is necessary to force
literal interpretation by the regex engine.

This is made more complicated when multiple processing subsystems are
involved -- in this case the RML engine (RMLMapper), the RML
specifications it follows, Saxon the XPath processor, and Turtle the RDF
syntax used to write the RML rules.

Unfortunately, it did not seem possible to use escape characters or the
necessary characters in `rr:template` without escaping, such as curly
braces `{}` and square braces `[]`. Fortunately, after several rounds of
failed experiments, a trick with a pair of unassuming XPath functions
was discovered.

With `translate()` and `codepoints-to-string()`, one is able to replace
certain characters with alternatives, where the latter transforms a
given code to an ASCII character, and the former replaces a given
sequence of those with another. With this we could finally replace the
stubborn `Q{}` namespace qualifier in the XPath returned by `path()`,
when in an `rr:template`.

We develop here two separate implementations of URI rewrites: (i) a
standard regex substitution (with three levels of replacements) for
`rml:reference`, and (ii) a clever one with the aforementioned technique
for `rr:template` (that has the `{}` characters reserved for itself and
cannot interpret backslash escapes).

P.S: This technique will NOT work if in the source there is more than
one XML element with the same name but different prefix.

Co-authored-by: csnyulas <csongor.nyulas@gmail.com>

Author: schivmeister

src/mappings-can/Business-can.rml.ttl

@@ -113,7 +113,7 @@ tedm:MG-Identifier-identifier-Person_ND-UBO a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-UBO" ;
-            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_BeneficialOwnerTechnicalIdentifier_{encode-for-uri(path())}" ;
+            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_BeneficialOwnerTechnicalIdentifier_{replace(translate(path(), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
             rr:class adms:Identifier
         ] ;
     rr:predicateObjectMap
@@ -213,7 +213,7 @@ tedm:MG-Address-registeredAddress-Person_ND-UBOAddress a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-UBOAddress" ;
-            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_UBOAddress_{encode-for-uri(path())}" ;
+            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_UBOAddress_{replace(translate(path(), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
             rr:class locn:Address
         ] ;
      rr:predicateObjectMap

src/mappings-can/Contract-can.rml.ttl

@@ -35,7 +35,7 @@ tedm:MG-Identifier-identifier-Contract_ND-SettledContract  a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-SettledContract" ;
-            rml:reference "if (exists(efac:ContractReference/cbc:ID)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_SettledContractIdentifier_' || encode-for-uri(path(.)) else null" ;
+            rml:reference "if (exists(efac:ContractReference/cbc:ID)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_SettledContractIdentifier_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class adms:Identifier
         ] ;
     rr:predicateObjectMap
@@ -217,7 +217,7 @@ tedm:MG-Fund_ND-SettledContract  a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-SettledContract" ;
-             rml:reference "if (exists(efac:Funding)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || encode-for-uri(path(.)) else null" ;
+             rml:reference "if (exists(efac:Funding)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:Fund
         ] ;
     .
@@ -232,7 +232,7 @@ tedm:MG-Fund-isFundedBy-Contract_ND-SettledContract   a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-SettledContract" ;
-             rml:reference "if (exists(efac:Funding)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || encode-for-uri(path(.)) else null" ;
+             rml:reference "if (exists(efac:Funding)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:Fund
         ] ;
     rr:predicateObjectMap
@@ -263,7 +263,7 @@ tedm:MG-Identifier-identifier-Fund-isFundedBy-Contract_ND-ContractEUFunds a rr:T
     rr:subjectMap
         [
             rdfs:label "ND-ContractEUFunds" ;
-            rml:reference "if (exists(efbc:FinancingIdentifier)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFundIdentifier_' || encode-for-uri(path(..)) else null" ;
+            rml:reference "if (exists(efbc:FinancingIdentifier)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFundIdentifier_' || replace(replace(replace(path(..), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class adms:Identifier
         ] ;
     rr:predicateObjectMap
@@ -298,7 +298,7 @@ tedm:MG-Fund-isFundedBy-Contract_ND-ContractEUFunds a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-ContractEUFunds" ;
-            rml:reference "if((exists(cbc:FundingProgramCode)) or (exists(cbc:Description))) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || encode-for-uri(path(..)) else null" ;
+            rml:reference "if((exists(cbc:FundingProgramCode)) or (exists(cbc:Description))) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || replace(replace(replace(path(..), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:Fund
         ] ;
 
@@ -609,7 +609,7 @@ tedm:MG-langString-description-Fund-isFundedBy-Contract_ND-ContractEUFunds a rr:
     rr:subjectMap [
         rr:class epo-not:Fund ;
         rdfs:label "ND-ContractEUFunds" ;
-        rml:reference "'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || encode-for-uri(path(../..))" ;
+        rml:reference "'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractFund_' || replace(replace(replace(path(../..), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_')" ;
     ] ;
     rr:predicateObjectMap [
         tedm:minSDKVersion "1.9.1" ;

src/mappings-can/ContractModificationInformation.rml.ttl

@@ -36,7 +36,7 @@ tedm:MG-ContractModificationInformation_ND-ModificationReason a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-ModificationReason" ;
-            rml:reference "if (exists(cbc:ReasonCode) or exists(efbc:ReasonDescription)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractModificationInformation_' || encode-for-uri(path(..)) else null" ;
+            rml:reference "if (exists(cbc:ReasonCode) or exists(efbc:ReasonDescription)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractModificationInformation_' || replace(replace(replace(path(..), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo-con:ContractModificationInformation
         ] ;
     rr:predicateObjectMap
@@ -81,7 +81,7 @@ tedm:MG-ContractAmendment-concernsContractAmendment-ContractModificationInformat
     rr:subjectMap
         [
             rdfs:label "ND-ModificationReason" ;
-            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractAmendment_{encode-for-uri(path(..))}" ;
+            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractAmendment_{replace(translate(path(..), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
             rr:class epo-con:ContractAmendment
         ] ;
 .
@@ -96,7 +96,7 @@ tedm:MG-langString-hasModificationReasonDescription-ContractModificationInformat
     rr:subjectMap [
         rr:class epo-con:ContractModificationInformation ;
         rdfs:label "ND-ModificationReason" ;
-        rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractModificationInformation_{encode-for-uri(path(../..))}" ;
+        rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractModificationInformation_{replace(translate(path(../..), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
     ] ;
     rr:predicateObjectMap [
         rr:predicate  epo:hasModificationReasonDescription ;
@@ -134,7 +134,7 @@ tedm:MG-langString-hasModificationReasonDescription-ContractModificationInformat
     rr:subjectMap [
         rr:class epo-con:ContractModificationInformation ;
         rdfs:label "ND-ModificationReason" ;
-        rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractModificationInformation_{encode-for-uri(path(../..))}" ;
+        rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractModificationInformation_{replace(translate(path(../..), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
     ] ;
     rr:predicateObjectMap [
         rr:predicate  epo:hasModificationDescription ;

src/mappings-can/GreenProcurement-can.rml.ttl

@@ -35,7 +35,7 @@ tedm:MG-GreenProcurement-concernsGreenProcurement-VehicleInformation_ND-Strategi
     rr:subjectMap
         [
             rdfs:label "ND-StrategicProcurementInformationLotResult" ;
-            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_GreenProcurement_{encode-for-uri(path())}" ;
+            rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_GreenProcurement_{replace(translate(path(), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
             rr:class epo:GreenProcurement
         ] ;
     .

src/mappings-can/Lot-compl.rml.ttl

@@ -35,7 +35,7 @@ tedm:MG-ContractTerm-foreseesContractSpecificTerm-Lot_ND-LotDuration a rr:Triple
     rr:subjectMap
         [
             rdfs:label "ND-LotDuration";
-            rml:reference "if (exists(cbc:Description)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractTerm_' || encode-for-uri(path()) else null" ;
+            rml:reference "if (exists(cbc:Description)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractTerm_' || replace(replace(replace(path(), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:ContractTerm
         ] ;
 .
@@ -81,7 +81,7 @@ tedm:MG-langString-hasExceedingDurationJustification-ContractTerm_ND-LotDuration
     rr:subjectMap [
         rr:class epo:ContractTerm ;
         rdfs:label "ND-LotDuration" ;
-        rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractTerm_{encode-for-uri(path(..))}" ;
+        rr:template "http://data.europa.eu/a4g/resource/id_{replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-')}_ContractTerm_{replace(translate(path(..), concat(codepoints-to-string(123), codepoints-to-string(125), '/[]'), '  _'), 'Q .*? ', '')}" ;
     ] ;
     rr:predicateObjectMap [
         rr:predicate   epo:hasExceedingDurationJustification  ;

src/mappings-can/LotAwardOutcome-can.rml.ttl

@@ -123,7 +123,7 @@ tedm:MG-MonetaryValue-hasApproximateFrameworkAgreementValue-LotAwardOutcome_ND-L
    rr:subjectMap
         [
             rdfs:label "ND-LotResultFAValues";
-            rml:reference "if (exists(efbc:ReestimatedValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_MonetaryValue_' || encode-for-uri(path(.)) else null" ;
+            rml:reference "if (exists(efbc:ReestimatedValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_MonetaryValue_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:MonetaryValue
         ] ;
    rr:predicateObjectMap
@@ -168,7 +168,7 @@ tedm:MG-MonetaryValue-hasFrameworkAgreementMaximumValue-LotAwardOutcome_ND-LotRe
     rr:subjectMap
         [
             rdfs:label "ND-LotResultFAValues";
-            rml:reference "if (exists(cbc:MaximumValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_FrameworkAgreementMaximumMonetaryValue_' || encode-for-uri(path(.)) else null" ;
+            rml:reference "if (exists(cbc:MaximumValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_FrameworkAgreementMaximumMonetaryValue_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:MonetaryValue
         ] ;
     rr:predicateObjectMap

src/mappings-can/LotGroup-can.rml.ttl

@@ -173,7 +173,7 @@ tedm:MG-LotGroup_ND-NoticeResultGroupFA  a rr:TriplesMap ;
 #    rr:subjectMap
 #        [
 #            rdfs:label "ND-NoticeResultGroupFA";
-#            rml:reference "if (exists(efbc:GroupFrameworkReestimatedValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_MonetaryValue_' || encode-for-uri(path()) else null" ;
+#            rml:reference "if (exists(efbc:GroupFrameworkReestimatedValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_MonetaryValue_' || replace(replace(replace(path(), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
 #            rr:class epo:MonetaryValue
 #        ] ;
 #   rr:predicateObjectMap

src/mappings-can/LotGroupAwardInformation-can.rml.ttl

@@ -35,7 +35,7 @@ tedm:MG-MonetaryValue-hasGroupFrameworkAgreementMaximumValue-LotGroupAwardInform
    rr:subjectMap
         [
             rdfs:label "ND-NoticeResultGroupFA";
-            rml:reference "if (exists(efbc:GroupFrameworkMaximumValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_MonetaryValue_' || encode-for-uri(path(.)) else null" ;
+            rml:reference "if (exists(efbc:GroupFrameworkMaximumValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_MonetaryValue_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:MonetaryValue
         ] ;
    rr:predicateObjectMap
@@ -77,7 +77,7 @@ tedm:MG-LotGroupAwardInformation_ND-NoticeResultGroupFA a rr:TriplesMap ;
     rr:subjectMap
         [
             rdfs:label "ND-LotsGroupFA";
-              rml:reference "if (exists(efbc:GroupFrameworkMaximumValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_LotGroupAwardInformation_' || encode-for-uri(path(.)) else null" ;
+              rml:reference "if (exists(efbc:GroupFrameworkMaximumValueAmount)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_LotGroupAwardInformation_' || replace(replace(replace(path(.), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:LotGroupAwardInformation
         ] ;
     rr:predicateObjectMap

src/mappings-can/PaymentExecutor-can.rml.ttl

@@ -36,7 +36,7 @@ tedm:MG-ContractTerm-isSubjectToContractSpecificTerm-Contract_ND-PayerParty  a r
         [
             rdfs:label "ND-PayerParty" ;
             rdfs:comment "LotResultTechnicalIdentifier of MG-LotAwardOutcome under ND-LotResult" ;
-            rml:reference "if (exists(../../efac:SettledContract/cbc:ID)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractTerm_' || encode-for-uri(path()) else null" ;
+            rml:reference "if (exists(../../efac:SettledContract/cbc:ID)) then 'http://data.europa.eu/a4g/resource/id_' || replace(replace(/*/cbc:ID[@schemeName='notice-id'], ' ', '-' ), '/' , '-') || '_ContractTerm_' || replace(replace(replace(path(), 'Q\\{[^}]+\\}', ''), '\\[([0-9]+)\\]', '$1'), '/', '_') else null" ;
             rr:class epo:ContractTerm
         ] ;
     rr:predicateObjectMap