Skip to content

Commit f1fe8c5

Browse files
namedgraphnamedgraph
authored
Rate-limited LinkedDataClient (#203)
* Attempt to handle 429 responses by respecting `Retry-After` headers * Attempt to handle 429 responses in LinkedDataClient using RetryAfterHelper * Wrap `LinkedDataClient` methods into `RetryAfterHelper` * Comment * Comment * Max retry count support in RetryAfterHelper * Replaced HEAD requests with PUT If-None-Match: * * Fixed if/else condition * SNAPSHOT bump * Evaluate preconditions as soon as we have the `existingModel` * Comment * Simplified pre-condition evaluation * Fixed `lastModified` NPEs * HTTP tests for conditional `PUT` requests * HTTP test fixes * Web-Client bump Server bump * Fixed `getVariants()` call * Use HTMLMediaTypePredicate * Use getInternalResponse() to evaluate the preconditions * Override getResponseBuilder * Fixed test * New HTTP scripts for conditional requests * More HTTP tests for conditional requests
1 parent d3d7f21 commit f1fe8c5

27 files changed

Lines changed: 698 additions & 113 deletions

‎http-tests/document-hierarchy/DELETE-conditional-412.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# create container
11+
12+
slug="test"
13+
14+
container=$(create-container.sh \
15+
-f "$OWNER_CERT_FILE" \
16+
-p "$OWNER_CERT_PWD" \
17+
-b "$END_USER_BASE_URL" \
18+
--title "Test" \
19+
--slug "$slug" \
20+
--parent "$END_USER_BASE_URL")
21+
22+
# add an explicit read/write authorization for the owner because add-agent-to-group.sh won't work non-existing URI
23+
24+
create-authorization.sh \
25+
-b "$ADMIN_BASE_URL" \
26+
-f "$OWNER_CERT_FILE" \
27+
-p "$OWNER_CERT_PWD" \
28+
--label "Write base" \
29+
--agent "$AGENT_URI" \
30+
--to "$container" \
31+
--read \
32+
--write
33+
34+
# attempt to delete the graph using a random ETag value
35+
36+
(
37+
curl -k -w "%{http_code}\n" -o /dev/null -s \
38+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
39+
-H "Accept: application/n-triples" \
40+
-X DELETE \
41+
-H "If-Match: \"random-etag\"" \
42+
"$container"
43+
) \
44+
| grep -q "$STATUS_PRECONDITION_FAILED"

‎http-tests/document-hierarchy/DELETE-conditional.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# create container
11+
12+
slug="test"
13+
14+
container=$(create-container.sh \
15+
-f "$OWNER_CERT_FILE" \
16+
-p "$OWNER_CERT_PWD" \
17+
-b "$END_USER_BASE_URL" \
18+
--title "Test" \
19+
--slug "$slug" \
20+
--parent "$END_USER_BASE_URL")
21+
22+
# add an explicit read/write authorization for the owner because add-agent-to-group.sh won't work non-existing URI
23+
24+
create-authorization.sh \
25+
-b "$ADMIN_BASE_URL" \
26+
-f "$OWNER_CERT_FILE" \
27+
-p "$OWNER_CERT_PWD" \
28+
--label "Write base" \
29+
--agent "$AGENT_URI" \
30+
--to "$container" \
31+
--read \
32+
--write
33+
34+
# store the ETag value
35+
36+
etag_before=$(
37+
curl -k -i -f -s -G \
38+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
39+
-H "Accept: application/n-triples" \
40+
"$container" \
41+
| grep 'ETag' \
42+
| tr -d '\r' \
43+
| sed -En 's/^ETag: (.*)$/\1/p')
44+
45+
# delete the graph only if the ETag value matches
46+
47+
(
48+
curl -k -w "%{http_code}\n" -o /dev/null -f -s \
49+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
50+
-H "Accept: application/n-triples" \
51+
-X DELETE \
52+
-H "If-Match: ${etag_before}" \
53+
"$container"
54+
) \
55+
| grep -q "$STATUS_NO_CONTENT"

‎http-tests/document-hierarchy/GET-conditional-412.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# add agent to the readers group
11+
12+
add-agent-to-group.sh \
13+
-f "$OWNER_CERT_FILE" \
14+
-p "$OWNER_CERT_PWD" \
15+
--agent "$AGENT_URI" \
16+
"${ADMIN_BASE_URL}acl/groups/readers/"
17+
18+
# attempt to read the graph using a random ETag value
19+
20+
curl -k -w "%{http_code}\n" -o /dev/null -s -G \
21+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
22+
-H "Accept: application/n-triples" \
23+
"$END_USER_BASE_URL" \
24+
-H "If-Match: \"random-etag\"" \
25+
| grep -q "$STATUS_PRECONDITION_FAILED"

http-tests/document-hierarchy/GET-304.sh renamed to http-tests/document-hierarchy/GET-conditional.sh

File renamed without changes.

‎http-tests/document-hierarchy/HEAD-accept-lang.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ content_lang=$(curl -G --head -k -f -s -o /dev/null -D - \
2121
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
2222
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
2323
-H 'Accept-Language: es-ES,es;q=0.9,en-US;q=0.8,en;q=0.7' \
24-
--data-urlencode "accept=text/turtle" \
2524
"$END_USER_BASE_URL"\
2625
| tr -d '\r' | sed -En 's/^Content-Language: (.*)$/\1/p')
2726

‎http-tests/document-hierarchy/POST-conditional-412.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# add agent to the writers group
11+
12+
add-agent-to-group.sh \
13+
-f "$OWNER_CERT_FILE" \
14+
-p "$OWNER_CERT_PWD" \
15+
--agent "$AGENT_URI" \
16+
"${ADMIN_BASE_URL}acl/groups/writers/"
17+
18+
# attempt to replace the graph with a random ETag value
19+
20+
(
21+
curl -k -w "%{http_code}\n" -o /dev/null -s \
22+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
23+
-X POST \
24+
-H "Accept: application/n-triples" \
25+
-H "Content-Type: application/n-triples" \
26+
-H "If-Match: \"random-etag\"" \
27+
--data-binary @- \
28+
"$END_USER_BASE_URL" <<EOF
29+
<${END_USER_BASE_URL}> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/atomgraph/linkeddatahub/default#Root> .
30+
<${END_USER_BASE_URL}> <http://purl.org/dc/terms/title> "Root" .
31+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/default-predicate> "named object PUT" .
32+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/another-predicate> "another named object PUT" .
33+
EOF
34+
) \
35+
| grep -q "$STATUS_PRECONDITION_FAILED"

‎http-tests/document-hierarchy/POST-conditional.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# add agent to the writers group
11+
12+
add-agent-to-group.sh \
13+
-f "$OWNER_CERT_FILE" \
14+
-p "$OWNER_CERT_PWD" \
15+
--agent "$AGENT_URI" \
16+
"${ADMIN_BASE_URL}acl/groups/writers/"
17+
18+
# store the ETag value
19+
20+
etag_before=$(
21+
curl -k -i -f -s -G \
22+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
23+
-H "Accept: application/n-triples" \
24+
"$END_USER_BASE_URL" \
25+
| grep 'ETag' \
26+
| tr -d '\r' \
27+
| sed -En 's/^ETag: (.*)$/\1/p')
28+
29+
# append to the graph only if the ETag value matches
30+
31+
(
32+
curl -k -w "%{http_code}\n" -o /dev/null -f -s \
33+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
34+
-X POST \
35+
-H "Accept: application/n-triples" \
36+
-H "Content-Type: application/n-triples" \
37+
-H "If-Match: ${etag_before}" \
38+
--data-binary @- \
39+
"$END_USER_BASE_URL" <<EOF
40+
<${END_USER_BASE_URL}> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/atomgraph/linkeddatahub/default#Root> .
41+
<${END_USER_BASE_URL}> <http://purl.org/dc/terms/title> "Root" .
42+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/default-predicate> "named object PUT" .
43+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/another-predicate> "another named object PUT" .
44+
EOF
45+
) \
46+
| grep -q "$STATUS_NO_CONTENT"

‎http-tests/document-hierarchy/POST-ntriples-etag.sh‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,6 @@ add-agent-to-group.sh \
1515
--agent "$AGENT_URI" \
1616
"${ADMIN_BASE_URL}acl/groups/writers/"
1717

18-
# create public authorization
19-
20-
create-authorization.sh \
21-
-f "$OWNER_CERT_FILE" \
22-
-p "$OWNER_CERT_PWD" \
23-
-b "$ADMIN_BASE_URL" \
24-
--label "Public access authorization" \
25-
--agent-class 'http://xmlns.com/foaf/0.1/Agent' \
26-
--to "$END_USER_BASE_URL" \
27-
--read
28-
2918
etag_before=$(
3019
curl -k -i -f -s -G \
3120
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \

‎http-tests/document-hierarchy/PUT-conditional-412.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# add agent to the writers group
11+
12+
add-agent-to-group.sh \
13+
-f "$OWNER_CERT_FILE" \
14+
-p "$OWNER_CERT_PWD" \
15+
--agent "$AGENT_URI" \
16+
"${ADMIN_BASE_URL}acl/groups/writers/"
17+
18+
# attempt to replace the graph with a random ETag value
19+
20+
(
21+
curl -k -w "%{http_code}\n" -o /dev/null -s \
22+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
23+
-X PUT \
24+
-H "Accept: application/n-triples" \
25+
-H "Content-Type: application/n-triples" \
26+
-H "If-Match: \"random-etag\"" \
27+
--data-binary @- \
28+
"$END_USER_BASE_URL" <<EOF
29+
<${END_USER_BASE_URL}> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/atomgraph/linkeddatahub/default#Root> .
30+
<${END_USER_BASE_URL}> <http://purl.org/dc/terms/title> "Root" .
31+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/default-predicate> "named object PUT" .
32+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/another-predicate> "another named object PUT" .
33+
EOF
34+
) \
35+
| grep -q "$STATUS_PRECONDITION_FAILED"

‎http-tests/document-hierarchy/PUT-conditional.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
initialize_dataset "$END_USER_BASE_URL" "$TMP_END_USER_DATASET" "$END_USER_ENDPOINT_URL"
5+
initialize_dataset "$ADMIN_BASE_URL" "$TMP_ADMIN_DATASET" "$ADMIN_ENDPOINT_URL"
6+
purge_cache "$END_USER_VARNISH_SERVICE"
7+
purge_cache "$ADMIN_VARNISH_SERVICE"
8+
purge_cache "$FRONTEND_VARNISH_SERVICE"
9+
10+
# add agent to the writers group
11+
12+
add-agent-to-group.sh \
13+
-f "$OWNER_CERT_FILE" \
14+
-p "$OWNER_CERT_PWD" \
15+
--agent "$AGENT_URI" \
16+
"${ADMIN_BASE_URL}acl/groups/writers/"
17+
18+
# store the ETag value
19+
20+
etag_before=$(
21+
curl -k -i -f -s -G \
22+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
23+
-H "Accept: application/n-triples" \
24+
"$END_USER_BASE_URL" \
25+
| grep 'ETag' \
26+
| tr -d '\r' \
27+
| sed -En 's/^ETag: (.*)$/\1/p')
28+
29+
# replace the graph only if the ETag value matches
30+
31+
(
32+
curl -k -w "%{http_code}\n" -o /dev/null -f -s \
33+
-E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
34+
-X PUT \
35+
-H "Accept: application/n-triples" \
36+
-H "Content-Type: application/n-triples" \
37+
-H "If-Match: ${etag_before}" \
38+
--data-binary @- \
39+
"$END_USER_BASE_URL" <<EOF
40+
<${END_USER_BASE_URL}> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/atomgraph/linkeddatahub/default#Root> .
41+
<${END_USER_BASE_URL}> <http://purl.org/dc/terms/title> "Root" .
42+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/default-predicate> "named object PUT" .
43+
<${END_USER_BASE_URL}named-subject-put> <http://example.com/another-predicate> "another named object PUT" .
44+
EOF
45+
) \
46+
| grep -q "$STATUS_OK"

0 commit comments

Comments
 (0)