Merge pull request #437 from C85297/292-paths-filter

Fix tags filtering resulting in paths filter being ignored

Author: taylorsilva

README.md

@@ -63,6 +63,8 @@ Tracks the commits in a [git](http://git-scm.com/) repository.
   compatible (extended matching enabled, matches entire lines only). Ignored if
   `tag_filter` is also specified.
 
+* `tag_behaviour`: *Optional.* If `match_tagged` (the default), then the resource will only detect commits that are tagged with a tag matching `tag_regex` and `tag_filter`, and match all other filters. If `match_tag_ancestors`, then the resource will only detect commits matching all other filters and that are ancestors of a commit that are tagged with a tag matching `tag_regex` and `tag_filter`.
+
 * `fetch_tags`: *Optional.* If `true` the flag `--tags` will be used to fetch
   all tags in the repository. If `false` no tags will be fetched.
 
@@ -146,7 +148,7 @@ Tracks the commits in a [git](http://git-scm.com/) repository.
     skipped
   * `include_all_match`: *Optional.* Boolean wheater it should match all the include filters "AND", default: false
 
-  **Note**: *You must escape any regex sensitive characters, since the string is used as a regex filter.*  
+  **Note**: *You must escape any regex sensitive characters, since the string is used as a regex filter.*
   For example, using `[skip deploy]` or `[deploy skip]` to skip non-deployment related commits in a deployment pipeline:
 
   ```yaml
@@ -397,7 +399,7 @@ will stop the build.
 Run the tests with the following command:
 
 ```sh
-docker build -t git-resource --target tests --build-arg base_image=paketobuildpacks/run-jammy-base:latest .
+docker build -t git-resource --target tests --build-arg base_image=paketobuildpacks/run-noble-base:latest .
 
 ```
 

assets/check

@@ -24,6 +24,7 @@ paths="$(jq -r '(.source.paths // ["."])[]' <<< "$payload")" # those "'s are imp
 ignore_paths="$(jq -r '":!" + (.source.ignore_paths // [])[]' <<< "$payload")" # these ones too
 tag_filter=$(jq -r '.source.tag_filter // ""' <<< "$payload")
 tag_regex=$(jq -r '.source.tag_regex // ""' <<< "$payload")
+tag_behaviour=$(jq -r '.source.tag_behaviour // "match_tagged"' <<< "$payload")
 git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload")
 ref=$(jq -r '.version.ref // ""' <<< "$payload")
 skip_ci_disabled=$(jq -r '.source.disable_ci_skip // false' <<< "$payload")
@@ -65,12 +66,20 @@ then
 fi
 
 tagflag=""
-if [ -n "$tag_filter" ] || [ -n "$tag_regex" ] ; then
+if [ -n "$tag_filter" ] && [ -n "$tag_regex" ] ; then
+  echo "Cannot provide both tag_filter and tag_regex"
+  exit 1
+elif [ -n "$tag_filter" ] || [ -n "$tag_regex" ] ; then
   tagflag="--tags"
 else
   tagflag="--no-tags"
 fi
 
+if [ "$tag_behaviour" != "match_tagged" ] && [ "$tag_behaviour" != "match_tag_ancestors" ]; then
+  echo "Invalid tag_behaviour. Must be one of 'match_tagged' or 'match_tag_ancestors'."
+  exit 1
+fi
+
 for filter in "$filter_include" "$filter_exclude"
 do
   if jq -e 'type != "array"' <<<"$filter" &>/dev/null
@@ -81,6 +90,11 @@ do
   fi
 done
 
+if [ "$version_depth" -le 0 ]; then
+  echo "Invalid version_depth. Must be <= 0."
+  exit 1
+fi
+
 # We're just checking for commits; we don't ever need to fetch LFS files here!
 export GIT_LFS_SKIP_SMUDGE=1
 
@@ -167,58 +181,88 @@ lines_including_and_after() {
   sed -ne "/$escaped_string/,$ p"
 }
 
-get_commit(){
-  for tag in $*; do
-    commit=$(git rev-list -n 1 $tag)
-    jq -n '{ref: $tag, commit: $commit}' --arg tag $tag --arg commit $commit
-  done
-}
-
 #if no range is selected just grab the last commit that fits the filter
-if [ -z "$log_range" ]
+if [ -z "$log_range" ] && [ -z "$tag_filter" ] && [ -z "$tag_regex" ]
 then
     list_command+="| git rev-list --stdin --date-order --no-walk=unsorted -$version_depth --reverse"
 fi
 
-if [ "$reverse" == "true" ]
+if [ "$reverse" == "true" ] && [ -z "$tag_filter" ] && [ -z "$tag_regex" ]
 then
     list_command+="| git rev-list --stdin --date-order  --first-parent --no-walk=unsorted --reverse"
 fi
 
-if [ -n "$tag_filter" ]; then
-  {
-    if [ -n "$ref" ] && [ -n "$branch" ]; then
-      tags=$(git tag --list "$tag_filter" --sort=creatordate --contains $ref --merged $branch)
-      get_commit $tags
-    elif [ -n "$ref" ]; then
-      tags=$(git tag --list "$tag_filter" --sort=creatordate | lines_including_and_after $ref)
-      get_commit $tags
-    else
-      branch_flag=
-      if [ -n "$branch" ]; then
-        branch_flag="--merged $branch"
+get_tags_matching_filter() {
+  local list_command=$1
+  local tags=$2
+  for tag in $tags; do
+    # We turn the tag ref (e.g. v1.0.0) into the object name
+    # (e.g. 1a410efbd13591db07496601ebc7a059dd55cfe9) and use grep to check it is in the output
+    # of list_command - if it isn't, it doesn't pass one of the other filters and shouldn't be
+    # outputted.
+    local commit=$(git rev-list -n 1 $tag)
+    local this_list_command="$list_command | grep -cFx \"$commit\""
+    local list_output="$(set -f; eval "$this_list_command"; set +f)"
+    if [ "$list_output" -ge 1 ]; then
+      jq -cn '{ref: $tag, commit: $commit}' --arg tag $tag --arg commit $commit
+    fi
+  done
+}
+
+get_tags_match_ancestors_filter() {
+  local list_command=$1
+  local tags=$2
+
+  # Sort commits so that we look at the oldest commits first
+  local this_list_command="$list_command | git rev-list --stdin --date-order --first-parent --no-walk=unsorted --reverse"
+  local list_output="$(set -f; eval "$this_list_command"; set +f)"
+  for commit in $list_output; do
+    # Output the commit if it is an ancestor of any of the matching tags
+    local is_ancestor=false
+    for tag in $tags; do
+      tag_commit=$(git rev-list -n 1 $tag)
+      if [ "$tag_commit" == "$commit" ] || git merge-base --is-ancestor "$commit" "$tag_commit"; then
+        is_ancestor=true
+        break
       fi
-      tag=$(git tag --list "$tag_filter" --sort=creatordate $branch_flag | tail -$version_depth)
-      get_commit $tag
+    done
+    if [ "$is_ancestor" = true ]; then
+      jq -cn '{ref: $commit}' --arg commit $commit
     fi
-  } | jq -s "map(.)" >&3
-elif [ -n "$tag_regex" ]; then
-  {
-    if [ -n "$ref" ] && [ -n "$branch" ]; then
-      tags=$(git tag --list --sort=creatordate --contains $ref --merged $branch | grep -Ex "$tag_regex")
-      get_commit $tags
-    elif [ -n "$ref" ]; then
-      tags=$(git tag --list --sort=creatordate | grep -Ex "$tag_regex" | lines_including_and_after $ref)
-      get_commit $tags
+  done
+}
+
+if [ -n "$tag_filter" ] || [ -n "$tag_regex" ]; then
+  # Create a suffix to "git tag" that will apply the tag filter
+  if [ -n "$tag_filter" ]; then
+    tag_filter_cmd="--list \"$tag_filter\""
+  elif [ -n "$tag_regex" ]; then
+    tag_filter_cmd="| grep -Ex \"$tag_regex\""
+  fi
+
+  # Build a list of tag refs (e.g. v1.0.0) that match the filter
+  if [ -n "$ref" ] && [ -n "$branch" ]; then
+    tags=$(set -f; eval "git tag --sort=creatordate --contains $ref --merged $branch $tag_filter_cmd"; set +f)
+  elif [ -n "$ref" ]; then
+    tags=$(set -f; eval "git tag --sort=creatordate $tag_filter_cmd | lines_including_and_after $ref"; set +f)
+  else
+    branch_flag=
+    if [ -n "$branch" ]; then
+      branch_flag="--merged $branch"
+    fi
+    tags=$(set -f; eval "git tag --sort=creatordate $branch_flag $tag_filter_cmd"; set +f)
+  fi
+
+  # Only proceed if we actually found any tags
+  if [ -n "$tags" ]; then
+    if [ "$tag_behaviour" == "match_tagged" ]; then
+      get_tags_matching_filter "$list_command" "$tags" | tail "-$version_depth" | jq -s "map(.)" >&3
     else
-      branch_flag=
-      if [ -n "$branch" ]; then
-        branch_flag="--merged $branch"
-      fi
-      tag=$(git tag --list --sort=creatordate $branch_flag | grep -Ex "$tag_regex" | tail -$version_depth)
-      get_commit $tag
+      get_tags_match_ancestors_filter "$list_command" "$tags" | tail "-$version_depth" | jq -s "map(.)" >&3
     fi
-  } | jq -s "map(.)" >&3
+  else
+    jq -n "[]" >&3
+  fi
 else
   {
     set -f

test/check.sh

@@ -4,6 +4,16 @@ set -e
 
 source $(dirname $0)/helpers.sh
 
+check_jq_functionality() {
+  # Ensure JQ correctly treats empty input as invalid
+  set +e
+  jq -e 'type == "string"' </dev/null
+  if [ $? -eq 0 ]; then
+    echo "WARNING - Outdated JQ - please update! Some tests may incorrectly pass"
+  fi
+  set -e
+}
+
 it_can_check_from_head() {
   local repo=$(init_repo)
   local ref=$(make_commit $repo)
@@ -644,6 +654,97 @@ it_can_check_with_tag_filter() {
   "
 }
 
+it_can_check_with_tag_and_path_match_filter() {
+  local repo=$(init_repo)
+  local ref1=$(make_commit_to_file $repo file-a)
+  local ref2=$(make_annotated_tag $repo "1.0-staging" "tag 1" true)
+  local ref3=$(make_commit_to_file $repo file-b)
+  local ref4=$(make_annotated_tag $repo "1.0-production" "tag 2" true)
+  local ref5=$(make_annotated_tag $repo "2.0-staging" "tag 3" true)
+  local ref6=$(make_commit_to_file $repo file-c)
+  local ref7=$(make_annotated_tag $repo "2.0-staging" "tag 5" true)
+  local ref8=$(make_commit_to_file $repo file-b)
+  local ref9=$(make_annotated_tag $repo "2.0-production" "tag 4" true)
+  local ref10=$(make_commit_to_file $repo file-c)
+
+  check_uri_with_tag_and_path_filter $repo "2*" "match_tagged" file-c | jq -e "
+    . == [{ref: \"2.0-staging\", commit: \"$ref6\"}]
+  "
+
+  # No matching files. ref3's tag was replaced by the tag on ref6 - which does not path match
+  check_uri_with_tag_and_path_filter $repo "*-staging" "match_tagged" file-b | jq -e "
+    . == []
+  "
+
+  # Although multiple tagged commits modify the files, ref8 is the latest (and version_depth is default 1)
+  check_uri_with_tag_and_path_filter $repo "*" "match_tagged" file-b file-c | jq -e "
+    . == [
+      {ref: \"2.0-production\", commit: \"$ref8\"}
+    ]
+  "
+
+  # file-f was never created
+  check_uri_with_tag_and_path_filter $repo "*" "match_tagged" file-f | jq -e "
+    . == []
+  "
+
+  # no tags matching 4.0-*
+  check_uri_with_tag_and_path_filter $repo "4.0-*" "match_tagged" file-c | jq -e "
+    . == []
+  "
+}
+
+it_can_check_with_tag_and_path_match_ancestors_filter() {
+  local repo=$(init_repo)
+  local ref1=$(make_commit_to_file $repo file-a)
+  local ref2=$(make_annotated_tag $repo "1.0-staging" "tag 1" true)
+  local ref3=$(make_commit_to_file $repo file-b)
+  local ref4=$(make_annotated_tag $repo "1.0-production" "tag 2" true)
+  local ref5=$(make_annotated_tag $repo "2.0-staging" "tag 3" true)
+  local ref6=$(make_commit_to_file $repo file-c)
+  local ref7=$(make_annotated_tag $repo "2.0-staging" "tag 5" true)
+  local ref8=$(make_commit_to_file $repo file-b)
+  local ref9=$(make_annotated_tag $repo "2.0-production" "tag 4" true)
+  local ref10=$(make_commit_to_file $repo file-c)
+
+  # ref10 is the most recent to modify file-c, but the latest 2* tag is before it. Therefore ref6 matches.
+  check_uri_with_tag_and_path_filter $repo "2*" "match_tag_ancestors" file-c | jq -e "
+    . == [{ref: \"$ref6\"}]
+  "
+
+  # ref3 is the most recent commit to modify file-b, and following commits have -staging tags
+  check_uri_with_tag_and_path_filter $repo "*-staging" "match_tag_ancestors" file-b | jq -e "
+    . == [{ref: \"$ref3\"}]
+  "
+
+  # although no 2.* tagged commits modified file-a, they follow on from commits that did
+  check_uri_with_tag_and_path_filter $repo "2.*" "match_tag_ancestors" file-a | jq -e "
+    . == [{ref: \"$ref1\"}]
+  "
+
+  # no commits creating file-c are followed by 1.* tags
+  check_uri_with_tag_and_path_filter $repo "1.*" "match_tag_ancestors" file-c | jq -e "
+    . == []
+  "
+
+  # Although multiple tagged commits modify the files, ref8 is the latest (and version_depth is default 1)
+  check_uri_with_tag_and_path_filter $repo "*" "match_tag_ancestors" file-b file-c | jq -e "
+    . == [
+      {ref: \"$ref8\"}
+    ]
+  "
+
+  # file-f was never created
+  check_uri_with_tag_and_path_filter $repo "*" "match_tag_ancestors" file-f | jq -e "
+    . == []
+  "
+
+  # no tags matching 4.0-*
+  check_uri_with_tag_and_path_filter $repo "4.0-*" "match_tag_ancestors" file-c | jq -e "
+    . == []
+  "
+}
+
 it_can_check_with_tag_regex() {
   local repo=$(init_repo)
   local ref1=$(make_commit $repo)
@@ -679,7 +780,7 @@ it_can_check_with_tag_filter_with_cursor() {
   local ref13=$(make_commit $repo)
 
   x=$(check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging")
-  check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging" | jq -e "
+  check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging" 2 | jq -e "
     . == [{ref: \"2.0-staging\", commit: \"$ref5\"}, {ref: \"3.0-staging\", commit: \"$ref9\"}]
   "
 }
@@ -701,7 +802,7 @@ it_can_check_with_tag_regex_with_cursor() {
   local ref13=$(make_commit $repo)
 
   x=$(check_uri_with_tag_regex_from $repo ".*-staging" "2.0-staging")
-  check_uri_with_tag_regex_from $repo ".*-staging" "2.0-staging" | jq -e "
+  check_uri_with_tag_regex_from $repo ".*-staging" "2.0-staging" 2 | jq -e "
     . == [{ref: \"2.0-staging\", commit: \"$ref5\"}, {ref: \"3.0-staging\", commit: \"$ref9\"}]
   "
 }
@@ -765,7 +866,7 @@ it_can_check_with_tag_filter_over_all_branches_with_cursor() {
   local ref13=$(make_annotated_tag $repo "3.0-production" "tag 6")
   local ref14=$(make_commit_to_branch $repo branch-a)
 
-  check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging" | jq -e "
+  check_uri_with_tag_filter_from $repo "*-staging" "2.0-staging" 2 | jq -e "
     . == [{ref: \"2.0-staging\", commit: \"$ref6\"}, {ref: \"3.0-staging\", commit: \"$ref10\"}]
   "
 }
@@ -787,7 +888,7 @@ it_can_check_with_tag_regex_over_all_branches_with_cursor() {
   local ref13=$(make_annotated_tag $repo "3.0-production" "tag 6")
   local ref14=$(make_commit_to_branch $repo branch-a)
 
-  check_uri_with_tag_regex_from $repo ".*-staging" "2.0-staging" | jq -e "
+  check_uri_with_tag_regex_from $repo ".*-staging" "2.0-staging" 2 | jq -e "
     . == [{ref: \"2.0-staging\", commit: \"$ref6\"}, {ref: \"3.0-staging\", commit: \"$ref10\"}]
   "
 }
@@ -1014,6 +1115,7 @@ it_checks_uri_with_tag_filter_and_version_depth() {
     ]"
 }
 
+run check_jq_functionality
 run it_can_check_from_head
 run it_can_check_from_a_ref
 run it_can_check_from_a_first_commit_in_repo
@@ -1054,6 +1156,8 @@ run it_can_check_with_tag_filter_with_bogus_ref
 run it_can_check_with_tag_regex_with_bogus_ref
 run it_can_check_with_tag_filter_with_replaced_tags
 run it_can_check_with_tag_regex_with_replaced_tags
+run it_can_check_with_tag_and_path_match_filter
+run it_can_check_with_tag_and_path_match_ancestors_filter
 run it_can_check_from_head_only_fetching_single_branch
 run it_can_check_and_set_git_config
 run it_can_check_from_a_ref_and_only_show_merge_commit