Merge pull request #14625 from jan-cerny/hummingbird_remediations

Add hummingbird remediations type

Author: Mab879

build-scripts/generate_profile_remediations.py

@@ -4,14 +4,15 @@
 import collections
 import os
 import re
+import textwrap
 import xml.etree.ElementTree as ET
-import yaml
 
 import ssg.ansible
 import ssg.yaml
 from ssg.constants import (
     ansible_system,
     bash_system,
+    hummingbird_system,
     datastream_namespace,
     OSCAP_PROFILE,
     OSCAP_RULE,
@@ -21,9 +22,12 @@
 
 DEFAULT_SELECTOR = "__DEFAULT"
 HASH_ROW = "#" * 79
-LANGUAGE_TO_SYSTEM = {"ansible": ansible_system, "bash": bash_system}
-LANGUAGE_TO_TARGET = {"ansible": "playbook", "bash": "script"}
-LANGUAGE_TO_EXTENSION = {"ansible": "yml", "bash": "sh"}
+LANGUAGE_TO_SYSTEM = {
+    "ansible": ansible_system,
+    "bash": bash_system,
+    "hummingbird": hummingbird_system}
+LANGUAGE_TO_TARGET = {"ansible": "playbook", "bash": "script", "hummingbird": "script"}
+LANGUAGE_TO_EXTENSION = {"ansible": "yml", "bash": "sh", "hummingbird": "sh"}
 ANSIBLE_VAR_PATTERN = re.compile(
     "- name: XCCDF Value [^ ]+ # promote to variable\n  set_fact:\n"
     "    ([^:]+): !!str (.+)\n  tags:\n    - always\n")
@@ -44,8 +48,8 @@ def parse_args():
         help="Product ID, eg. 'rhel9'"
     )
     parser.add_argument(
-        "--language", required=True, choices=["bash", "ansible"],
-        help="Remediation language, either 'bash' or 'ansible'"
+        "--language", required=True, choices=["bash", "ansible", "hummingbird"],
+        help="Remediation language, either 'bash' or 'ansible' or 'hummingbird'"
     )
     args = parser.parse_args()
     return args
@@ -125,7 +129,7 @@ def get_variable_values(variable):
         if selector is None:
             selector = DEFAULT_SELECTOR
         if value.text is None:
-            values["selector"] = ""
+            values[selector] = ""
         else:
             values[selector] = value.text
     return values
@@ -206,8 +210,10 @@ def load_all_remediations(self, benchmark):
     def generate_profile_remediation_script(self, profile_el):
         if self.language == "ansible":
             output = self.create_output_ansible(profile_el)
-        elif self.language == "bash":
-            output = self.create_output_bash(profile_el)
+        elif self.language in ("bash", "hummingbird"):
+            output = self.create_output_linear(profile_el)
+        else:
+            raise ValueError("Unknown language %s" % self.language)
         file_path = self.get_output_file_path(profile_el)
         with open(file_path, "wb") as f:
             f.write(output.encode("utf-8"))
@@ -254,20 +260,38 @@ def collect_ansible_vars_and_tasks(self, profile_el):
             all_tasks.extend(rule_tasks)
         return (all_vars, all_tasks)
 
-    def create_output_bash(self, profile):
+    def create_output_linear(self, profile):
         output = []
         selected_rules = get_selected_rules(profile)
         refinements = get_value_refinenements(profile)
         header = self.create_header(profile)
         output.append(header)
         total = len(selected_rules)
+        if self.language == "hummingbird":
+            newroot_assign = textwrap.dedent(
+                """
+                # The first argument is the root directory of the system
+                NEWROOT="$1"
+                if [[ -z "$NEWROOT" ]] ; then
+                    echo "Missing required NEWROOT argument" >&2
+                    exit 1
+                fi
+                """
+            )
+            output.append(newroot_assign)
         current = 1
         for rule_id in self.remediations:
             if rule_id not in selected_rules:
                 continue
             status = (current, total)
-            rule_remediation = self.generate_bash_rule_remediation(
-                rule_id, status, refinements)
+            if self.language == "bash":
+                rule_remediation = self.generate_bash_rule_remediation(
+                    rule_id, status, refinements)
+            elif self.language == "hummingbird":
+                rule_remediation = self.generate_hummingbird_rule_remediation(
+                    rule_id, refinements)
+            else:
+                raise ValueError("Unknown language %s" % self.language)
             output.append(rule_remediation)
             current += 1
         return "".join(output)
@@ -286,11 +310,26 @@ def create_header(self, profile):
             shebang_with_newline = "#!/usr/bin/env bash\n"
             remediation_type = "Bash Remediation Script"
             how_to_apply = "# $ sudo ./remediation-script.sh\n"
+        elif self.language == "hummingbird":
+            shebang_with_newline = "#!/usr/bin/env bash\n"
+            remediation_type = (
+                "Bash Remediation Script for building Project Hummingbird "
+                "container images")
+            how_to_apply = "# RUN remediation-script.sh ${NEWROOT}\n"
+        else:
+            raise ValueError("Unknown language %s" % self.language)
         profile_title = profile.find("./{%s}title" % XCCDF12_NS).text
         description = profile.find("./{%s}description" % XCCDF12_NS).text
         commented_profile_description = comment(description)
         xccdf_version_name = "1.2"
         profile_id = profile.get("id")
+        if self.language == "bash":
+            generation_text = (
+                "# This file can be generated by OpenSCAP using:\n"
+                "# $ oscap xccdf generate fix --profile %s --fix-type %s %s\n"
+                "#\n" % (profile_id, self.language, self.ds_file_name))
+        else:
+            generation_text = ""
         fix_header = (
             "%s"
             "%s\n"
@@ -305,9 +344,7 @@ def create_header(self, profile):
             "# Benchmark Version:  %s\n"
             "# XCCDF Version:  %s\n"
             "#\n"
-            "# This file can be generated by OpenSCAP using:\n"
-            "# $ oscap xccdf generate fix --profile %s --fix-type %s %s\n"
-            "#\n"
+            "%s"
             "# This %s is generated from an XCCDF profile without"
             " preliminary evaluation.\n"
             "# It attempts to fix every selected rule, even if the system is"
@@ -320,7 +357,7 @@ def create_header(self, profile):
                 shebang_with_newline, HASH_ROW, remediation_type,
                 profile_title, commented_profile_description, profile_id,
                 self.benchmark_id, self.benchmark_version, xccdf_version_name,
-                profile_id, self.language, self.ds_file_name,
+                generation_text,
                 remediation_type, remediation_type, how_to_apply, HASH_ROW))
         return fix_header
 
@@ -351,6 +388,29 @@ def generate_bash_rule_remediation(self, rule_id, status, refinements):
         output.append(end_msg)
         return "".join(output)
 
+    def generate_hummingbird_rule_remediation(self, rule_id, refinements):
+        fix_el = self.remediations[rule_id]
+        if fix_el is None:
+            return ""
+        expanded_remediation = expand_variables(
+            fix_el, refinements, self.variables)
+        # For Hummingbird we intentionally don't add any warning if the
+        # rule Hummingbird remediation is missing because we expect that
+        # it will be normal that most of rules won't have any Hummingbird
+        # remediation
+        if expanded_remediation is None:
+            return ""
+        output = []
+        header = (
+            "%s\n"
+            "# BEGIN fix for '%s'\n"
+            "%s\n" % (HASH_ROW, rule_id, HASH_ROW))
+        output.append(header)
+        output.append(expanded_remediation)
+        end_msg = "\n# END fix for '%s'\n\n" % (rule_id)
+        output.append(end_msg)
+        return "".join(output)
+
     def generate_ansible_rule_remediation(self, fix_el, refinements):
         rule_vars = {}
         tasks = []

cmake/SSGCommon.cmake

@@ -258,7 +258,7 @@ macro(ssg_build_ansible_playbooks PRODUCT)
 endmacro()
 
 macro(ssg_build_remediations PRODUCT)
-    message(STATUS "Scanning for dependencies of ${PRODUCT} fixes (bash, ansible, puppet, anaconda, ignition, kubernetes and blueprint)...")
+    message(STATUS "Scanning for dependencies of ${PRODUCT} fixes (${PRODUCT_REMEDIATION_LANGUAGES})...")
 
     ssg_collect_remediations(${PRODUCT} "${PRODUCT_REMEDIATION_LANGUAGES}")
 
@@ -575,6 +575,20 @@ macro(ssg_build_profile_bash_scripts PRODUCT)
         DEPENDS "${CMAKE_BINARY_DIR}/bash/all-profile-bash-scripts-${PRODUCT}"
     )
 endmacro()
+macro(ssg_build_profile_hummingbird_scripts PRODUCT)
+    add_custom_command(
+        OUTPUT "${CMAKE_BINARY_DIR}/hummingbird_scripts/all-profile-hummingbird-scripts-${PRODUCT}"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/hummingbird_scripts"
+        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${Python_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/generate_profile_remediations.py" --language hummingbird --data-stream "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" --output-dir "${CMAKE_BINARY_DIR}/hummingbird_scripts" --product "${PRODUCT}"
+        COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_BINARY_DIR}/hummingbird_scripts/all-profile-hummingbird-scripts-${PRODUCT}"
+        DEPENDS generate-ssg-${PRODUCT}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
+        COMMENT "[${PRODUCT}-hummingbird-scripts] generating hummingbird remediation scripts for all profiles in ssg-${PRODUCT}-ds.xml"
+    )
+    add_custom_target(
+        generate-all-profile-hummingbird-scripts-${PRODUCT}
+        DEPENDS "${CMAKE_BINARY_DIR}/hummingbird_scripts/all-profile-hummingbird-scripts-${PRODUCT}"
+    )
+endmacro()
 
 # Build per-profile Ansible remediation scripts that can be used independently
 # of OpenSCAP execution.
@@ -784,6 +798,16 @@ macro(ssg_build_product PRODUCT)
         add_dependencies(zipfile ${PRODUCT}-profile-bash-scripts)
     endif()
 
+    if("${PRODUCT_HUMMINGBIRD_REMEDIATION_ENABLED}")
+        ssg_build_profile_hummingbird_scripts(${PRODUCT})
+        add_custom_target(
+            ${PRODUCT}-profile-hummingbird-scripts
+            DEPENDS generate-all-profile-hummingbird-scripts-${PRODUCT} "${CMAKE_BINARY_DIR}/hummingbird_scripts/all-profile-hummingbird-scripts-${PRODUCT}"
+        )
+        add_dependencies(${PRODUCT} ${PRODUCT}-profile-hummingbird-scripts)
+        add_dependencies(zipfile ${PRODUCT}-profile-hummingbird-scripts)
+    endif()
+
     ssg_build_html_guides(${PRODUCT})
 
     add_custom_target(

docs/manual/developer/06_contributing_with_content.md

@@ -518,6 +518,8 @@ then contain the following subdirectories:
 
 - `bootc` - for remediation content used in the `oscap-im` tool internally, ending in `.bo`
 
+- `hummingbird` - for remediation content used during the build of Project Hummingbird container images, ending in `.sh`
+
 In each of these subdirectories, a file named `shared.ext` will apply to
 all products and be included in all builds, but `{{{ product }}}.ext`
 will only get included in the build for `{{{ product }}}` (e.g.,

docs/manual/developer/07_understanding_build_system.md

@@ -114,7 +114,7 @@ refer to their help text for more information and usage:
   framework) to expand Jinja in test scripts.
 - `generate_guides.py` -- Generate HTML guides and HTML index for every profile in the built SCAP source data stream.
 - `generate_man_page.py` -- generates the ComplianceAsCode man page.
-- `generate_profile_remediations.py` -- Generate profile oriented Bash remediation scripts or profile oriented Ansible Playbooks from the built SCAP source data stream. The output is similar to the output of the `oscap xccdf generate fix` command, but the tool `generate_profile_remediations.py` generates the scripts or Playbooks for all profiles in the given SCAP source data stream at once.
+- `generate_profile_remediations.py` -- Generate profile oriented Bash remediations (Bash scripts or Ansible Playbooks or Bash scripts for Hummingbird) from the built SCAP source data stream. The output is similar to the output of the `oscap xccdf generate fix` command, but the tool `generate_profile_remediations.py` generates the output for all profiles in the given SCAP source data stream at once.
 - `profile_tool.py` -- utility script to generate statistics about profiles
   in a specific XCCDF/data stream file.
 - `verify_references.py` -- used by the test system to verify cross-linkage

linux_os/guide/services/obsolete/r_services/no_host_based_files/hummingbird/shared.sh

@@ -0,0 +1,2 @@
+# platform = multi_platform_all
+find "$NEWROOT" -type f -name "shosts.equiv" -exec rm -f {} \;

products/hummingbird/CMakeLists.txt

@@ -3,4 +3,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     message(FATAL_ERROR "cmake has to be used on the root CMakeLists.txt, see the Building ComplianceAsCode section in the Developer Guide!")
 endif()
 
+set(PRODUCT_REMEDIATION_LANGUAGES "hummingbird")
+
 ssg_build_product("hummingbird")

ssg/build_remediations.py

@@ -26,7 +26,8 @@
     'kubernetes': '.yml',
     'blueprint': '.toml',
     'kickstart': '.cfg',
-    'bootc': '.bo'
+    'bootc': '.bo',
+    'hummingbird': '.sh',
 }
 
 
@@ -515,6 +516,15 @@ def __init__(self, file_path):
             file_path, "bootc")
 
 
+class HummingbirdRemediation(Remediation):
+    """
+    This provides class for Hummingbird remediations
+    """
+    def __init__(self, file_path):
+        super(HummingbirdRemediation, self).__init__(
+            file_path, "hummingbird")
+
+
 REMEDIATION_TO_CLASS = {
     'anaconda': AnacondaRemediation,
     'ansible': AnsibleRemediation,
@@ -525,6 +535,7 @@ def __init__(self, file_path):
     'blueprint': BlueprintRemediation,
     'kickstart': KickstartRemediation,
     'bootc': BootcRemediation,
+    'hummingbird': HummingbirdRemediation,
 }
 
 
@@ -666,6 +677,8 @@ def expand_xccdf_subs(fix, remediation_type):
         pattern = r'\(kickstart-populate\s*(\S+)\)'
     elif remediation_type == "bootc":
         pattern = r'\(bootc-populate\s*(\S+)\)'
+    elif remediation_type == "hummingbird":
+        pattern = r'\(bash-populate\s*(\S+)\)'
     else:
         sys.stderr.write("Unknown remediation type '%s'\n" % (remediation_type))
         sys.exit(1)

ssg/constants.py

@@ -89,6 +89,7 @@
 anaconda_system = "urn:redhat:anaconda:pre"
 kickstart_system = "urn:xccdf:fix:script:kickstart"
 bootc_system = "urn:xccdf:fix:script:bootc"
+hummingbird_system = "urn:xccdf:fix:script:hummingbird"
 cce_uri = "https://ncp.nist.gov/cce"
 stig_ns = "https://www.cyber.mil/stigs/srg-stig-tools/"
 ccn_ns = "https://www.ccn-cert.cni.es/pdf/guias/series-ccn-stic/guias-de-acceso-publico-ccn-stic/6768-ccn-stic-610a22-perfilado-de-seguridad-red-hat-enterprise-linux-9-0/file.html"
@@ -157,6 +158,7 @@
     "anaconda": anaconda_system,
     "kickstart": kickstart_system,
     "bootc": bootc_system,
+    "hummingbird": hummingbird_system,
 }
 
 for prefix, url_part in OVAL_SUB_NS.items():

ssg/templates.py

@@ -35,7 +35,8 @@
     "puppet": TemplatingLang("puppet", ".pp",           TemplateType.REMEDIATION, "puppet"),
     "sce-bash": TemplatingLang("sce-bash", ".sh",       TemplateType.CHECK,       "sce"),
     "kickstart": TemplatingLang("kickstart", ".cfg",    TemplateType.REMEDIATION, "kickstart"),
-    "bootc": TemplatingLang("bootc", ".bo",             TemplateType.REMEDIATION, "bootc")
+    "bootc": TemplatingLang("bootc", ".bo",             TemplateType.REMEDIATION, "bootc"),
+    "hummingbird": TemplatingLang("hummingbird", ".sh", TemplateType.REMEDIATION, "hummingbird")
 }
 
 PREPROCESSING_FILE_NAME = "template.py"

tests/CMakeLists.txt

@@ -22,6 +22,7 @@ endmacro()
 if(PY_PYTEST)
     ssg_python_unit_tests("utils" "utils" "oscal/")
     ssg_python_unit_tests("ssg-module" "." "")
+    ssg_python_unit_tests("build-scripts" "." "")
     ssg_python_unit_tests("ssg_test_suite" "tests" "")
     if(Python_VERSION_MAJOR GREATER 2 AND Python_VERSION_MINOR GREATER 7 AND PY_TRESTLE AND PY_LXML)
         ssg_python_unit_tests("utils/oscal" "utils/oscal" "")