CPP-791 - Process creds.zip for automatic cloud secure connection configuration (#269)

- Ensuring zlib is on by default and adding zlib to CI
- Read cloud secure connection config from zip file in memory
- Exposing unzip implementation for driver use
- Exposing zip implementation for unit test use
- Vendoring minizip from zlib repository
  - Patched minizip for C90 compliant code and fixed missing typedef for older zlib versions

Author: mikefero

CMakeLists.txt

@@ -53,7 +53,7 @@ option(DSE_USE_SPARSEHASH "Use sparsehash" OFF)
 option(DSE_USE_STATIC_LIBS "Link static libraries when building executables" OFF)
 option(DSE_USE_STD_ATOMIC "Use C++11 atomics library" OFF)
 option(DSE_USE_TCMALLOC "Use tcmalloc" OFF)
-option(DSE_USE_ZLIB "Use zlib" OFF)
+option(DSE_USE_ZLIB "Use zlib" ON)
 option(DSE_USE_TIMERFD "Use timerfd (Linux only)" ON)
 
 # Handle testing dependencies

appveyor.yml

@@ -45,6 +45,7 @@ environment:
   LIBUV_VERSION: 1.29.1
   OPENSSL_1_0_VERSION: 1.0.2s
   OPENSSL_1_1_VERSION: 1.1.1c
+  ZLIB_VERSION: 1.2.11
   matrix:
     - CMAKE_GENERATOR: Visual Studio 10 2010
       OPENSSL_MAJOR_MINOR: 1.0

cpp-driver/.build.linux.sh

@@ -54,6 +54,10 @@ install_openssl() {
   true # Already installed on image
 }
 
+install_zlib() {
+  true # Already installed on image
+}
+
 install_driver() {(
   cd packaging
 

cpp-driver/.build.osx.sh

@@ -39,6 +39,16 @@ install_openssl() {
   fi
 }
 
+install_zlib() {
+  if brew ls --versions zlib > /dev/null; then
+    if ! brew outdated zlib; then
+      brew upgrade zlib
+    fi
+  else
+    brew install zlib
+  fi
+}
+
 install_driver() {
   true
 }

cpp-driver/.build.sh

@@ -61,6 +61,7 @@ get_driver_version() {
 install_dependencies() {
   install_libuv
   install_openssl
+  install_zlib
 }
 
 build_driver() {

cpp-driver/CMakeLists.txt

@@ -43,7 +43,7 @@ option(CASS_USE_OPENSSL "Use OpenSSL" ON)
 option(CASS_USE_STATIC_LIBS "Link static libraries when building executables" OFF)
 option(CASS_USE_STD_ATOMIC "Use C++11 atomics library" OFF)
 option(CASS_USE_TCMALLOC "Use tcmalloc" OFF)
-option(CASS_USE_ZLIB "Use zlib" OFF)
+option(CASS_USE_ZLIB "Use zlib" ON)
 option(CASS_USE_TIMERFD "Use timerfd (Linux only)" ON)
 
 # Handle testing dependencies

cpp-driver/appveyor.ps1

@@ -55,6 +55,7 @@ Boost:         v$($Env:BOOST_VERSION)
 libssh2:       v$($Env:LIBSSH2_VERSION)
 libuv:         v$($Env:LIBUV_VERSION)
 OpenSSL:       v$(Get-OpenSSL-Version)
+zlib:          v$($Env:ZLIB_VERSION)
 Build Number:  $($Env:APPVEYOR_BUILD_NUMBER)
 Branch:        $($Env:APPVEYOR_REPO_BRANCH)
 SHA:           $(Get-Commit-Sha)
@@ -104,6 +105,7 @@ Function Initialize-Build-Environment {
   $libuv_version = $Env:LIBUV_VERSION
   $openssl_version = Get-OpenSSL-Version
   $Env:OPENSSL_VERSION = $openssl_version
+  $zlib_version = $Env:ZLIB_VERSION
   $kerberos_version = "4.1"
   $bison_version = "2.4.1"
   $perl_version = "5.26.2.1"
@@ -149,13 +151,15 @@ Function Initialize-Build-Environment {
   $Env:LIBUV_ROOT_DIR = "$($dependencies_location_prefix)/libuv-$($libuv_version)"
   $Env:OPENSSL_BASE_DIR = "$($dependencies_location_prefix)/openssl-$($openssl_version)"
   $Env:OPENSSL_ROOT_DIR = "$($Env:OPENSSL_BASE_DIR)/shared"
+  $Env:ZLIB_ROOT_DIR = "$($dependencies_location_prefix)/zlib-$($zlib_version)"
   $Env:DRIVER_INSTALL_DIR = "C:/projects/driver/lib"
   $Env:DRIVER_ARTIFACTS_DIR = "C:/projects/driver/artifacts"
   $Env:DRIVER_ARTIFACTS_LOGS_DIR = "$($Env:DRIVER_ARTIFACTS_DIR)/logs"
 
   # Generate the environment variables for the third party archives
   $Env:LIBUV_ARTIFACT_ARCHIVE = "libuv-$($libuv_version)-win$($architecture)-msvc$($Env:VISUAL_STUDIO_INTERNAL_VERSION).zip"
   $Env:OPENSSL_ARTIFACT_ARCHIVE = "openssl-$($openssl_version)-win$($architecture)-msvc$($Env:VISUAL_STUDIO_INTERNAL_VERSION).zip"
+  $Env:ZLIB_ARTIFACT_ARCHIVE = "zlib-$($zlib_version)-win$($architecture)-msvc$($Env:VISUAL_STUDIO_INTERNAL_VERSION).zip"
 
   # Generate DataStax Enterprise specific environment variables
   If ($Env:DRIVER_TYPE -Like "dse") {
@@ -423,6 +427,53 @@ add_dependencies(`${PROJECT_NAME} `${OPENSSL_LIBRARY_NAME})
     }
   }
 
+  If (-Not (Test-Path -Path "$($Env:ZLIB_ROOT_DIR)/lib")) {
+    New-Item -ItemType Directory -Force -Path "$($dependencies_build_location_prefix)/zlib" | Out-Null
+    Push-Location -Path "$($dependencies_build_location_prefix)/zlib"
+
+    $cmakelists_contents = @"
+cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
+project(zlib)
+set(PROJECT_DISPLAY_NAME "AppVeyor CI Build for zlib")
+set(PROJECT_MODULE_DIR $cmake_modules_dir)
+set(CMAKE_MODULE_PATH `${CMAKE_MODULE_PATH} `${PROJECT_MODULE_DIR})
+include(ExternalProject-zlib)
+set(GENERATED_SOURCE_FILE `${CMAKE_CURRENT_BINARY_DIR}/main.cpp)
+file(REMOVE `${GENERATED_SOURCE_FILE})
+file(WRITE `${GENERATED_SOURCE_FILE} "int main () { return 0; }")
+add_executable(`${PROJECT_NAME} `${GENERATED_SOURCE_FILE})
+add_dependencies(`${PROJECT_NAME} `${ZLIB_LIBRARY_NAME})
+"@
+    $cmakelists_contents | Out-File -FilePath "CMakeLists.txt" -Encoding Utf8 -Force
+
+    Write-Host "Configuring zlib"
+    cmake -G "$($cmake_generator)" -DBUILD_SHARED_LIBS=On "-DZLIB_VERSION=$($Env:ZLIB_VERSION)" "-DZLIB_INSTALL_PREFIX=$($Env:ZLIB_ROOT_DIR)" .
+    If ($LastExitCode -ne 0) {
+      If (Test-Path -Path "build/CMakeFiles/CMakeOutput.log") {
+        Push-AppveyorArtifact "build/CMakeFiles/CMakeOutput.log" -DeploymentName "zlib Output Log"
+      }
+      If (Test-Path -Path "build/CMakeFiles/CMakeError.log") {
+        Push-AppveyorArtifact "build/CMakeFiles/CMakeError.log" -DeploymentName "zlib Error Log"
+      }
+      Pop-Location
+      Throw "Failed to configure zlib for MSVC $($Env:VISUAL_STUDIO_INTERNAL_VERSION)-$($Env:Platform)"
+    }
+    Write-Host "Building and Installing zlib"
+    cmake --build . --config RelWithDebInfo
+    If ($LastExitCode -ne 0) {
+      If (Test-Path -Path "build/CMakeFiles/CMakeOutput.log") {
+        Push-AppveyorArtifact "build/CMakeFiles/CMakeOutput.log" -DeploymentName "zlib Output Log"
+      }
+      If (Test-Path -Path "build/CMakeFiles/CMakeError.log") {
+        Push-AppveyorArtifact "build/CMakeFiles/CMakeError.log" -DeploymentName "zlib Error Log"
+      }
+      Pop-Location
+      Throw "Failed to build zlib for MSVC $($Env:VISUAL_STUDIO_INTERNAL_VERSION)-$($Env:Platform)"
+    }
+
+    Pop-Location
+  }
+
   # Handle installation of DataStax Enterprise dependencies
   If ($Env:DRIVER_TYPE -Like "dse") {
     # Determine if Kerberos for Windows should be installed (cached)
@@ -583,7 +634,7 @@ Function Build-Driver {
   New-Item -ItemType Directory -Force -Path "$($Env:APPVEYOR_BUILD_FOLDER)/build"
   Push-Location "$($Env:APPVEYOR_BUILD_FOLDER)/build"
   Write-Host "Configuring DataStax C/C++ $($driver_type) Driver"
-  cmake -G "$($cmake_generator)" "-D$($Env:DRIVER_TYPE)_MULTICORE_COMPILATION=On" "-D$($Env:DRIVER_TYPE)_USE_OPENSSL=On" "-D$($Env:DRIVER_TYPE)_USE_BOOST_ATOMIC=$($use_boost_atomic)" "-D$($Env:DRIVER_TYPE)_BUILD_EXAMPLES=On" "-D$($Env:DRIVER_TYPE)_BUILD_TESTS=On" "-D$($Env:DRIVER_TYPE)_USE_LIBSSH2=On" "-DCMAKE_INSTALL_PREFIX=`"$($Env:DRIVER_INSTALL_DIR)`"" ..
+  cmake -G "$($cmake_generator)" "-D$($Env:DRIVER_TYPE)_MULTICORE_COMPILATION=On" "-D$($Env:DRIVER_TYPE)_USE_OPENSSL=On" "-D$($Env:DRIVER_TYPE)_USE_ZLIB=On" "-D$($Env:DRIVER_TYPE)_USE_BOOST_ATOMIC=$($use_boost_atomic)" "-D$($Env:DRIVER_TYPE)_BUILD_EXAMPLES=On" "-D$($Env:DRIVER_TYPE)_BUILD_TESTS=On" "-D$($Env:DRIVER_TYPE)_USE_LIBSSH2=On" "-DCMAKE_INSTALL_PREFIX=`"$($Env:DRIVER_INSTALL_DIR)`"" ..
   If ($LastExitCode -ne 0) {
     Pop-Location
     Throw "Failed to configure DataStax C/C++ $($driver_type) Driver for MSVC $($Env:VISUAL_STUDIO_INTERNAL_VERSION)-$($Env:Platform)"
@@ -707,6 +758,17 @@ a -tzip "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:OPENSSL_ARTIFACT_ARCHIVE)" -r "$($E
   If ($process.ExitCode -ne 0) {
     Throw "Failed to archive OpenSSL for MSVC $($Env:VISUAL_STUDIO_INTERNAL_VERSION)-$($Env:Platform)"
   }
+
+  # Clean up the library dependency directories for zlib packaging
+  New-Item -ItemType Directory -Force -Path "$($Env:DRIVER_ARTIFACTS_DIR)/zlib" | Out-Null
+  Copy-Item -Force -Recurse -Path "$($Env:ZLIB_ROOT_DIR)/*" "$($Env:DRIVER_ARTIFACTS_DIR)/zlib" | Out-Null
+  $argument_list = @"
+a -tzip "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:ZLIB_ARTIFACT_ARCHIVE)" -r "$($Env:DRIVER_ARTIFACTS_DIR)/zlib/*"
+"@
+  $process = Start-Process -FilePath 7z -ArgumentList $argument_list -PassThru -Wait -NoNewWindow
+  If ($process.ExitCode -ne 0) {
+    Throw "Failed to archive zlib for MSVC $($Env:VISUAL_STUDIO_INTERNAL_VERSION)-$($Env:Platform)"
+  }
 }
 
 Function Push-Artifacts {
@@ -721,6 +783,7 @@ Function Push-Artifacts {
     Push-AppveyorArtifact "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:DRIVER_ARTIFACT_TESTS_ARCHIVE)" -DeploymentName "DataStax C/C++ $($driver_type) Driver Tests"
     Push-AppveyorArtifact "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:LIBUV_ARTIFACT_ARCHIVE)" -DeploymentName "libuv v$($Env:LIBUV_VERSION)"
     Push-AppveyorArtifact "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:OPENSSL_ARTIFACT_ARCHIVE)" -DeploymentName "OpenSSL v$($Env:OPENSSL_VERSION)"
+    Push-AppveyorArtifact "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:ZLIB_ARTIFACT_ARCHIVE)" -DeploymentName "zlib v$($Env:ZLIB_VERSION)"
   }
 }
 
@@ -781,6 +844,8 @@ Function Publish-Artifacts {
     #TODO: Need to handle OpenSSL v1.1.x if enabled
     $openssl_uri = "$($base_uri)/dependencies/openssl/v$($Env:OPENSSL_VERSION)/$($Env:OPENSSL_ARTIFACT_ARCHIVE)"
     $openssl_archive = "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:OPENSSL_ARTIFACT_ARCHIVE)"
+    $zlib_uri = "$($base_uri)/dependencies/zlib/v$($Env:ZLIB_VERSION)/$($Env:ZLIB_ARTIFACT_ARCHIVE)"
+    $zlib_archive = "$($Env:DRIVER_ARTIFACTS_DIR)/$($Env:ZLIB_ARTIFACT_ARCHIVE)"
 
     # Publish/Upload the driver and it dependencies to Artifactory
     $is_failure = $False
@@ -798,6 +863,10 @@ Function Publish-Artifacts {
       $is_failure = $True
       $failed_upload += "OpenSSL"
     }
+    If ((Publish-Artifact-To-Artifactory -Uri "$($zlib_uri)" -FilePath "$($zlib_archive)") -ne 0) {
+      $is_failure = $True
+      $failed_upload += "zlib"
+    }
 
     # Check to see if there was a failure uploading the artifacts
     If ($is_failure) {

cpp-driver/appveyor.yml

@@ -44,6 +44,7 @@ environment:
   LIBUV_VERSION: 1.29.1
   OPENSSL_1_0_VERSION: 1.0.2s
   OPENSSL_1_1_VERSION: 1.1.1c
+  ZLIB_VERSION: 1.2.11
   matrix:
     - CMAKE_GENERATOR: Visual Studio 10 2010
       OPENSSL_MAJOR_MINOR: 1.0

cpp-driver/cmake/modules/CppDriver.cmake

@@ -459,6 +459,29 @@ macro(CassRapidJson)
   include_directories(${RAPID_JSON_INCLUDE_DIR})
 endmacro()
 
+#------------------------
+# CassMiniZip
+#
+# Set some MINIZIP_* variables, set up some source_group's, and add the
+# MINIZIP include dir to our list of include dirs.
+#
+# Input: CASS_SRC_DIR
+# Output: MINIZIP_INCLUDE_DIR, MINIZIP_HEADER_FILES, MINIZIP_SOURCE_FILES
+#------------------------
+macro(CassMiniZip)
+  if (ZLIB_FOUND)
+    set(MINIZIP_INCLUDE_DIR "${CASS_SRC_DIR}/third_party/minizip")
+    set(MINIZIP_HEADER_FILES ${MINIZIP_INCLUDE_DIR}/crypt.h
+                             ${MINIZIP_INCLUDE_DIR}/ioapi.h
+                             ${MINIZIP_INCLUDE_DIR}/unzip.h)
+    set(MINIZIP_SOURCE_FILES ${MINIZIP_INCLUDE_DIR}/ioapi.c
+                             ${MINIZIP_INCLUDE_DIR}/unzip.c)
+    source_group("Header Files\\minizip" FILES ${MINIZIP_HEADER_FILES})
+    source_group("Source Files\\minizip" FILES ${MINIZIP_SOURCE_FILES})
+    include_directories(${MINIZIP_INCLUDE_DIR})
+  endif()
+endmacro()
+
 #------------------------
 # CassSimulacron
 #
@@ -687,6 +710,7 @@ macro(CassUseZlib)
       # Assign zlib properties
       set(CASS_INCLUDES ${CASS_INCLUDES} ${ZLIB_INCLUDE_DIRS})
       set(CASS_LIBS ${CASS_LIBS} ${ZLIB_LIBRARIES})
+      set(HAVE_ZLIB On)
     else()
       message(WARNING "Could not find zlib, try to set the path to zlib root folder in the system variable ZLIB_ROOT_DIR")
       message(WARNING "zlib libraries will not be linked into build")
@@ -986,6 +1010,10 @@ macro(CassFindSourceFiles)
       ${CASS_SRC_DIR}/ssl/ssl_no_impl.cpp)
   endif()
 
+  CassMiniZip()
+  set(CASS_INC_FILES ${CASS_INC_FILES} ${MINIZIP_HEADER_FILES})
+  set(CASS_SRC_FILES ${CASS_SRC_FILES} ${MINIZIP_SOURCE_FILES})
+
   set(CASS_ALL_SOURCE_FILES ${CASS_SRC_FILES} ${CASS_API_HEADER_FILES} ${CASS_INC_FILES})
 endmacro()
 

cpp-driver/driver_config.hpp.in

@@ -12,5 +12,6 @@
 #cmakedefine HAVE_ARC4RANDOM
 #cmakedefine HAVE_GETRANDOM
 #cmakedefine HAVE_TIMERFD
+#cmakedefine HAVE_ZLIB
 
 #endif