Implements a client for Sonatype Nexus (#544)

By default when you deploy a release to Nexus, it uses a feature known as auto-staging. This uses a heuristic to determine how subsequent PUT requests are aggregated into common staging repositories. Invariably Gradle's publish plugin confuses this, possibly because it creates new TCP connections for every artifact (you can see this when running with --debug), or maybe because the Nexus server is overloaded. Rather than coerce the children into playing nice, this change creates a plugin which creates a named staging repository and deploys the release into it.

The plugin functionality boils down to:

1. Discover the staging profiles on Nexus with a GET request.
2. Create a new staging repository with a POST request.
3. Override the nexus URL given to the maven-publish plugin with https://oss.sonatype.org/service/local/staging/deployByRepositoryId/.

There's more detailed documentation in the code. The implementation was based on these docs: https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API

Change-Id: I19767b3ccd99461b167997ea15730f109dff9443

Author: nwbirnie

buildSrc/build.gradle

@@ -24,3 +24,8 @@ plugins {
 repositories {
     gradlePluginPortal()
 }
+
+dependencies {
+    implementation "com.google.http-client:google-http-client:1.40.1"
+    implementation "com.google.http-client:google-http-client-xml:1.40.1"
+}

buildSrc/src/main/groovy/com.google.api-ads.java-conventions.gradle

@@ -24,7 +24,7 @@ import java.util.regex.Matcher
 
 plugins {
   id 'java-library'
-  id 'maven-publish'
+  id 'com.google.api-ads.nexus-publish'
   id 'java-test-fixtures'
   id 'signing'
 }
@@ -57,26 +57,6 @@ javadoc {
   options.addStringOption('Xdoclint:none', '-quiet')
 }
 
-publishing {
-  publications {
-    maven(MavenPublication) {
-      from(components.java)
-    }
-  }
-  repositories {
-    maven {
-      url 'https://oss.sonatype.org/service/local/staging/deploy/maven2'
-      name = "sonatype"
-      credentials {
-        // Avoids storing Sonatype credentials in plain-text. Specify these on
-        // the command line: -PsonatypeUser=foo
-        username project.properties.get("sonatypeUser")
-        password project.properties.get("sonatypePassword")
-      }
-    }
-  }
-}
-
 components.java.withVariantsFromConfiguration(configurations.testFixturesApiElements) { skip() }
 components.java.withVariantsFromConfiguration(configurations.testFixturesRuntimeElements) { skip() }
 

buildSrc/src/main/groovy/com.google.api-ads.nexus-publish.gradle

@@ -0,0 +1,273 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Defines a plugin which will automatically configure gradle publish.
+ *
+ * Test this class by setting a non-snapshot version in gradle.properties.
+ * Nexus rejects snapshot artifacts with HTTP 400 (and little else).
+ */
+
+plugins {
+    id 'maven-publish'
+}
+
+import com.google.api.client.http.GenericUrl
+import com.google.api.client.http.HttpHeaders
+import com.google.api.client.http.HttpRequest
+import com.google.api.client.http.HttpRequestInitializer
+import com.google.api.client.http.javanet.NetHttpTransport
+import com.google.api.client.http.xml.XmlHttpContent
+import com.google.api.client.util.Key
+import com.google.api.client.xml.XmlNamespaceDictionary
+import com.google.api.client.xml.XmlObjectParser
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.ConcurrentHashMap
+import org.gradle.api.GradleException
+
+// ---------------- Google HTTP client data binding classes --------------------
+// Nexus REST API uses XML over HTTP - yey!
+// Leverages the data-binding in google-java-http-client to avoid handling XML.
+
+/** Defines data binding class for HTTP basic auth headers. */
+class AuthHeaders extends HttpHeaders {
+    @Key("Authorization")
+    public String authorization
+
+    AuthHeaders(String user, String password) {
+        // Generates the base 64 encoded version of user:pass which is specified
+        // in headers.
+        def encodedCredential = Base64.encoder.encode("${user}:${password}".bytes)
+        this.authorization = "Basic " + new String(encodedCredential, StandardCharsets.UTF_8)
+    }
+}
+
+/**
+ * Defines data-binding to retrieve the staging profiles from sonatype. Staging repositories are
+ * created from a staging profile.
+ */
+class StagingProfiles {
+    @Key
+    public StagingProfilesData data
+
+    static class StagingProfilesData {
+        @Key
+        public List<StagingProfile> stagingProfile
+
+        static class StagingProfile {
+            @Key
+            public String resourceURI
+        }
+    }
+
+    def getStagingProfileUrls() {
+        return data.stagingProfile.collect { it.resourceURI }
+    }
+}
+
+/** Defines data binding class for Nexus POST request to create staging repo. */
+class PromoteRequestXmlContent extends XmlHttpContent {
+    PromoteRequestXmlContent(namespaceDictionary, description) {
+        super(namespaceDictionary,
+                "promoteRequest",
+                new PromoteRequest(description))
+    }
+
+    /** Defines top-level payload of PromoteRequest. */
+    static class PromoteRequest {
+        @Key
+        public PromoteRequestData data
+
+        PromoteRequest() {}
+
+        PromoteRequest(String description) {
+            data = new PromoteRequestData(description)
+        }
+
+        static class PromoteRequestData {
+            /** Appears next to the staging repo in the Nexus UI. */
+            @Key
+            public String description
+
+            /** Returned by Nexus. This is the newly created repository ID. */
+            @Key
+            public String stagedRepositoryId
+
+            PromoteRequestData() {}
+
+            PromoteRequestData(String description) {
+                this.description = description
+            }
+        }
+    }
+}
+
+// -------------------------- Nexus client implementation ----------------------
+
+/**
+ * Implements a client for Sonatype Nexus.
+ *
+ * Pass --info to gradle to see the HTTP traffic.
+ * Logging of raw HTTP is handled by the the HTTP client (google-java-http-client) which uses JUL.
+ *
+ * This implementation is based on:
+ * https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API
+ *
+ * Note that there are two versions of Nexus: v2 and v3. The OSS instance (which hosts our project)
+ * uses v2.
+ *
+ * The Nexus docs are here: https://help.sonatype.com/repomanager2
+ * The REST docs are here: https://repository.sonatype.org/nexus-restlet1x-plugin/default/docs/index.html
+ * The docs are very patchy!
+ */
+class SonatypeClient {
+
+    private static final logger = org.slf4j.LoggerFactory.getLogger(SonatypeClient.class)
+    private static final AUTO_STAGING_URL =
+            "https://oss.sonatype.org/service/local/staging/deploy/maven2"
+    private static final DEPLOY_BY_REPO_URL_STEM =
+            "https://oss.sonatype.org/service/local/staging/deployByRepositoryId/"
+    private static final STAGING_PROFILES_URL = new GenericUrl(
+            "https://oss.sonatype.org/service/local/staging/profiles")
+    private static final NAMESPACE_DICTIONARY = new XmlNamespaceDictionary().set("", "")
+    private static final REQUEST_FACTORY = new NetHttpTransport()
+            .createRequestFactory(new HttpRequestInitializer() {
+                @Override
+                void initialize(HttpRequest request) throws IOException {
+                    request.setParser(new XmlObjectParser(NAMESPACE_DICTIONARY))
+                }
+            })
+    /**
+     * Global storage of URLs for a rootProject.
+     */
+    private static final STAGING_URL_CACHE = new ConcurrentHashMap();
+    private final project
+    private final authHeaders
+
+    SonatypeClient(project) {
+        this.project = project
+        this.authHeaders = new AuthHeaders(
+                project.properties.get("sonatypeUser"),
+                project.properties.get("sonatypePassword"))
+    }
+
+    /**
+     * Provides the Sonatype repository URL to use for deploying releases.
+     *
+     * Attempts to create a named staging repository to avoid using Nexus auto-staging if
+     * possible. If not possible to create a named staging repository, falls back to auto-staging.
+     *
+     * This is motivated by the observation that auto-staging's heuristic for which artifacts
+     * belong in the same repository is often confused by Gradle's publish mechanism, which
+     * appears to create a new TCP connection for each artifact. This also appears to be flaky
+     * based on the sonatype server load.
+     */
+    def discoverSonatypeRepositoryUrl() {
+        try {
+            // Ensures that we don't create spurious repositories if we're not running a release.
+            if (project.properties.containsKey("release")) {
+                // Gradle may reuse the demon (hence its classloader) so this uses a map to store
+                // the staging repo for each build.
+                return STAGING_URL_CACHE.computeIfAbsent(
+                        project.rootProject, { project -> this.createStagingRepository() })
+            } else {
+                logger.debug("Release not enabled, defaulting to auto-staging")
+                return AUTO_STAGING_URL
+            }
+        } catch (Exception ex) {
+            // Handles issues by falling back to auto-staging.
+            logger.warn("Failed to create new staging repo. Attempting to deploy with " +
+                    "auto-staging. The release *may* complete as planned. You can proceed by" +
+                    " making sure that all release artifacts are present in the auto-staging " +
+                    "repository. Please investigate why staging failed to create.", ex)
+            return AUTO_STAGING_URL
+        }
+    }
+
+    /** Creates a staging repository on Sonatype. */
+    private def createStagingRepository() {
+        logger.info("Creating staging repository")
+        def stagingProfileUrls = getStagingProfileUrls()
+        if (stagingProfileUrls.size() == 0) {
+            throw new GradleException("No staging profiles found")
+        } else if (stagingProfileUrls.size() > 1) {
+            logger.warn("Found multiple staging profiles, using the first one")
+        }
+        def stagingStartUrl = new GenericUrl(stagingProfileUrls[0] + "/start")
+        def content = new PromoteRequestXmlContent(
+                NAMESPACE_DICTIONARY, generateStagingRepoDescription())
+        def response = REQUEST_FACTORY
+                .buildPostRequest(stagingStartUrl, content)
+                .setHeaders(authHeaders)
+                .execute()
+        // Checks the response and returns the repository URL, throwing if we
+        // didn't get a valid response.
+        if (response.getStatusCode() == 201) { // HTTP 201 = Created
+            def parsed = response.parseAs(PromoteRequestXmlContent.PromoteRequest.class)
+            logger.info("Staging repository created with ID ${parsed.data.stagedRepositoryId}")
+            return DEPLOY_BY_REPO_URL_STEM + parsed.data.stagedRepositoryId
+        } else {
+            throw new GradleException("Nexus failed to create staging repo: " +
+                    "${response.getStatusCode()}: ${response.getStatusMessage()}")
+        }
+    }
+
+    /** Retrieves the staging profiles from Nexus. */
+    private def getStagingProfileUrls() {
+        logger.debug("Retrieving staging profiles")
+        def response = REQUEST_FACTORY
+                .buildGetRequest(STAGING_PROFILES_URL)
+                .setHeaders(authHeaders)
+                .execute()
+        if (response.getStatusCode() == 200) {
+            def parsed = response.parseAs(StagingProfiles.class)
+            return parsed.getStagingProfileUrls()
+        } else {
+            throw new GradleException("Failed to retrieve staging profiles: HTTP " +
+                    "${response.getStatusCode()}: ${response.getStatusMessage()}")
+        }
+    }
+
+    /** Generates a pseudo-unique and human readable description for a staging repo. */
+    private def generateStagingRepoDescription() {
+        return "Release of Google Ads API v${project.version} " +
+                "on ${InetAddress.getLocalHost().getCanonicalHostName()} " +
+                "${new Date()}"
+    }
+}
+
+// ----------------------- Gradle project configuration ------------------------
+
+// Configures gradle's native publishing now that debacle is over.
+publishing {
+    publications {
+        maven(MavenPublication) {
+            from(components.java)
+        }
+    }
+    repositories {
+        maven {
+            url new SonatypeClient(project).discoverSonatypeRepositoryUrl()
+            name = "sonatype"
+            credentials {
+                // Avoids storing Sonatype credentials in plain-text. Specify these on
+                // the command line: -PsonatypeUser=foo
+                username project.properties.get("sonatypeUser")
+                password project.properties.get("sonatypePassword")
+            }
+        }
+    }
+}