build: use trampoline_v2 for python samples and allow custom dockerfi…

…le (#1232) * Use trampoline_v2 ([Docker CI Helper](https://github.com/GoogleCloudPlatform/docker-ci-helper)) for all python samples. * Optionally allow samples to run with a Dockerfile specified in the library repository at `.kokoro/docker/samples`. On a successful test run, this Dockerfile is uploaded to `gcr.io/cloud-devrel-kokoro-resources/{repo-name}-samples-docker`. The option is enabled by passing `custom_samples_dockerfile=True` to `common.py_library` in `owlbot.py`. ```py templated_files = common.py_library(microgenerator=True, custom_samples_dockerfile=True) ``` See googleapis/python-asset#308 for an example of moving samples to to trampoline_v2. See googleapis/python-recaptcha-enterprise#128 for an example of moving to trampoline_v2 + using a custom Dockerfile.
googleapis · Oct 5, 2021 · a7ed11e · a7ed11e
1 parent 2607ffc
commit a7ed11e
Show file tree

Hide file tree

Showing 16 changed files with 136 additions and 21 deletions.
diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py
@@ -237,6 +237,10 @@ def py_library(self, **kwargs) -> Path:
         if "samples" not in kwargs:
             self.excludes += ["samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md"]
 
+        # Assume the python-docs-samples Dockerfile is used for samples by default
+        if "custom_samples_dockerfile" not in kwargs:
+            kwargs["custom_samples_dockerfile"] = False
+
         ret = self._generic_library("python_library", **kwargs)
 
         # If split_system_tests is set to True, we disable the system

diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg
@@ -17,18 +17,29 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh"
 }
+{% if custom_samples_dockerfile %}
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker"
+}
 
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/samples/Dockerfile"
+}
+{% else %}
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
     value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 }
-
+{% endif %}
 # Download secrets for samples
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
 
 # Download trampoline resources.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh"
+build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh"
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg
@@ -23,18 +23,29 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh"
 }
+{% if custom_samples_dockerfile %}
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker"
+}
 
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/samples/Dockerfile"
+}
+{% else %}
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
     value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 }
-
+{% endif %}
 # Download secrets for samples
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
 
 # Download trampoline resources.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh"
+build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh"
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic-head.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic-head.cfg
@@ -9,3 +9,10 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples-against-head.sh"
 }
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg
@@ -3,4 +3,11 @@
 env_vars: {
     key: "INSTALL_LIBRARY_FROM_SOURCE"
     value: "False"
-}
+}
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg
@@ -23,18 +23,29 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh"
 }
+{% if custom_samples_dockerfile %}
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker"
+}
 
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/samples/Dockerfile"
+}
+{% else %}
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
     value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 }
-
+{% endif %}
 # Download secrets for samples
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
 
 # Download trampoline resources.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh"
+build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh"
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg
@@ -3,4 +3,11 @@
 env_vars: {
     key: "INSTALL_LIBRARY_FROM_SOURCE"
     value: "False"
-}
+}
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg
@@ -23,18 +23,29 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh"
 }
+{% if custom_samples_dockerfile %}
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker"
+}
 
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/samples/Dockerfile"
+}
+{% else %}
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
     value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 }
-
+{% endif %}
 # Download secrets for samples
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
 
 # Download trampoline resources.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh"
+build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh"
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic-head.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic-head.cfg
@@ -9,3 +9,10 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples-against-head.sh"
 }
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg
@@ -3,4 +3,11 @@
 env_vars: {
     key: "INSTALL_LIBRARY_FROM_SOURCE"
     value: "False"
-}
+}
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/common.cfg
@@ -23,18 +23,29 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh"
 }
+{% if custom_samples_dockerfile %}
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker"
+}
 
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/samples/Dockerfile"
+}
+{% else %}
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
     value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 }
-
+{% endif %}
 # Download secrets for samples
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
 
 # Download trampoline resources.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh"
+build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh"
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic-head.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic-head.cfg
@@ -9,3 +9,10 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples-against-head.sh"
 }
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic.cfg
@@ -3,4 +3,11 @@
 env_vars: {
     key: "INSTALL_LIBRARY_FROM_SOURCE"
     value: "False"
-}
+}
+{% if custom_samples_dockerfile %}
+# Upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+{% endif %}
diff --git a/synthtool/gcp/templates/python_library/.kokoro/test-samples-against-head.sh b/synthtool/gcp/templates/python_library/.kokoro/test-samples-against-head.sh
@@ -23,6 +23,4 @@ set -eo pipefail
 # Enables `**` to include files nested inside sub-folders
 shopt -s globstar
 
-cd github/{{ metadata['repo']['repo'].split('/')[1] }}
-
 exec .kokoro/test-samples-impl.sh
diff --git a/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh b/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh
@@ -24,8 +24,6 @@ set -eo pipefail
 # Enables `**` to include files nested inside sub-folders
 shopt -s globstar
 
-cd github/{{ metadata['repo']['repo'].split('/')[1] }}
-
 # Run periodic samples tests at latest release
 if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
     # preserving the test runner implementation.

diff --git a/synthtool/gcp/templates/python_library/.trampolinerc b/synthtool/gcp/templates/python_library/.trampolinerc
@@ -16,15 +16,26 @@
 
 # Add required env vars here.
 required_envvars+=(
-    "STAGING_BUCKET"
-    "V2_STAGING_BUCKET"
 )
 
 # Add env vars which are passed down into the container here.
 pass_down_envvars+=(
+    "NOX_SESSION"
+    ###############
+    # Docs builds
+    ###############
     "STAGING_BUCKET"
     "V2_STAGING_BUCKET"
-    "NOX_SESSION"
+    ##################
+    # Samples builds
+    ##################
+    "INSTALL_LIBRARY_FROM_SOURCE"
+    "RUN_TESTS_SESSION"
+    "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    # Target directories.
+    "RUN_TESTS_DIRS"
+    # The nox session to run.
+    "RUN_TESTS_SESSION"
 )
 
 # Prevent unintentional override on the default image.