revert generator and librarian

Author: ohmayr

.generator/Dockerfile

@@ -40,27 +40,25 @@ RUN apt-get update && \
     && apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# `make altinstall` is used to prevent replacing the system's default python binary.
+ENV PYTHON_VERSION=3.14
+
 # The full Python version, including the minor version, is needed for download/install
-# TODO(https://github.com/googleapis/librarian/issues/2945): Remove `3.13` when the linked issue is resolved.
-RUN for PYTHON_VERSION_WITH_MINOR in 3.13.9 3.14.0; do \
-    wget https://www.python.org/ftp/python/${PYTHON_VERSION_WITH_MINOR}/Python-${PYTHON_VERSION_WITH_MINOR}.tgz && \
+ENV PYTHON_VERSION_WITH_MINOR=3.14.2
+
+# `make altinstall` is used to prevent replacing the system's default python binary.
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION_WITH_MINOR}/Python-${PYTHON_VERSION_WITH_MINOR}.tgz && \
     tar -xvf Python-${PYTHON_VERSION_WITH_MINOR}.tgz && \
     cd Python-${PYTHON_VERSION_WITH_MINOR} && \
     ./configure --enable-optimizations --prefix=/usr/local && \
     make -j$(nproc) && \
     make altinstall && \
     cd / && \
-    rm -rf Python-${PYTHON_VERSION_WITH_MINOR}* \
-    ; done
+    rm -rf Python-${PYTHON_VERSION_WITH_MINOR}*
+
 
-# Install pip for each python version
-# TODO(https://github.com/googleapis/librarian/issues/2945): Remove `3.13` when the linked issue is resolved.
-RUN for PYTHON_VERSION in 3.13 3.14; do \
-    wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && \
+RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && \
     python${PYTHON_VERSION} /tmp/get-pip.py && \
-    rm /tmp/get-pip.py \
-    ; done
+    rm /tmp/get-pip.py
 
 # Download/extract protoc
 RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-linux-x86_64.zip
@@ -112,32 +110,20 @@ COPY --from=builder synthtool /synthtool
 COPY --from=builder /usr/local/bin/python${PYTHON_VERSION_DEFAULT}  /usr/local/bin/
 COPY --from=builder /usr/local/lib/python${PYTHON_VERSION_DEFAULT}  /usr/local/lib/python${PYTHON_VERSION_DEFAULT}
 
-# TODO(https://github.com/googleapis/librarian/issues/2945): Remove `3.13` when the linked issue is resolved.
-COPY --from=builder /usr/local/bin/python3.13  /usr/local/bin
-COPY --from=builder /usr/local/lib/python3.13  /usr/local/lib/python3.13
-
 # Set the working directory in the container.
 WORKDIR /app
 
-# --- CACHE CHECKPOINT ---
-# Everything above this line (including the pip install below) will be cached 
-# as long as 'requirements.in' doesn't change.
+# Install dependencies of the CLI such as click.
+# Install gapic-generator which is used to generate libraries.
+# Install nox which is used for running client library tests.
+# Install starlark-pyo3 which is used to parse BUILD.bazel files.
 COPY .generator/requirements.in .
 RUN python${PYTHON_VERSION_DEFAULT} -m pip install -r requirements.in
-
-# --- LOCAL OVERRIDES START HERE ---
-# By putting these instructions last, only these steps re-run when you change code.
-
-# 1. Overwrite the 'synthtool' directory with your local version
-# Note: This overwrites the version copied from 'builder' earlier.
-COPY synthtool /synthtool
 RUN python${PYTHON_VERSION_DEFAULT} -m pip install /synthtool
 
-# 2. Copy and install your local 'gapic-generator-python'
-COPY gapic-generator-python /gapic-generator-python
-RUN python${PYTHON_VERSION_DEFAULT} -m pip install /gapic-generator-python
-
-# --- FINAL CONFIGURATION ---
+# Install build which is used to get the metadata of package config files.
+COPY .generator/requirements.in .
+RUN python${PYTHON_VERSION_DEFAULT} -m pip install -r requirements.in
 
 # Copy the CLI script into the container.
 COPY .generator/cli.py .
@@ -147,5 +133,4 @@ RUN chmod a+rx ./cli.py
 COPY .generator/parse_googleapis_content.py .
 RUN chmod a+rx ./parse_googleapis_content.py
 
-ENV ENABLE_PERF_LOGS=1
 ENTRYPOINT ["python3.14", "./cli.py"]

.generator/cli.py

@@ -23,7 +23,6 @@
 import subprocess
 import sys
 import tempfile
-import time
 import yaml
 from datetime import date, datetime
 from functools import lru_cache
@@ -32,40 +31,6 @@
 import build.util
 import parse_googleapis_content
 
-logging.basicConfig(stream=sys.stdout, level=logging.INFO)
-
-import functools
-
-PERF_LOGGING_ENABLED = os.environ.get("ENABLE_PERF_LOGS") == "1"
-
-if PERF_LOGGING_ENABLED:
-    perf_logger = logging.getLogger("performance_metrics")
-    perf_logger.setLevel(logging.INFO)
-    perf_handler = logging.FileHandler("/tmp/performance_metrics.log", mode='w')
-    perf_formatter = logging.Formatter('%(asctime)s | %(message)s', datefmt='%H:%M:%S')
-    perf_handler.setFormatter(perf_formatter)
-    perf_logger.addHandler(perf_handler)
-    perf_logger.propagate = False
-
-def track_time(func):
-    """
-    Decorator. Usage: @track_time
-    If logging is OFF, it returns the original function (Zero Overhead).
-    If logging is ON, it wraps the function to measure execution time.
-    """
-    if not PERF_LOGGING_ENABLED:
-        return func
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        start_time = time.perf_counter()
-        try:
-            return func(*args, **kwargs)
-        finally:
-            duration = time.perf_counter() - start_time
-            perf_logger.info(f"{func.__name__:<30} | {duration:.4f} seconds")
-            
-    return wrapper
 
 try:
     import synthtool
@@ -325,9 +290,12 @@ def handle_configure(
         )
         prepared_config = _prepare_new_library_config(new_library_config)
 
-        # Create a `CHANGELOG.md` and `docs/CHANGELOG.md` file for the new library
+        is_mono_repo = _is_mono_repo(input)
         library_id = _get_library_id(prepared_config)
-        _create_new_changelog_for_library(library_id, output)
+        path_to_library = f"packages/{library_id}" if is_mono_repo else "."
+        if not Path(f"{repo}/{path_to_library}").exists():
+            # Create a `CHANGELOG.md` and `docs/CHANGELOG.md` file for the new library
+            _create_new_changelog_for_library(library_id, output)
 
         # Write the new library configuration to configure-response.json.
         _write_json_file(f"{librarian}/configure-response.json", prepared_config)
@@ -355,9 +323,8 @@ def _get_library_id(request_data: Dict) -> str:
     return library_id
 
 
-@track_time
 def _run_post_processor(output: str, library_id: str, is_mono_repo: bool):
-    """Runs the synthtool post-processor (templates) and Ruff formatter (lint/format).
+    """Runs the synthtool post-processor on the output directory.
 
     Args:
         output(str): Path to the directory in the container where code
@@ -367,58 +334,25 @@ def _run_post_processor(output: str, library_id: str, is_mono_repo: bool):
     """
     os.chdir(output)
     path_to_library = f"packages/{library_id}" if is_mono_repo else "."
-    
-    # 1. Run Synthtool (Templates & Fixers only)
-    # Note: This relies on 'nox' being disabled in your environment (via run_fast.sh shim)
-    # to avoid the slow formatting step inside owlbot.
-    logger.info("Running Python post-processor (Templates & Fixers)...")
+    logger.info("Running Python post-processor...")
     if SYNTHTOOL_INSTALLED:
-        try:
-            if is_mono_repo:
-                python_mono_repo.owlbot_main(path_to_library)
+        if is_mono_repo:
+            python_mono_repo.owlbot_main(path_to_library)
+        else:
+            # Some repositories have customizations in `librarian.py`.
+            # If this file exists, run those customizations instead of `owlbot_main`
+            if Path(f"{output}/librarian.py").exists():
+                subprocess.run(["python3.14", f"{output}/librarian.py"])
             else:
-                # Handle custom librarian scripts if present
-                if Path(f"{output}/librarian.py").exists():
-                    subprocess.run(["python3.14", f"{output}/librarian.py"])
-                else:
-                    python.owlbot_main()
-        except Exception as e:
-            logger.warning(f"Synthtool warning (non-fatal): {e}")
-
-    # 2. Run RUFF (Fast Formatter & Import Sorter)
-    # This replaces both 'isort' and 'black' and runs in < 1 second.
-    # We hardcode flags here to match Black defaults so you don't need config files.
-    # logger.info("🚀 Running Ruff (Fast Formatter)...")
-    try:
-        # STEP A: Fix Imports (like isort)
-        subprocess.run(
-            [
-                "ruff", "check", 
-                "--select", "I",               # Only run Import sorting rules
-                "--fix",                       # Auto-fix them
-                "--line-length=88",            # Match Black default
-                "--known-first-party=google",  # Prevent 'google' moving to 3rd party block
-                output
-            ],
-            check=False, 
-            stdout=subprocess.DEVNULL, 
-            stderr=subprocess.DEVNULL
-        )
+                python.owlbot_main()
+    else:
+        raise SYNTHTOOL_IMPORT_ERROR  # pragma: NO COVER
 
-        # STEP B: Format Code (like black)
-        subprocess.run(
-            [
-                "ruff", "format", 
-                "--line-length=88",            # Match Black default
-                output
-            ],
-            check=False, 
-            stdout=subprocess.DEVNULL, 
-            stderr=subprocess.DEVNULL
-        )
-    except FileNotFoundError:
-        logger.warning("⚠️ Ruff binary not found. Code will be unformatted.")
-        logger.warning("   Please run: pip install ruff")
+    # If there is no noxfile, run `isort`` and `black` on the output.
+    # This is required for proto-only libraries which are not GAPIC.
+    if not Path(f"{output}/{path_to_library}/noxfile.py").exists():
+        subprocess.run(["isort", output])
+        subprocess.run(["black", output])
 
     logger.info("Python post-processor ran successfully.")
 
@@ -458,7 +392,6 @@ def _add_header_to_files(directory: str) -> None:
                 f.writelines(lines)
 
 
-@track_time
 def _copy_files_needed_for_post_processing(
     output: str, input: str, library_id: str, is_mono_repo: bool
 ):
@@ -480,13 +413,19 @@ def _copy_files_needed_for_post_processing(
     destination_dir = f"{output}/{path_to_library}"
 
     if Path(source_dir).exists():
-        shutil.copytree(
-            source_dir,
-            destination_dir,
-            dirs_exist_ok=True,
-        )
-        # Apply headers only to the generator-input files copied above.
-        _add_header_to_files(destination_dir)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            shutil.copytree(
+                source_dir,
+                tmp_dir,
+                dirs_exist_ok=True,
+            )
+            # Apply headers only to the generator-input files copied above.
+            _add_header_to_files(tmp_dir)
+            shutil.copytree(
+                tmp_dir,
+                destination_dir,
+                dirs_exist_ok=True,
+            )
 
     # We need to create these directories so that we can copy files necessary for post-processing.
     os.makedirs(
@@ -505,7 +444,6 @@ def _copy_files_needed_for_post_processing(
                 )
 
 
-@track_time
 def _clean_up_files_after_post_processing(
     output: str, library_id: str, is_mono_repo: bool
 ):
@@ -652,7 +590,6 @@ def _get_repo_name_from_repo_metadata(base: str, library_id: str, is_mono_repo:
     return repo_name
 
 
-@track_time
 def _generate_repo_metadata_file(
     output: str, library_id: str, source: str, apis: List[Dict], is_mono_repo: bool
 ):
@@ -694,7 +631,6 @@ def _generate_repo_metadata_file(
     _write_json_file(output_repo_metadata, metadata_content)
 
 
-@track_time
 def _copy_readme_to_docs(output: str, library_id: str, is_mono_repo: bool):
     """Copies the README.rst file for a generated library to docs/README.rst.
 
@@ -736,7 +672,6 @@ def _copy_readme_to_docs(output: str, library_id: str, is_mono_repo: bool):
         f.write(content)
 
 
-@track_time
 def handle_generate(
     librarian: str = LIBRARIAN_DIR,
     source: str = SOURCE_DIR,
@@ -785,7 +720,6 @@ def handle_generate(
         _run_post_processor(output, library_id, is_mono_repo)
         _copy_readme_to_docs(output, library_id, is_mono_repo)
         _clean_up_files_after_post_processing(output, library_id, is_mono_repo)
-
     except Exception as e:
         raise ValueError("Generation failed.") from e
     logger.info("'generate' command executed.")
@@ -999,7 +933,6 @@ def _stage_gapic_library(tmp_dir: str, staging_dir: str) -> None:
     shutil.copytree(tmp_dir, staging_dir, dirs_exist_ok=True)
 
 
-@track_time
 def _generate_api(
     api_path: str,
     library_id: str,
@@ -1062,18 +995,12 @@ def _run_nox_sessions(library_id: str, repo: str, is_mono_repo: bool):
             the config.yaml.
         is_mono_repo(bool): True if the current repository is a mono-repo.
     """
-    path_to_library = f"{repo}/packages/{library_id}" if is_mono_repo else repo
-    _python_314_supported = Path(
-        f"{path_to_library}/testing/constraints-3.14.txt"
-    ).exists()
-
-    if _python_314_supported:
-        session_runtime = "3.14"
-    else:
-        session_runtime = "3.13"
-
+    session_runtime = "3.14"
+    # TODO(https://github.com/googleapis/google-cloud-python/issues/14992): Switch the protobuf
+    # implementation back to upb once we identify the root cause of the crash that occurs during testing.
+    # It's not trivial to debug this since it only happens in cloud build.
     sessions = [
-        f"unit-{session_runtime}(protobuf_implementation='upb')",
+        f"unit-{session_runtime}(protobuf_implementation='python')",
     ]
     current_session = None
     try:
@@ -1112,7 +1039,8 @@ def _run_individual_session(
         "-f",
         f"{library_path}/noxfile.py",
     ]
-    result = subprocess.run(command, text=True, check=True, timeout=600)
+    # TODO(#14992): Revert to 600 seconds (10 minutes) after debugging is complete.
+    result = subprocess.run(command, text=True, check=True, timeout=1200)
     logger.info(result)
 
 
@@ -1820,7 +1748,6 @@ def handle_release_stage(
             output=args.output,
             input=args.input,
         )
-
     elif args.command == "build":
         args.func(librarian=args.librarian, repo=args.repo)
     elif args.command == "release-stage":

.generator/requirements.in

@@ -1,6 +1,7 @@
 click
-gapic-generator==1.28.3 # format generated samples
+gapic-generator==1.30.3 # Fix mypy checks https://github.com/googleapis/gapic-generator-python/pull/2520
 nox
 starlark-pyo3>=2025.1
 build
-ruff
+black==23.7.0
+isort==5.11.0