|
| 1 | +""" |
| 2 | + Copyright 2024 Google LLC |
| 3 | +
|
| 4 | + Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + you may not use this file except in compliance with the License. |
| 6 | + You may obtain a copy of the License at |
| 7 | +
|
| 8 | + https://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | + Unless required by applicable law or agreed to in writing, software |
| 11 | + distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + See the License for the specific language governing permissions and |
| 14 | + limitations under the License. |
| 15 | + """ |
| 16 | + |
| 17 | +from abc import ABC |
| 18 | +from contextlib import nullcontext |
| 19 | +import os |
| 20 | +import json |
| 21 | +import functools |
| 22 | +import jax |
| 23 | +import jax.numpy as jnp |
| 24 | +from jax.sharding import Mesh, PositionalSharding, PartitionSpec as P |
| 25 | +import orbax.checkpoint as ocp |
| 26 | +import grain.python as grain |
| 27 | +from maxdiffusion import ( |
| 28 | + max_utils, |
| 29 | + FlaxAutoencoderKL, |
| 30 | + max_logging, |
| 31 | +) |
| 32 | +from maxdiffusion.models.flux.transformers.transformer_flux_flax import FluxTransformer2DModel |
| 33 | +from ..pipelines.flux.flux_pipeline import FluxPipeline |
| 34 | + |
| 35 | +from transformers import (CLIPTokenizer, FlaxCLIPTextModel, T5EncoderModel, FlaxT5EncoderModel, AutoTokenizer) |
| 36 | + |
| 37 | +from maxdiffusion.checkpointing.checkpointing_utils import ( |
| 38 | + create_orbax_checkpoint_manager, |
| 39 | + load_stable_diffusion_configs, |
| 40 | +) |
| 41 | +from maxdiffusion.models.flux.util import load_flow_model |
| 42 | + |
| 43 | +FLUX_CHECKPOINT = "FLUX_CHECKPOINT" |
| 44 | +_CHECKPOINT_FORMAT_DIFFUSERS = "CHECKPOINT_FORMAT_DIFFUSERS" |
| 45 | +_CHECKPOINT_FORMAT_ORBAX = "CHECKPOINT_FORMAT_ORBAX" |
| 46 | + |
| 47 | + |
| 48 | +class FluxCheckpointer(ABC): |
| 49 | + |
| 50 | + def __init__(self, config, checkpoint_type): |
| 51 | + self.config = config |
| 52 | + self.checkpoint_type = checkpoint_type |
| 53 | + self.checkpoint_format = None |
| 54 | + |
| 55 | + self.rng = jax.random.PRNGKey(self.config.seed) |
| 56 | + self.devices_array = max_utils.create_device_mesh(config) |
| 57 | + self.mesh = Mesh(self.devices_array, self.config.mesh_axes) |
| 58 | + self.total_train_batch_size = self.config.total_train_batch_size |
| 59 | + |
| 60 | + self.checkpoint_manager = create_orbax_checkpoint_manager( |
| 61 | + self.config.checkpoint_dir, |
| 62 | + enable_checkpointing=True, |
| 63 | + save_interval_steps=1, |
| 64 | + checkpoint_type=checkpoint_type, |
| 65 | + dataset_type=config.dataset_type, |
| 66 | + ) |
| 67 | + |
| 68 | + def _create_optimizer(self, config, learning_rate): |
| 69 | + |
| 70 | + learning_rate_scheduler = max_utils.create_learning_rate_schedule( |
| 71 | + learning_rate, config.learning_rate_schedule_steps, config.warmup_steps_fraction, config.max_train_steps |
| 72 | + ) |
| 73 | + tx = max_utils.create_optimizer(config, learning_rate_scheduler) |
| 74 | + return tx, learning_rate_scheduler |
| 75 | + |
| 76 | + def create_flux_state(self, pipeline, params, checkpoint_item_name, is_training): |
| 77 | + transformer = pipeline.flux |
| 78 | + |
| 79 | + tx, learning_rate_scheduler = None, None |
| 80 | + if is_training: |
| 81 | + learning_rate = self.config.learning_rate |
| 82 | + |
| 83 | + tx, learning_rate_scheduler = self._create_optimizer(self.config, learning_rate) |
| 84 | + |
| 85 | + transformer_eval_params = transformer.init_weights( |
| 86 | + rngs=self.rng, max_sequence_length=self.config.max_sequence_length, eval_only=True |
| 87 | + ) |
| 88 | + |
| 89 | + transformer_params = load_flow_model(self.config.flux_name, transformer_eval_params, "cpu") |
| 90 | + |
| 91 | + weights_init_fn = functools.partial(pipeline.flux.init_weights, rngs=self.rng, max_sequence_length=self.config.max_sequence_length) |
| 92 | + flux_state, state_mesh_shardings = max_utils.setup_initial_state( |
| 93 | + model=pipeline.flux, |
| 94 | + tx=tx, |
| 95 | + config=self.config, |
| 96 | + mesh=self.mesh, |
| 97 | + weights_init_fn=weights_init_fn, |
| 98 | + model_params=None, |
| 99 | + checkpoint_manager=self.checkpoint_manager, |
| 100 | + checkpoint_item=checkpoint_item_name, |
| 101 | + training=is_training, |
| 102 | + ) |
| 103 | + if not self.config.train_new_flux: |
| 104 | + flux_state = flux_state.replace(params=transformer_params) |
| 105 | + flux_state = jax.device_put(flux_state, state_mesh_shardings) |
| 106 | + return flux_state, state_mesh_shardings, learning_rate_scheduler |
| 107 | + |
| 108 | + def create_vae_state(self, pipeline, params, checkpoint_item_name, is_training=False): |
| 109 | + |
| 110 | + # Currently VAE training is not supported. |
| 111 | + weights_init_fn = functools.partial(pipeline.vae.init_weights, rng=self.rng) |
| 112 | + return max_utils.setup_initial_state( |
| 113 | + model=pipeline.vae, |
| 114 | + tx=None, |
| 115 | + config=self.config, |
| 116 | + mesh=self.mesh, |
| 117 | + weights_init_fn=weights_init_fn, |
| 118 | + model_params=params, |
| 119 | + checkpoint_manager=self.checkpoint_manager, |
| 120 | + checkpoint_item=checkpoint_item_name, |
| 121 | + training=is_training, |
| 122 | + ) |
| 123 | + |
| 124 | + def restore_data_iterator_state(self, data_iterator): |
| 125 | + if ( |
| 126 | + self.config.dataset_type == "grain" |
| 127 | + and data_iterator is not None |
| 128 | + and (self.checkpoint_manager.directory / str(self.checkpoint_manager.latest_step()) / "iter").exists() |
| 129 | + ): |
| 130 | + max_logging.log("Restoring data iterator from checkpoint") |
| 131 | + restored = self.checkpoint_manager.restore( |
| 132 | + self.checkpoint_manager.latest_step(), |
| 133 | + args=ocp.args.Composite(iter=grain.PyGrainCheckpointRestore(data_iterator.local_iterator)), |
| 134 | + ) |
| 135 | + data_iterator.local_iterator = restored["iter"] |
| 136 | + else: |
| 137 | + max_logging.log("data iterator checkpoint not found") |
| 138 | + return data_iterator |
| 139 | + |
| 140 | + def _get_pipeline_class(self): |
| 141 | + return FluxPipeline |
| 142 | + |
| 143 | + def _set_checkpoint_format(self, checkpoint_format): |
| 144 | + self.checkpoint_format = checkpoint_format |
| 145 | + |
| 146 | + def save_checkpoint(self, train_step, pipeline, train_states): |
| 147 | + items = { |
| 148 | + "config": ocp.args.JsonSave({"model_name": self.config.model_name}), |
| 149 | + } |
| 150 | + |
| 151 | + items["flux_state"] = ocp.args.PyTreeSave(train_states["flux_state"]) |
| 152 | + |
| 153 | + self.checkpoint_manager.save(train_step, args=ocp.args.Composite(**items)) |
| 154 | + |
| 155 | + def load_params(self, step=None): |
| 156 | + |
| 157 | + self.checkpoint_format = _CHECKPOINT_FORMAT_ORBAX |
| 158 | + |
| 159 | + def load_checkpoint(self, step=None, scheduler_class=None): |
| 160 | + clip_encoder = FlaxCLIPTextModel.from_pretrained( |
| 161 | + self.config.clip_model_name_or_path, dtype=self.config.weights_dtype |
| 162 | + ) |
| 163 | + clip_tokenizer = CLIPTokenizer.from_pretrained( |
| 164 | + self.config.clip_model_name_or_path, max_length=77, use_fast=True |
| 165 | + ) |
| 166 | + |
| 167 | + t5_encoder = FlaxT5EncoderModel.from_pretrained(self.config.t5xxl_model_name_or_path, dtype=self.config.weights_dtype) |
| 168 | + t5_tokenizer = AutoTokenizer.from_pretrained( |
| 169 | + self.config.t5xxl_model_name_or_path, max_length=self.config.max_sequence_length, use_fast=True |
| 170 | + ) |
| 171 | + encoders_sharding = PositionalSharding(self.devices_array).replicate() |
| 172 | + partial_device_put_replicated = functools.partial(max_utils.device_put_replicated, sharding=encoders_sharding) |
| 173 | + clip_encoder.params = jax.tree_util.tree_map(lambda x: x.astype(jnp.bfloat16), clip_encoder.params) |
| 174 | + clip_encoder.params = jax.tree_util.tree_map(partial_device_put_replicated, clip_encoder.params) |
| 175 | + t5_encoder.params = jax.tree_util.tree_map(lambda x: x.astype(jnp.bfloat16), t5_encoder.params) |
| 176 | + t5_encoder.params = jax.tree_util.tree_map(partial_device_put_replicated, t5_encoder.params) |
| 177 | + |
| 178 | + |
| 179 | + |
| 180 | + vae, vae_params = FlaxAutoencoderKL.from_pretrained( |
| 181 | + self.config.pretrained_model_name_or_path, subfolder="vae", from_pt=True, use_safetensors=True, dtype="bfloat16" |
| 182 | + ) |
| 183 | + |
| 184 | + flash_block_sizes = max_utils.get_flash_block_sizes(self.config) |
| 185 | + # loading from pretrained here causes a crash when trying to compile the model |
| 186 | + # Failed to load HSACO: HIP_ERROR_NoBinaryForGpu |
| 187 | + transformer = FluxTransformer2DModel.from_config( |
| 188 | + self.config.pretrained_model_name_or_path, |
| 189 | + subfolder="transformer", |
| 190 | + mesh=self.mesh, |
| 191 | + split_head_dim=self.config.split_head_dim, |
| 192 | + attention_kernel=self.config.attention, |
| 193 | + flash_block_sizes=flash_block_sizes, |
| 194 | + dtype=self.config.activations_dtype, |
| 195 | + weights_dtype=self.config.weights_dtype, |
| 196 | + precision=max_utils.get_precision(self.config), |
| 197 | + ) |
| 198 | + |
| 199 | + return FluxPipeline(t5_encoder, |
| 200 | + clip_encoder, |
| 201 | + vae, |
| 202 | + t5_tokenizer, |
| 203 | + clip_tokenizer, |
| 204 | + transformer, |
| 205 | + None, |
| 206 | + dtype=self.config.activations_dtype, |
| 207 | + mesh=self.mesh, |
| 208 | + config=self.config, |
| 209 | + rng=self.rng), vae_params |
| 210 | + |
0 commit comments