From cc12f3ec929d0359080cb6233c598e4f6d1f4f69 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Mon, 16 Oct 2023 19:34:46 +0530 Subject: [PATCH] [Examples] Update with HFApi (#5393) * update training examples to use HFAPI. * update training example. * reflect the changes in the korean version too. * Empty-Commit --- docs/source/en/tutorials/basic_training.md | 29 ++++++-------- docs/source/ko/tutorials/basic_training.md | 32 +++++++-------- examples/dreambooth/train_dreambooth_flax.py | 40 ++++++------------- .../textual_inversion.py | 40 +++++++------------ .../train_unconditional.py | 40 ++++++------------- .../train_unconditional.py | 40 ++++++------------- 6 files changed, 79 insertions(+), 142 deletions(-) diff --git a/docs/source/en/tutorials/basic_training.md b/docs/source/en/tutorials/basic_training.md index c97447e54b..b2243a7597 100644 --- a/docs/source/en/tutorials/basic_training.md +++ b/docs/source/en/tutorials/basic_training.md @@ -284,22 +284,11 @@ Now you can wrap all these components together in a training loop with ๐Ÿค— Acce ```py >>> from accelerate import Accelerator ->>> from huggingface_hub import HfFolder, Repository, whoami +>>> from huggingface_hub import create_repo, upload_folder >>> from tqdm.auto import tqdm >>> from pathlib import Path >>> import os - ->>> def get_full_repo_name(model_id: str, organization: str = None, token: str = None): -... if token is None: -... token = HfFolder.get_token() -... if organization is None: -... username = whoami(token)["name"] -... return f"{username}/{model_id}" -... else: -... return f"{organization}/{model_id}" - - >>> def train_loop(config, model, noise_scheduler, optimizer, train_dataloader, lr_scheduler): ... # Initialize accelerator and tensorboard logging ... accelerator = Accelerator( @@ -309,11 +298,12 @@ Now you can wrap all these components together in a training loop with ๐Ÿค— Acce ... project_dir=os.path.join(config.output_dir, "logs"), ... ) ... if accelerator.is_main_process: -... if config.push_to_hub: -... repo_name = get_full_repo_name(Path(config.output_dir).name) -... repo = Repository(config.output_dir, clone_from=repo_name) -... elif config.output_dir is not None: +... if config.output_dir is not None: ... os.makedirs(config.output_dir, exist_ok=True) +... if config.push_to_hub: +... repo_id = create_repo( +... repo_id=config.hub_model_id or Path(config.output_dir).name, exist_ok=True +... ) ... accelerator.init_trackers("train_example") ... # Prepare everything @@ -371,7 +361,12 @@ Now you can wrap all these components together in a training loop with ๐Ÿค— Acce ... if (epoch + 1) % config.save_model_epochs == 0 or epoch == config.num_epochs - 1: ... if config.push_to_hub: -... repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=True) +... upload_folder( +... repo_id=repo_id, +... folder_path=config.output_dir, +... commit_message=f"Epoch {epoch}", +... ignore_patterns=["step_*", "epoch_*"], +... ) ... else: ... pipeline.save_pretrained(config.output_dir) ``` diff --git a/docs/source/ko/tutorials/basic_training.md b/docs/source/ko/tutorials/basic_training.md index e18c82c4fd..7bd5ad44dd 100644 --- a/docs/source/ko/tutorials/basic_training.md +++ b/docs/source/ko/tutorials/basic_training.md @@ -283,36 +283,27 @@ TensorBoard์— ๋กœ๊น…, ๊ทธ๋ž˜๋””์–ธํŠธ ๋ˆ„์  ๋ฐ ํ˜ผํ•ฉ ์ •๋ฐ€๋„ ํ•™์Šต์„ ์‰ฝ ```py >>> from accelerate import Accelerator ->>> from huggingface_hub import HfFolder, Repository, whoami +>>> from huggingface_hub import create_repo, upload_folder >>> from tqdm.auto import tqdm >>> from pathlib import Path >>> import os ->>> def get_full_repo_name(model_id: str, organization: str = None, token: str = None): -... if token is None: -... token = HfFolder.get_token() -... if organization is None: -... username = whoami(token)["name"] -... return f"{username}/{model_id}" -... else: -... return f"{organization}/{model_id}" - - >>> def train_loop(config, model, noise_scheduler, optimizer, train_dataloader, lr_scheduler): -... # accelerator์™€ tensorboard ๋กœ๊น… ์ดˆ๊ธฐํ™” +... # Initialize accelerator and tensorboard logging ... accelerator = Accelerator( ... mixed_precision=config.mixed_precision, ... gradient_accumulation_steps=config.gradient_accumulation_steps, ... log_with="tensorboard", -... logging_dir=os.path.join(config.output_dir, "logs"), +... project_dir=os.path.join(config.output_dir, "logs"), ... ) ... if accelerator.is_main_process: -... if config.push_to_hub: -... repo_name = get_full_repo_name(Path(config.output_dir).name) -... repo = Repository(config.output_dir, clone_from=repo_name) -... elif config.output_dir is not None: +... if config.output_dir is not None: ... os.makedirs(config.output_dir, exist_ok=True) +... if config.push_to_hub: +... repo_id = create_repo( +... repo_id=config.hub_model_id or Path(config.output_dir).name, exist_ok=True +... ) ... accelerator.init_trackers("train_example") ... # ๋ชจ๋“  ๊ฒƒ์ด ์ค€๋น„๋˜์—ˆ์Šต๋‹ˆ๋‹ค. @@ -369,7 +360,12 @@ TensorBoard์— ๋กœ๊น…, ๊ทธ๋ž˜๋””์–ธํŠธ ๋ˆ„์  ๋ฐ ํ˜ผํ•ฉ ์ •๋ฐ€๋„ ํ•™์Šต์„ ์‰ฝ ... if (epoch + 1) % config.save_model_epochs == 0 or epoch == config.num_epochs - 1: ... if config.push_to_hub: -... repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=True) +... upload_folder( +... repo_id=repo_id, +... folder_path=config.output_dir, +... commit_message=f"Epoch {epoch}", +... ignore_patterns=["step_*", "epoch_*"], +... ) ... else: ... pipeline.save_pretrained(config.output_dir) ``` diff --git a/examples/dreambooth/train_dreambooth_flax.py b/examples/dreambooth/train_dreambooth_flax.py index 4ac4f969ee..a436d36ceb 100644 --- a/examples/dreambooth/train_dreambooth_flax.py +++ b/examples/dreambooth/train_dreambooth_flax.py @@ -4,7 +4,6 @@ import logging import math import os from pathlib import Path -from typing import Optional import jax import jax.numpy as jnp @@ -16,7 +15,7 @@ import transformers from flax import jax_utils from flax.training import train_state from flax.training.common_utils import shard -from huggingface_hub import HfFolder, Repository, create_repo, whoami +from huggingface_hub import create_repo, upload_folder from jax.experimental.compilation_cache import compilation_cache as cc from PIL import Image from torch.utils.data import Dataset @@ -318,16 +317,6 @@ class PromptDataset(Dataset): return example -def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None): - if token is None: - token = HfFolder.get_token() - if organization is None: - username = whoami(token)["name"] - return f"{username}/{model_id}" - else: - return f"{organization}/{model_id}" - - def get_params_to_save(params): return jax.device_get(jax.tree_util.tree_map(lambda x: x[0], params)) @@ -392,22 +381,14 @@ def main(): # Handle the repository creation if jax.process_index() == 0: - if args.push_to_hub: - if args.hub_model_id is None: - repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token) - else: - repo_name = args.hub_model_id - create_repo(repo_name, exist_ok=True, token=args.hub_token) - repo = Repository(args.output_dir, clone_from=repo_name, token=args.hub_token) - - with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore: - if "step_*" not in gitignore: - gitignore.write("step_*\n") - if "epoch_*" not in gitignore: - gitignore.write("epoch_*\n") - elif args.output_dir is not None: + if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + # Load the tokenizer and add the placeholder token as a additional special token if args.tokenizer_name: tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name) @@ -668,7 +649,12 @@ def main(): if args.push_to_hub: message = f"checkpoint-{step}" if step is not None else "End of training" - repo.push_to_hub(commit_message=message, blocking=False, auto_lfs_prune=True) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message=message, + ignore_patterns=["step_*", "epoch_*"], + ) global_step = 0 diff --git a/examples/research_projects/intel_opts/textual_inversion_dfq/textual_inversion.py b/examples/research_projects/intel_opts/textual_inversion_dfq/textual_inversion.py index b19dd6e110..4366718759 100644 --- a/examples/research_projects/intel_opts/textual_inversion_dfq/textual_inversion.py +++ b/examples/research_projects/intel_opts/textual_inversion_dfq/textual_inversion.py @@ -4,7 +4,7 @@ import math import os import random from pathlib import Path -from typing import Iterable, Optional +from typing import Iterable import numpy as np import PIL @@ -13,7 +13,7 @@ import torch.nn.functional as F import torch.utils.checkpoint from accelerate import Accelerator from accelerate.utils import ProjectConfiguration, set_seed -from huggingface_hub import HfFolder, Repository, whoami +from huggingface_hub import create_repo, upload_folder from neural_compressor.utils import logger from packaging import version from PIL import Image @@ -413,16 +413,6 @@ class TextualInversionDataset(Dataset): return example -def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None): - if token is None: - token = HfFolder.get_token() - if organization is None: - username = whoami(token)["name"] - return f"{username}/{model_id}" - else: - return f"{organization}/{model_id}" - - def freeze_params(params): for param in params: param.requires_grad = False @@ -461,21 +451,14 @@ def main(): # Handle the repository creation if accelerator.is_main_process: - if args.push_to_hub: - if args.hub_model_id is None: - repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token) - else: - repo_name = args.hub_model_id - repo = Repository(args.output_dir, clone_from=repo_name) - - with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore: - if "step_*" not in gitignore: - gitignore.write("step_*\n") - if "epoch_*" not in gitignore: - gitignore.write("epoch_*\n") - elif args.output_dir is not None: + if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + # Load the tokenizer and add the placeholder token as a additional special token if args.tokenizer_name: tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name) @@ -982,7 +965,12 @@ def main(): ) if args.push_to_hub: - repo.push_to_hub(commit_message="End of training", blocking=False, auto_lfs_prune=True) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) accelerator.end_training() diff --git a/examples/research_projects/onnxruntime/unconditional_image_generation/train_unconditional.py b/examples/research_projects/onnxruntime/unconditional_image_generation/train_unconditional.py index ba5ccd238f..5cad9f2fbe 100644 --- a/examples/research_projects/onnxruntime/unconditional_image_generation/train_unconditional.py +++ b/examples/research_projects/onnxruntime/unconditional_image_generation/train_unconditional.py @@ -4,7 +4,6 @@ import logging import math import os from pathlib import Path -from typing import Optional import accelerate import datasets @@ -14,7 +13,7 @@ from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import ProjectConfiguration from datasets import load_dataset -from huggingface_hub import HfFolder, Repository, create_repo, whoami +from huggingface_hub import create_repo, upload_folder from onnxruntime.training.optim.fp16_optimizer import FP16_Optimizer as ORT_FP16_Optimizer from onnxruntime.training.ortmodule import ORTModule from packaging import version @@ -277,16 +276,6 @@ def parse_args(): return args -def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None): - if token is None: - token = HfFolder.get_token() - if organization is None: - username = whoami(token)["name"] - return f"{username}/{model_id}" - else: - return f"{organization}/{model_id}" - - def main(args): logging_dir = os.path.join(args.output_dir, args.logging_dir) accelerator_project_config = ProjectConfiguration( @@ -360,22 +349,14 @@ def main(args): # Handle the repository creation if accelerator.is_main_process: - if args.push_to_hub: - if args.hub_model_id is None: - repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token) - else: - repo_name = args.hub_model_id - create_repo(repo_name, exist_ok=True, token=args.hub_token) - repo = Repository(args.output_dir, clone_from=repo_name, token=args.hub_token) - - with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore: - if "step_*" not in gitignore: - gitignore.write("step_*\n") - if "epoch_*" not in gitignore: - gitignore.write("epoch_*\n") - elif args.output_dir is not None: + if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + # Initialize the model if args.model_config_name_or_path is None: model = UNet2DModel( @@ -691,7 +672,12 @@ def main(args): ema_model.restore(unet.parameters()) if args.push_to_hub: - repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=False) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message=f"Epoch {epoch}", + ignore_patterns=["step_*", "epoch_*"], + ) accelerator.end_training() diff --git a/examples/unconditional_image_generation/train_unconditional.py b/examples/unconditional_image_generation/train_unconditional.py index 12b63439fa..74b8ed1068 100644 --- a/examples/unconditional_image_generation/train_unconditional.py +++ b/examples/unconditional_image_generation/train_unconditional.py @@ -6,7 +6,6 @@ import os import shutil from datetime import timedelta from pathlib import Path -from typing import Optional import accelerate import datasets @@ -16,7 +15,7 @@ from accelerate import Accelerator, InitProcessGroupKwargs from accelerate.logging import get_logger from accelerate.utils import ProjectConfiguration from datasets import load_dataset -from huggingface_hub import HfFolder, Repository, create_repo, whoami +from huggingface_hub import create_repo, upload_folder from packaging import version from torchvision import transforms from tqdm.auto import tqdm @@ -273,16 +272,6 @@ def parse_args(): return args -def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None): - if token is None: - token = HfFolder.get_token() - if organization is None: - username = whoami(token)["name"] - return f"{username}/{model_id}" - else: - return f"{organization}/{model_id}" - - def main(args): logging_dir = os.path.join(args.output_dir, args.logging_dir) accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) @@ -356,22 +345,14 @@ def main(args): # Handle the repository creation if accelerator.is_main_process: - if args.push_to_hub: - if args.hub_model_id is None: - repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token) - else: - repo_name = args.hub_model_id - create_repo(repo_name, exist_ok=True, token=args.hub_token) - repo = Repository(args.output_dir, clone_from=repo_name, token=args.hub_token) - - with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore: - if "step_*" not in gitignore: - gitignore.write("step_*\n") - if "epoch_*" not in gitignore: - gitignore.write("epoch_*\n") - elif args.output_dir is not None: + if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + # Initialize the model if args.model_config_name_or_path is None: model = UNet2DModel( @@ -708,7 +689,12 @@ def main(args): ema_model.restore(unet.parameters()) if args.push_to_hub: - repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=False) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message=f"Epoch {epoch}", + ignore_patterns=["step_*", "epoch_*"], + ) accelerator.end_training()