#!/usr/bin/env python3 """Build a *single-glacier* (or few-glacier) prepro tar from an existing prepro_base_url, reproducing OGGM's exact nested-tar folder structure. OGGM serves preprocessed glacier directories as a two-level tar: {base}/RGI62/b_{border:03d}/L{level}/RGI60-11/RGI60-11.00.tar (uncompressed) +-- RGI60-11.00/RGI60-11.00897.tar.gz (one .tar.gz per glacier, ~1000) +-- RGI60-11.00/RGI60-11.00898.tar.gz +-- ... `init_glacier_directories(..., from_prepro_level=...)` downloads that whole ~1000-glacier `RGI60-11.00.tar` just to extract one glacier - which is why pulling a single test glacier can take many minutes. This script downloads only the requested glacier(s) (via the normal OGGM machinery), then rewrites the same `RGI60-11.00.tar` containing *only* those glaciers, under a target location that mirrors `prepro_base_url`. The result is a drop-in `prepro_base_url` for tests/light runs: tiny and fast to fetch. Example (the cluster use case):: python make_test_gdir.py \ --prepro-base-url https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.6/L3-L5_files/2023.1/elev_bands/W5E5/ \ --target-dir /home/www/oggm/test_gdirs/ \ --level 3 --border 160 \ --rgi-id RGI60-11.00897 writes:: /home/www/oggm/test_gdirs/oggm_v1.6/L3-L5_files/2023.1/elev_bands/W5E5/RGI62/b_160/L3/RGI60-11/RGI60-11.00.tar `--target-dir` is just the destination root: everything in `--prepro-base-url` after `gdirs/` is reproduced underneath it, plus OGGM's canonical RGI/b_/L//.tar suffix (via oggm.utils.get_prepro_base_url), so the output always matches what OGGM will later look for. Pass several `--rgi-id` to bundle more than one glacier; glaciers sharing a sub-group land in the same .tar. """ import argparse import os import shutil import sys import tempfile def main(): p = argparse.ArgumentParser( description="Repackage prepro gdir(s) into a single small OGGM tar.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument("--prepro-base-url", required=True, help="Source prepro_base_url (as passed to " "init_glacier_directories).") p.add_argument("--target-dir", required=True, help="Destination root directory (e.g. /home/www/oggm/" "test_gdirs/). Everything in --prepro-base-url after " "'gdirs/' is reproduced underneath it, plus the " "RGI/b_/L/... suffix.") p.add_argument("--rgi-id", required=True, action="append", dest="rgi_ids", metavar="RGI_ID", help="Glacier ID, e.g. RGI60-11.00897. Repeatable.") p.add_argument("--level", required=True, type=int, help="Prepro level (e.g. 3).") p.add_argument("--border", required=True, type=int, help="Prepro border (e.g. 160).") p.add_argument("--rgi-version", default="62", help="RGI version for the URL.") p.add_argument("--workdir", default=None, help="Working dir (default: a temp dir, removed afterwards).") p.add_argument("--keep-workdir", action="store_true", help="Do not delete the working dir at the end.") args = p.parse_args() # Reproduce everything after the 'gdirs/' segment of the source url under # the target dir (e.g. .../gdirs/oggm_v1.6/.../W5E5/ -> target/oggm_v1.6/.../W5E5/). marker = "/gdirs/" if marker not in args.prepro_base_url: p.error("--prepro-base-url must contain '{}' (got: {})" .format(marker, args.prepro_base_url)) url_suffix = args.prepro_base_url.split(marker, 1)[1] target_base = os.path.join(args.target_dir, url_suffix) # Import here so --help works without a full OGGM env. from oggm import cfg, utils, workflow cfg.initialize(logging_level="WORKFLOW") cfg.PARAMS["rgi_version"] = args.rgi_version cfg.PARAMS["border"] = args.border cfg.PARAMS["use_multiprocessing"] = False own_workdir = args.workdir is None workdir = args.workdir or tempfile.mkdtemp(prefix="oggm_make_test_gdir_") cfg.PATHS["working_dir"] = workdir utils.mkdir(workdir) try: # 1. Fetch only the requested glacier(s). This downloads the big source # tar once and extracts just these gdirs into working_dir/per_glacier. gdirs = workflow.init_glacier_directories( args.rgi_ids, from_prepro_level=args.level, prepro_border=args.border, prepro_base_url=args.prepro_base_url, ) if not gdirs or any(gd is None for gd in gdirs): raise RuntimeError("init_glacier_directories returned no/empty gdir " "- check the ids / url / level / border.") # 2. Compress each gdir to .tar.gz (deletes the source dir), then # 3. bundle each sub-group dir into .tar (the OGGM layout). for gd in gdirs: utils.gdir_to_tar(gd, delete=True) utils.base_dir_to_tar(delete=True) # 4. Copy the resulting /.tar bundles to the target, # appending OGGM's canonical RGI/b_/L/ suffix so the # output is a valid prepro_base_url. per_glacier = os.path.join(workdir, "per_glacier") target_prepro = utils.get_prepro_base_url( base_url=target_base, rgi_version=args.rgi_version, border=args.border, prepro_level=args.level, ) written = [] for root, _, files in os.walk(per_glacier): for fn in files: if not fn.endswith(".tar"): continue src = os.path.join(root, fn) rel = os.path.relpath(src, per_glacier) # e.g. RGI60-11/RGI60-11.00.tar dest = os.path.join(target_prepro, rel) utils.mkdir(os.path.dirname(dest)) shutil.copy2(src, dest) written.append((dest, os.path.getsize(dest))) if not written: raise RuntimeError("No .tar bundle was produced - nothing to copy.") print("\nWrote {} bundle(s):".format(len(written))) for dest, size in written: print(" {} ({:.2f} MB)".format(dest, size / 1e6)) print("\nServe this dir as your prepro_base_url:\n {}".format(target_base)) finally: if own_workdir and not args.keep_workdir: shutil.rmtree(workdir, ignore_errors=True) elif args.keep_workdir: print("\nWorking dir kept at: {}".format(workdir)) if __name__ == "__main__": sys.exit(main())