CAMELS-IND

Overview

CAMELS-IND is the India hydrological dataset implementation. Indian CAMELS dataset with monsoon-influenced catchments.

Dataset Information

• Region: India
• Module: hydrodataset.camels_ind
• Class: CamelsInd

Features

Static Attributes

Static catchment attributes include: - Basin area - Mean precipitation - Topographic characteristics - Land cover information - Soil properties - Climate indices

Dynamic Variables

Timeseries variables available (varies by dataset): - Streamflow - Precipitation - Temperature (min, max, mean) - Potential evapotranspiration - Solar radiation - And more...

Usage

Basic Usage

from hydrodataset.camels_ind import CamelsInd
from hydrodataset import resolve_data_path

# Initialize dataset
data_path = resolve_data_path("camels_ind")
ds = CamelsInd(data_path)

# Get basin IDs
basin_ids = ds.read_object_ids()
print(f"Number of basins: {len(basin_ids)}")

# Check available features
print("Static features:", ds.available_static_features)
print("Dynamic features:", ds.available_dynamic_features)

# Read timeseries data
timeseries = ds.read_ts_xrdataset(
    gage_id_lst=basin_ids[:5],
    t_range=ds.default_t_range,
    var_lst=["streamflow", "precipitation"]
)
print(timeseries)

# Read attribute data
attributes = ds.read_attr_xrdataset(
    gage_id_lst=basin_ids[:5],
    var_lst=["area", "p_mean"]
)
print(attributes)

Reading Specific Variables

# Read with specific time range
ts_data = ds.read_ts_xrdataset(
    gage_id_lst=basin_ids[:10],
    t_range=["1990-01-01", "1995-12-31"],
    var_lst=["streamflow", "precipitation", "temperature_mean"]
)

# Read basin area
areas = ds.read_area(gage_id_lst=basin_ids[:10])

# Read mean precipitation
mean_precip = ds.read_mean_prcp(gage_id_lst=basin_ids[:10])

Data Sources

The dataset supports multiple data sources for certain variables. Check the class documentation for available sources and use tuple notation to specify:

# Request specific data source
ts_data = ds.read_ts_xrdataset(
    gage_id_lst=basin_ids[:5],
    t_range=["1990-01-01", "1995-12-31"],
    var_lst=[
        ("precipitation", "era5land"),  # Specify ERA5-Land source
        "streamflow"  # Use default source
    ]
)

API Reference

hydrodataset.camels_ind.CamelsInd

Bases: HydroDataset

Source code in hydrodataset/camels_ind.py

class CamelsInd(HydroDataset):

    _FORCING_COL_MAP = {
        "prcp(mm/day)":         "pcp_mm",
        "tmax(C)":              "airtemp_c_max",
        "tmin(C)":              "airtemp_c_min",
        "tavg(C)":              "airtemp_c_mean",
        "srad_lw(w/m2)":        "lwdownrad_wm2",
        "srad_sw(w/m2)":        "solrad_wm2",
        "wind_u(m/s)":          "windspeedu_mps",
        "wind_v(m/s)":          "windspeedv_mps",
        "wind(m/s)":            "windspeed_mps",
        "rel_hum(%)":           "rh_",
        "pet(mm/day)":          "pet_mm",
        "pet_gleam(mm/day)":    "pet_mm_gleam",
        "aet_gleam(mm/day)":    "aet_mm_gleam",
        "evap_canopy(mm/day)":  "evap_canopy",
        "evap_surface(mm/day)": "evap_surface",
        "sm_lvl1(kg/m2)":       "sm_lvl1",
        "sm_lvl2(kg/m2)":       "sm_lvl2",
        "sm_lvl3(kg/m2)":       "sm_lvl3",
        "sm_lvl4(kg/m2)":       "sm_lvl4",
    }
    _ATTR_FILES = [
        "camels_ind_topo.txt",
        "camels_ind_clim.txt",
        "camels_ind_geol.txt",
        "camels_ind_hydro.txt",
        "camels_ind_land.txt",
        "camels_ind_soil.txt",
        "camels_ind_anth.txt",
        "camels_ind_name.txt",
    ]
    _DATA_REL = "CAMELS_IND/CAMELS_IND_All_Catchments"
    """CAMELS_IND dataset class extending HydroDataset.

    This class provides access to the CAMELS_IND dataset, which contains
    hydrological and meteorological data for various watersheds in India.
    It uses a custom implementation to support the latest dataset version.

    The class relies on AquaFetch for data reading but overrides certain
    methods to support the new file structure in the latest Zenodo release.

    Attributes:
        region: Geographic region identifier
        download: Whether to download data automatically
        aqua_fetch: CustomCAMELS_IND instance for data access
    """

    def __init__(
        self, uri: str, region: Optional[str] = None, download: bool = False
    ) -> None:
        """Initialize CAMELS_IND dataset.

        Args:
            uri: Path to the data directory
            region: Geographic region identifier (optional)
            download: Whether to download data automatically (default: False)
        """
        super().__init__(uri)
        self.region = region
        self.download = download
        if str(uri).startswith("s3://"):
            return

        try:
            # Use custom class that supports the latest dataset version
            self.aqua_fetch = CustomCAMELS_IND(uri)
        except Exception as e:
            print(e)
            # If initialization fails, try to extract zip files
            check_zip_extract = False
            zip_files = [
                "CAMELS_IND_All_Catchments.zip",
                "CAMELS_IND_Catchments_Streamflow_Sufficient.zip",
            ]
            for filename in tqdm(zip_files, desc="Checking zip files"):
                extracted_dir = self.data_source_dir.joinpath(
                    "CAMELS_IND", filename[:-4]
                )
                if not extracted_dir.exists():
                    check_zip_extract = True
                    break
            if check_zip_extract:
                hydro_file.zip_extract(self.data_source_dir.joinpath("CAMELS_IND"))
            # Retry initialization after extraction
            self.aqua_fetch = CustomCAMELS_IND(uri)

    def read_object_ids(self) -> np.ndarray:
        uri = str(self.data_source_dir).rstrip("/")
        forcing_rel = f"{self._DATA_REL}/catchment_mean_forcings"
        if self._is_cloud():
            fs = self._make_s3fs()
            names = [p.split("/")[-1] for p in fs.ls(f"{uri}/{forcing_rel}".removeprefix("s3://"))]
        else:
            names = os.listdir(os.path.join(uri, *forcing_rel.split("/")))
        ids = sorted(n.replace(".csv", "") for n in names if n.endswith(".csv"))
        return np.array(ids)

    def cache_attributes_to_zarr(self) -> None:
        import zarr
        fs = self._make_s3fs()
        uri = str(self.data_source_dir).rstrip("/")
        attr_base = f"{uri}/{self._DATA_REL}/attributes_txt"
        dfs = []
        for fname in self._ATTR_FILES:
            path = f"{attr_base}/{fname}".removeprefix("s3://")
            try:
                with fs.open(path) as fh:
                    df = pd.read_csv(fh, sep=";", index_col="gauge_id",
                                     dtype={"gauge_id": str})
                df.index = df.index.astype(str)
                dfs.append(df)
            except Exception as e:
                print(f"  WARN {fname}: {e}")
        static = pd.concat(dfs, axis=1)
        static = static.loc[~static.index.duplicated(keep="first")]
        stations = self.read_object_ids().tolist()
        static = static.reindex(stations)
        static.columns = self._clean_feature_names(list(static.columns))
        static = static.rename(columns={"cwc_area": "area_km2", "cwc_lat": "lat"})

        zarr_name = self._attributes_cache_filename.replace(".nc", ".zarr")
        out, opts = self._zarr_path_and_opts(zarr_name)
        n = len(stations)
        root = zarr.open_group(out, mode="w", storage_options=opts, zarr_format=2)
        for col in static.columns:
            vals = static[col].values.astype(str) if static[col].dtype == object else static[col].values
            arr = root.create_array(col, shape=(n,), chunks=(n,), dtype=vals.dtype)
            arr[:] = vals
            arr.attrs["_ARRAY_DIMENSIONS"] = ["basin"]
        basin_arr = root.create_array("basin", shape=(n,), chunks=(n,), dtype=str)
        basin_arr[:] = stations
        basin_arr.attrs["_ARRAY_DIMENSIONS"] = ["basin"]
        root.attrs["coordinates"] = "basin"
        self._write_zarr_units(root, "static")
        print(f"Attributes zarr written to: {out}")

    def cache_timeseries_to_zarr(self) -> None:
        import zarr
        fs = self._make_s3fs()
        uri = str(self.data_source_dir).rstrip("/")
        base = f"{uri}/{self._DATA_REL}"

        stations = self.read_object_ids().tolist()
        all_times = pd.date_range(self.default_t_range[0], self.default_t_range[1], freq="D")
        n, nt = len(stations), len(all_times)
        times_ns = all_times.asi8
        all_vars = list(self._FORCING_COL_MAP.values()) + ["q_cms_obs"]

        # Read wide streamflow once
        print("Reading streamflow_observed.csv...")
        q_df = None
        try:
            q_path = f"{base}/streamflow_timeseries/streamflow_observed.csv".removeprefix("s3://")
            with fs.open(q_path) as fh:
                q_raw = pd.read_csv(fh)
            q_raw.index = pd.to_datetime(
                {"year": q_raw["year"], "month": q_raw["month"], "day": q_raw["day"]}
            )
            q_raw = q_raw.drop(columns=["year", "month", "day"])
            q_raw.columns = q_raw.columns.astype(str)
            q_df = q_raw.reindex(all_times)
        except Exception as e:
            print(f"  WARN streamflow: {e}")

        data: dict[str, np.ndarray] = {vn: np.full((n, nt), np.nan) for vn in all_vars}
        print(f"Reading {n} station forcings from OSS...")
        for i, stn in enumerate(tqdm(stations, desc="CAMELS_IND zarr")):
            forcing_path = f"{base}/catchment_mean_forcings/{stn}.csv".removeprefix("s3://")
            try:
                with fs.open(forcing_path) as fh:
                    df = pd.read_csv(fh)
                df.index = pd.to_datetime(
                    {"year": df["year"], "month": df["month"], "day": df["day"]}
                )
                df = df.reindex(all_times)
                for raw_col, zarr_vn in self._FORCING_COL_MAP.items():
                    if raw_col in df.columns:
                        data[zarr_vn][i] = df[raw_col].values.astype(float)
            except Exception as e:
                print(f"  WARN {stn}: {e}")

            # Streamflow: column is int(stn) without leading zeros
            if q_df is not None:
                q_col = str(int(stn))
                if q_col in q_df.columns:
                    data["q_cms_obs"][i] = q_df[q_col].values.astype(float)

        zarr_name = self._timeseries_cache_filename.replace(".nc", ".zarr")
        out, opts = self._zarr_path_and_opts(zarr_name)
        root = zarr.open_group(out, mode="w", storage_options=opts, zarr_format=2)
        for vn in all_vars:
            arr = root.create_array(vn, shape=(n, nt), chunks=(min(n, 100), min(nt, 365)), dtype="float64")
            arr[:] = data[vn]
            arr.attrs["_ARRAY_DIMENSIONS"] = ["basin", "time"]

        time_arr = root.create_array("time", shape=(nt,), chunks=(min(nt, 365),), dtype="int64")
        time_arr[:] = times_ns
        time_arr.attrs["_ARRAY_DIMENSIONS"] = ["time"]
        time_arr.attrs["units"] = "nanoseconds since 1970-01-01"
        time_arr.attrs["calendar"] = "proleptic_gregorian"

        basin_arr = root.create_array("basin", shape=(n,), chunks=(n,), dtype=str)
        basin_arr[:] = stations
        basin_arr.attrs["_ARRAY_DIMENSIONS"] = ["basin"]

        root.attrs["coordinates"] = "basin time"
        self._write_zarr_units(root, "dynamic")
        print(f"Timeseries zarr written to: {out}")

    @property
    def _attributes_cache_filename(self):
        return "camels_ind_attributes.nc"

    @property
    def _timeseries_cache_filename(self):
        return "camels_ind_timeseries.nc"

    @property
    def default_t_range(self):
        return ["1980-01-01", "2020-12-31"]

    # get the information of features from dataset file"00_CAMELS_IND_Data_Description.pdf"
    _subclass_static_definitions = {
        "p_mean": {"specific_name": "p_mean", "unit": "mm/day"},
        "area": {"specific_name": "area_km2", "unit": "km^2"},
    }

    _dynamic_variable_mapping = {
        StandardVariable.STREAMFLOW: {
            "default_source": "obs",
            "sources": {"obs": {"specific_name": "q_cms_obs", "unit": "m^3/s"}},
        },
        StandardVariable.PRECIPITATION: {
            "default_source": "imd",
            "sources": {"imd": {"specific_name": "pcp_mm", "unit": "mm/day"}},
        },
        StandardVariable.TEMPERATURE_MAX: {
            "default_source": "imd",
            "sources": {"imd": {"specific_name": "airtemp_c_max", "unit": "°C"}},
        },
        StandardVariable.TEMPERATURE_MIN: {
            "default_source": "imd",
            "sources": {"imd": {"specific_name": "airtemp_c_min", "unit": "°C"}},
        },
        StandardVariable.TEMPERATURE_MEAN: {
            "default_source": "imd",
            "sources": {"imd": {"specific_name": "airtemp_c_mean", "unit": "°C"}},
        },
        StandardVariable.SOLAR_RADIATION: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "solrad_wm2", "unit": "W/m^2"}},
        },
        StandardVariable.LONGWAVE_SOLAR_RADIATION: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "lwdownrad_wm2", "unit": "W/m^2"}},
        },
        StandardVariable.WIND_SPEED: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "windspeed_mps", "unit": "m/s"}},
        },
        StandardVariable.V_WIND_SPEED: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "windspeedv_mps", "unit": "m/s"}},
        },
        StandardVariable.U_WIND_SPEED: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "windspeedu_mps", "unit": "m/s"}},
        },
        StandardVariable.RELATIVE_HUMIDITY: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "rh_", "unit": "%"}},
        },
        StandardVariable.POTENTIAL_EVAPOTRANSPIRATION: {
            "default_source": "default",
            "sources": {
                "default": {"specific_name": "pet_mm", "unit": "mm/day"},
                "gleam": {"specific_name": "pet_mm_gleam", "unit": "mm/day"},
            },
        },
        StandardVariable.EVAPOTRANSPIRATION: {
            "default_source": "gleam",
            "sources": {"gleam": {"specific_name": "aet_mm_gleam", "unit": "mm/day"}},
        },
        StandardVariable.EVAPORATION: {
            "default_source": "canopy",
            "sources": {
                "canopy": {"specific_name": "evap_canopy", "unit": "mm/day"},
                "surface": {"specific_name": "evap_surface", "unit": "mm/day"},
            },
        },
        StandardVariable.VOLUMETRIC_SOIL_WATER_LAYER1: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "sm_lvl1", "unit": "kg/m^2"}},
        },
        StandardVariable.VOLUMETRIC_SOIL_WATER_LAYER2: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "sm_lvl2", "unit": "kg/m^2"}},
        },
        StandardVariable.VOLUMETRIC_SOIL_WATER_LAYER3: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "sm_lvl3", "unit": "kg/m^2"}},
        },
        StandardVariable.VOLUMETRIC_SOIL_WATER_LAYER4: {
            "default_source": "imdaa",
            "sources": {"imdaa": {"specific_name": "sm_lvl4", "unit": "kg/m^2"}},
        },
    }

default_t_range property

__init__(uri, region=None, download=False)

Initialize CAMELS_IND dataset.

Parameters:

Name	Type	Description	Default
uri	str	Path to the data directory	required
region	Optional[str]	Geographic region identifier (optional)	None
download	bool	Whether to download data automatically (default: False)	False

Source code in hydrodataset/camels_ind.py

def __init__(
    self, uri: str, region: Optional[str] = None, download: bool = False
) -> None:
    """Initialize CAMELS_IND dataset.

    Args:
        uri: Path to the data directory
        region: Geographic region identifier (optional)
        download: Whether to download data automatically (default: False)
    """
    super().__init__(uri)
    self.region = region
    self.download = download
    if str(uri).startswith("s3://"):
        return

    try:
        # Use custom class that supports the latest dataset version
        self.aqua_fetch = CustomCAMELS_IND(uri)
    except Exception as e:
        print(e)
        # If initialization fails, try to extract zip files
        check_zip_extract = False
        zip_files = [
            "CAMELS_IND_All_Catchments.zip",
            "CAMELS_IND_Catchments_Streamflow_Sufficient.zip",
        ]
        for filename in tqdm(zip_files, desc="Checking zip files"):
            extracted_dir = self.data_source_dir.joinpath(
                "CAMELS_IND", filename[:-4]
            )
            if not extracted_dir.exists():
                check_zip_extract = True
                break
        if check_zip_extract:
            hydro_file.zip_extract(self.data_source_dir.joinpath("CAMELS_IND"))
        # Retry initialization after extraction
        self.aqua_fetch = CustomCAMELS_IND(uri)

read_object_ids()

Source code in hydrodataset/camels_ind.py

def read_object_ids(self) -> np.ndarray:
    uri = str(self.data_source_dir).rstrip("/")
    forcing_rel = f"{self._DATA_REL}/catchment_mean_forcings"
    if self._is_cloud():
        fs = self._make_s3fs()
        names = [p.split("/")[-1] for p in fs.ls(f"{uri}/{forcing_rel}".removeprefix("s3://"))]
    else:
        names = os.listdir(os.path.join(uri, *forcing_rel.split("/")))
    ids = sorted(n.replace(".csv", "") for n in names if n.endswith(".csv"))
    return np.array(ids)