readabs

15class ReadArgs(TypedDict):
16    """Type definition for ABS data reading arguments."""
17
18    verbose: NotRequired[bool]
19    ignore_errors: NotRequired[bool]
20    get_zip: NotRequired[bool]
21    get_excel_if_no_zip: NotRequired[bool]
22    get_excel: NotRequired[bool]
23    single_zip_only: NotRequired[str]
24    single_excel_only: NotRequired[str]
25    selected_excel: NotRequired[tuple[str, ...]]
26    history: NotRequired[str]
27    cache_only: NotRequired[bool]
28    keep_non_ts: NotRequired[bool]
29    zip_file: NotRequired[str]

 24@cache
 25def abs_catalogue(*, cache_only: bool = False, verbose: bool = False) -> DataFrame:
 26    """Return a DataFrame of ABS Catalogue numbers.
 27
 28    Downloads catalogue data from the ABS website on first call and caches
 29    for future use. The returned DataFrame contains catalogue numbers with
 30    their topics, themes, URLs, and status.
 31
 32    Parameters
 33    ----------
 34    cache_only : bool, default False
 35        If True, only use cached data and don't attempt to download.
 36    verbose : bool, default False
 37        If True, print progress messages.
 38
 39    Returns
 40    -------
 41    DataFrame
 42        DataFrame with columns ['Theme', 'Parent Topic', 'Topic', 'URL', 'Status']
 43        and index of catalogue IDs.
 44
 45    Raises
 46    ------
 47    CatalogueError
 48        If the catalogue data cannot be retrieved or parsed.
 49    HttpError
 50        If there's a network error downloading the catalogue.
 51    CacheError
 52        If cache_only=True but no cached data is available.
 53
 54    Example
 55    -------
 56    >>> import readabs as ra
 57    >>> catalogue = ra.abs_catalogue()
 58    >>> print(catalogue.head())
 59
 60    """
 61    try:
 62        # Download ABS catalogue page
 63        abs_bytes = get_file(ABS_CATALOGUE_URL, cache_only=cache_only, verbose=verbose)
 64
 65        if not abs_bytes:
 66            raise CatalogueError("No data retrieved from ABS catalogue URL")
 67
 68        # Parse HTML content
 69        try:
 70            html_content = abs_bytes.decode(DEFAULT_ENCODING, errors="replace")
 71        except UnicodeDecodeError as e:
 72            raise CatalogueError(f"Failed to decode HTML content: {e}") from e
 73
 74        # Extract tables from HTML
 75        try:
 76            tables = read_html(StringIO(html_content), extract_links="body")
 77            if not tables:
 78                raise CatalogueError("No tables found in HTML content")
 79            links = tables[-1]  # Get the last table
 80        except (ValueError, IndexError) as e:
 81            raise CatalogueError(f"Failed to parse HTML tables: {e}") from e
 82
 83        # Validate required columns exist
 84        required_cols = ["Catalogue number", "Topic"]
 85        missing_cols = [col for col in required_cols if col not in links.columns]
 86        if missing_cols:
 87            raise CatalogueError(f"Missing required columns: {missing_cols}")
 88
 89        # Extract catalogue numbers and URLs
 90        try:
 91            cats = links["Catalogue number"].apply(Series)[0]
 92            urls = links["Topic"].apply(Series)[1]
 93        except (KeyError, IndexError) as e:
 94            raise CatalogueError(f"Failed to extract catalogue data: {e}") from e
 95
 96        # Process topic URLs to create hierarchical structure
 97        url_snippets = _process_topic_urls(urls)
 98
 99        # Create main DataFrame with hierarchical topic structure
100        frame = _create_topic_frame(url_snippets)
101        frame["URL"] = urls
102
103        # Align catalogue numbers with processed frame
104        cats = cats[frame.index]
105
106        # Process catalogue status (active vs ceased)
107        cat_index, status = _process_catalogue_status(cats)
108
109        frame["Status"] = status
110        frame.index = Index(cat_index)
111        frame.index.name = CATALOGUE_INDEX_NAME
112
113    except (HttpError, CacheError, ValueError) as e:
114        raise CatalogueError(f"Error retrieving ABS catalogue: {e}") from e
115
116    return frame

 95def annualise_percentages(data: DataT, *, periods_per_year: float) -> DataT:
 96    """Annualise a growth rate (expressed as a percentage) for a period.
 97
 98    Args:
 99        data : pandas Series or DataFrame - The growth rate (expressed as a
100            percentage) to annualise. Note a growth percentage of 5% is a growth
101            rate of 0.05.
102        periods_per_year : int or float, default 12 - The number of periods in a
103            year. For monthly data, this is 12.
104
105    Returns:
106        pandas Series or DataFrame - The annualised growth expressed as a percentage.
107            For DataFrame input, the annualised growth rate is calculated for each column.
108
109    Raises:
110        InvalidParameterError - If periods_per_year is not positive.
111        InvalidDataError - If data is not a Series or DataFrame.
112
113    """
114    if not isinstance(data, (Series, DataFrame)):
115        raise InvalidDataError("data must be a pandas Series or DataFrame")
116
117    if not isinstance(periods_per_year, (int, float)) or periods_per_year <= 0:
118        raise InvalidParameterError("periods_per_year must be a positive number")
119
120    try:
121        rates = data / 100.0
122        return annualise_rates(rates, periods_per_year=periods_per_year)
123    except Exception as e:
124        raise InvalidDataError(f"Error annualising percentages: {e}") from e

62def annualise_rates(data: DataT, *, periods_per_year: float) -> DataT:
63    """Annualise a growth rate for a period.
64
65    Note: returns a percentage value (and not a rate)!
66
67    Args:
68        data : pandas Series or DataFrame - The growth rate to annualise.
69            Note a growth rate of 0.05 is 5%.
70        periods_per_year : int or float, default 12 - The number of periods in a year.
71            For monthly data, this is 12.
72
73    Returns:
74        pandas Series or DataFrame - The annualised growth expressed as a percentage
75            (not a rate). For DataFrame input, the annualised growth rate is
76            calculated for each column.
77
78    Raises:
79        InvalidParameterError - If periods_per_year is not positive.
80    InvalidDataError - If data is not a Series or DataFrame.
81
82    """
83    if not isinstance(data, (Series, DataFrame)):
84        raise InvalidDataError("data must be a pandas Series or DataFrame")
85
86    if not isinstance(periods_per_year, (int, float)) or periods_per_year <= 0:
87        raise InvalidParameterError("periods_per_year must be a positive number")
88
89    try:
90        return (((1 + data) ** periods_per_year) - 1) * 100
91    except Exception as e:
92        raise InvalidDataError(f"Error annualising rates: {e}") from e

126def find_abs_id(
127    meta: DataFrame,
128    search_terms: dict[str, str],
129    **kwargs: Any,
130) -> tuple[str, str, str]:  # table, series_id, units
131    """Find a unique ABS series identifier in the ABS metadata.
132
133    Parameters
134    ----------
135    meta : DataFrame
136        A pandas DataFrame of metadata from the ABS
137        (via read_abs_cat() or read_abs_series()).
138    search_terms : dict[str, str]
139        A dictionary {search_phrase: meta_column_name, ...} of search terms.
140        Note: the search terms must be unique, as a dictionary cannot hold the
141        same search term to be applied to different columns.
142    **kwargs : Any
143        Additional keyword arguments. The only additional keyword argument
144        that is used is validate_unique.
145    validate_unique : bool = True
146        Raise a ValueError if the search result is not a single
147        unique match. Note: the default is True for safety.
148
149    Returns
150    -------
151    tuple[str, str, str]
152        A tuple of the table, series_id and units for the unique
153        series_id that matches the search terms.
154
155    Metacol
156    -------
157    Because the meta data is a DataFrame, the columns can be referenced by either
158    their full textual name, or by the short name defined in the metacol object.
159    For example, if metacol is imported as mc, to refer to the
160    `Data Item Description` column, the user can refer to it as mc.did.
161
162    Example
163    -------
164    ```python
165    from readabs import metacol as mc  # alias for the ABS meta data column names
166    from readabs import read_abs_cat, find_abs_id, recalibrate
167    cat_num = "6202.0"  # The ABS labour force survey
168    data, meta = read_abs_cat(cat_num)
169    search_terms = {
170        "Employed total ;  Persons ;": mc.did,
171        "Seasonally Adjusted": mc.stype,
172        "62020001": mc.table,
173    }
174    table, series_id, units = find_abs_id(meta, search_terms)
175    print(f"Table: {table} Series ID: {series_id} Units: {units}")
176    recal_series, recal_units = recalibrate(data[table][series_id], units)
177    ```
178
179    """
180    validate_unique = kwargs.pop("validate_unique", True)
181    found = search_abs_meta(meta, search_terms, validate_unique=validate_unique, **kwargs).iloc[0]
182    table, series_id, units = (
183        found[mc.table],
184        found[mc.id],
185        found[mc.unit],
186    )
187
188    return table, series_id, units

37@cache  # minimise slowness with repeat business
38def grab_abs_url(
39    cat: str = "",
40    url: str = "",
41    **kwargs: Unpack[ReadArgs],
42) -> dict[str, DataFrame]:
43    """For a given URL, extract the data from the Excel and ZIP file links found on that page.
44
45    The data is returned as a dictionary of DataFrames. The Excel files are converted
46    into DataFrames, with each sheet in each Excel file becoming a separate DataFrame.
47    ZIP files are examined for Excel files, which are similarly converted into
48    DataFrames. The dictionary of DataFrames is returned.
49
50    The preferred mechanism for reading ABS data is to use the `read_abs_cat()`
51    or `read_abs_series()` functions. This function is provided for those
52    cases where the data is not available in the ABS catalogue, where the
53    data is not a timeseries, or where the user wants to extract data from
54    a specific ABS landingpage.
55
56
57    Parameters
58    ----------
59    url : str = ""
60        A URL for an ABS Catalogue landing page. Either a url or
61        a catalogue number must be provided. If both are provided, the
62        URL will be used.
63
64    cat : str = ""
65        An ABS Catalogue number. If provided, and the URL is not
66        provided, then the Catalogue number will be used to get the URL.
67
68    **kwargs : Unpack[ReadArgs]
69        Accepts the same keyword arguments as `read_abs_cat()`.
70
71    Returns
72    -------
73    dict[str, DataFrame]
74        A dictionary of DataFrames.
75
76    """
77    # check/get the keyword arguments
78    url = _get_url(url, cat)
79    check_kwargs(kwargs, "grab_abs_url")  # warn if invalid kwargs
80    args = get_args(kwargs, "grab_abs_url")  # get the valid kwargs
81    if verbose := args["verbose"]:
82        print(f"grab_abs_url(): {url=}, {args=}")
83
84    # get the URL links to the relevant ABS data files on that webpage
85    links = get_abs_links(url, **args)
86    if not links:
87        print(f"No data files found at URL: {url}")
88        return {}  # return an empty Dictionary
89
90    # read the data files into a dictionary of DataFrames
91    abs_dict: dict[str, DataFrame] = {}
92
93    # Process single file requests first
94    abs_dict = _process_single_files(abs_dict, links, args, verbose=verbose)
95    if abs_dict:  # If single file was found and processed, return it
96        return abs_dict
97
98    # Process all files based on configuration
99    return _process_all_files(abs_dict, links, args)

102def grab_abs_zip(
103    zip_path: Path | str,
104    **kwargs: Unpack[ReadArgs]
105) -> dict[str, DataFrame]:
106    """Grab and process a single ABS ZIP file from a file system location.
107
108    This is a convenience function that opens an ABS ZIP file from a local
109    filesystem path. Expect to be used rarely.
110
111    Parameters
112    ----------
113    zip_path : Path | str
114        The local filesystem path of the ABS ZIP file to open and process.
115
116    **kwargs : Unpack[ReadArgs]
117        Additional keyword arguments for file retrieval and processing.
118
119    Returns
120    -------
121    dict[str, DataFrame]
122        A dictionary of DataFrames extracted from the ZIP file.
123
124    """
125    check_kwargs(kwargs, "grab_abs_zip")  # warn if invalid kwargs
126    args = get_args(kwargs, "grab_abs_zip")  # get the valid kwargs
127
128    zp: Path = zip_path if isinstance(zip_path, Path) else Path(zip_path)
129    zip_bytes = zp.read_bytes()
130    abs_dict: dict[str, DataFrame] = {}
131    return _process_zip(abs_dict, zip_bytes, **args)

194def monthly_to_qtly(data: DataT, q_ending: str = "DEC", f: str = "mean") -> DataT:
195    """Convert monthly data to quarterly data.
196
197    This is done by taking the mean (or sum) of the three months in each quarter.
198    Ignore quarters with less than or more than three months data. Drop NA items.
199    Change f to "sum" for a quarterly sum.
200
201    Args:
202        data : pandas Series or DataFrame
203            The data to convert to quarterly frequency.
204        q_ending : str, default "DEC"
205            The month in which the quarter ends. For example, "DEC" for December.
206        f : str, default "mean"
207            The function to apply to the three months in each quarter.
208            Change to "sum" for a quarterly sum. The default is a
209            quarterly mean.
210
211    Returns:
212        pandas Series or DataFrame
213            The data with a quarterly PeriodIndex. If a quarter has less than
214            three months data, the quarter is dropped. If the quarter has more
215            than three months data, the quarter is dropped. Any NA data is removed.
216        For DataFrame input, the function is applied to each column.
217
218    Raises:
219        InvalidDataError - If data is not a Series or DataFrame.
220        InvalidParameterError - If q_ending or f parameters are invalid.
221
222    """
223    # Validate inputs
224    if not isinstance(data, (Series, DataFrame)):
225        raise InvalidDataError("data must be a pandas Series or DataFrame")
226
227    valid_endings = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"]
228    if q_ending.upper() not in valid_endings:
229        raise InvalidParameterError(f"q_ending must be one of {valid_endings}")
230
231    valid_aggregations = ["mean", "sum", "min", "max", "std", "var"]
232    if f not in valid_aggregations:
233        raise InvalidParameterError(f"f must be one of {valid_aggregations}")
234
235    try:
236        if isinstance(data, Series):
237            return _monthly_to_qtly_series(data, q_ending, f)
238        if isinstance(data, DataFrame):
239            result_dict = {}
240            for col in data.columns:
241                result_dict[col] = _monthly_to_qtly_series(data[col], q_ending, f)
242            return data.__class__(result_dict)
243        # This should never be reached due to validation above
244        raise InvalidDataError("Unexpected data type")  # noqa: TRY301
245    except Exception as e:
246        raise InvalidDataError(f"Error converting monthly to quarterly data: {e}") from e

31def percent_change(data: DataT, n_periods: int) -> DataT:
32    """Calculate a percentage change in a contiguous, ordered series over n_periods.
33
34    Args:
35        data : pandas Series or DataFrame
36            The data to calculate the percentage change for.
37        n_periods : int
38            The number of periods to calculate the percentage change over.
39            Typically 4 for quarterly data, and 12 for monthly data.
40
41    Returns:
42        pandas Series or DataFrame - The percentage change in the data over n_periods.
43            For DataFrame input, the percentage change is calculated for each column.
44
45    Raises:
46        InvalidParameterError - If n_periods is not a positive integer.
47        InvalidDataError - If data is not a Series or DataFrame.
48
49    """
50    if not isinstance(n_periods, int) or n_periods <= 0:
51        raise InvalidParameterError("n_periods must be a positive integer")
52
53    if not isinstance(data, (Series, DataFrame)):
54        raise InvalidDataError("data must be a pandas Series or DataFrame")
55
56    try:
57        return (data / data.shift(n_periods) - 1) * 100
58    except Exception as e:
59        raise InvalidDataError(f"Error calculating percentage change: {e}") from e

11def print_abs_catalogue(*, cache_only: bool = False, verbose: bool = False) -> None:
12    """Print a table of ABS Catalogue Numbers with their metadata.
13
14    Displays catalogue numbers that contain time-series data along with
15    their theme, parent topic, topic, and status information. The URL
16    column is excluded from the display for readability.
17
18    This is primarily a convenience function to help users identify the
19    correct catalogue number for data retrieval functions.
20
21    Parameters
22    ----------
23    cache_only : bool, default False
24        If True, only use cached catalogue data.
25    verbose : bool, default False
26        If True, print progress messages during catalogue retrieval.
27
28    Raises
29    ------
30    CatalogueError
31        If the catalogue data cannot be retrieved or processed.
32    HttpError
33        If there's a network error downloading the catalogue.
34    CacheError
35        If cache_only=True but no cached data is available.
36
37    Example
38    -------
39    >>> import readabs as ra
40    >>> ra.print_abs_catalogue()
41
42    """
43    try:
44        # Retrieve the catalogue data
45        catalogue = abs_catalogue(cache_only=cache_only, verbose=verbose)
46
47        # Validate catalogue is not empty
48        if catalogue.empty:
49            print("No catalogue data available.")
50            return
51
52        # Select columns for display (exclude URL for readability)
53        available_columns = [col for col in DISPLAY_COLUMNS if col in catalogue.columns]
54        if not available_columns:
55            print("Catalogue data does not contain expected columns.")
56            return
57
58        display_data = catalogue[available_columns]
59
60        # Generate and print markdown table
61        try:
62            markdown_output = display_data.to_markdown()
63            print(markdown_output)
64        except Exception:  # noqa: BLE001
65            print(display_data.to_string())
66
67    except (CatalogueError, HttpError, CacheError) as e:
68        print(f"Error retrieving catalogue: {e}")

47def print_rba_catalogue(*, cache_only: bool = False, verbose: bool = False) -> None:
48    """Print to standard output a table of the RBA Catalogue Numbers.
49
50    This function prints a formatted table of RBA catalogue numbers
51    to standard output.
52
53    Parameters
54    ----------
55    cache_only : bool = False
56        If True, only use the cache.
57    verbose : bool = False
58        If True, print progress messages.
59
60    Returns
61    -------
62    None
63        This function does not return anything.
64
65    Example
66    -------
67
68    ```python
69    import readabs as ra
70    ra.print_rba_catalogue()
71    ```
72
73    """
74    rba_catalog = rba_catalogue(cache_only=cache_only, verbose=verbose)
75    print(rba_catalog.loc[:, rba_catalog.columns != "URL"].to_markdown())

127def qtly_to_monthly(
128    data: DataT,
129    *,
130    interpolate: bool = True,
131    limit: int | None = 2,  # only used if interpolate is True
132    dropna: bool = True,
133) -> DataT:
134    """Convert data from Quarterly PeriodIndex to a Monthly PeriodIndex.
135
136    Args:
137        data: Series or DataFrame with quarterly PeriodIndex. Assumes the index is unique.
138            The data to convert to monthly frequency.
139        interpolate: bool, default True
140            Whether to interpolate the missing monthly data.
141        limit: int, default 2 - The maximum number of consecutive missing months
142            to interpolate.
143        dropna: bool, default True - Whether to drop NA data
144
145    Returns:
146        pandas Series or DataFrame - The data with a Monthly PeriodIndex.
147            If interpolate is True, the missing monthly data is interpolated.
148            If dropna is True, any NA data is removed.
149
150    Raises:
151        InvalidDataError - If data index is not a quarterly PeriodIndex or has issues.
152        InvalidParameterError - If limit parameter is invalid.
153
154    """
155    # Validate input data
156    if not isinstance(data, (Series, DataFrame)):
157        raise InvalidDataError("data must be a pandas Series or DataFrame")
158
159    if not isinstance(data.index, PeriodIndex):
160        raise InvalidDataError("data index must be a PeriodIndex")
161
162    if not (data.index.freqstr and data.index.freqstr[0] == "Q"):
163        raise InvalidDataError("data index must have quarterly frequency")
164
165    if not data.index.is_unique:
166        raise InvalidDataError("data index must be unique")
167
168    if not data.index.is_monotonic_increasing:
169        raise InvalidDataError("data index must be monotonic increasing")
170
171    if limit is not None and (not isinstance(limit, int) or limit < 0):
172        raise InvalidParameterError("limit must be a non-negative integer or None")
173
174    # do the heavy lifting
175    try:
176        data = (
177            data.set_axis(labels=data.index.to_timestamp(how="end"), axis="index")
178            .resample(rule="ME")  # adds in every missing month
179            .first(min_count=1)  # generates nans for new months
180            # assumes only one value per quarter (ie. unique index)
181            .pipe(_set_axis_monthly_periods)
182        )
183    except Exception as e:
184        raise InvalidDataError(f"Error in quarterly to monthly conversion: {e}") from e
185
186    if interpolate:
187        data = data.interpolate(limit_area="inside", limit=limit)
188    if dropna:
189        data = data.dropna()
190
191    return data

17@cache
18def rba_catalogue(*, cache_only: bool = False, verbose: bool = False) -> DataFrame:
19    """Return a DataFrame of RBA Catalogue numbers.
20
21    In the first instance, this is downloaded from the RBA website, and
22    cached for future use.
23
24    Parameters
25    ----------
26    cache_only : bool = False
27        If True, only use the cache.
28    verbose : bool = False
29        If True, print progress messages.
30
31    Returns
32    -------
33    DataFrame
34        A DataFrame of RBA Catalogue numbers.
35
36    Example
37    -------
38    ```python
39    import readabs as ra
40    catalogue = ra.rba_catalogue()
41    ```
42
43    """
44    return _get_rba_links(cache_only=cache_only, verbose=verbose)

144def read_abs_by_desc(
145    wanted: list[str] | dict[str, str] | dict[str, dict[str, Any]],
146    **kwargs: Any,
147) -> tuple[dict[str, pd.Series], pd.DataFrame]:
148    """Get specific ABS data series by searching the ABS meta data.
149
150    Parameters
151    ----------
152    wanted : list of str, dict of str:str, or dict of str:dict
153        The data
154        item descriptions to search for. If a list, it will be a list of
155        descriptions to search for. If a dictionary, the keys will a name.
156        The dictionary values can be either a string (the data item
157        description to search for) or a dictionary of keyword arguments, one of
158        which would be the data item description to search for.
159    **kwargs : Any
160        Keyword arguments to control the data retrieval.
161        The keyword arguments can include the following:
162        - abs_dict : dict - the dictionary of ABS data to search (from
163            read_abs_cat()).
164        - abs_meta : DataFrame - the metadata for the ABS data (from
165            read_abs_cat()).
166        - for the retrieval of data, the "cat" argument must be present.
167            The following arguments, if present, will also be used (ie.
168            passed to read_abs_cat()): ["ignore_errors", "get_zip",
169            "get_excel_if_no_zip", "get_excel", "cache_only",
170            "single_excel_only", "selected_excel", "single_zip_only",
171            "verbose"].
172        - for the selection of data, the following metacol names, if present,
173            will be used to construct the selector: "cat", "did"
174            "stype", "id", "start", "end", "num", "unit", "dtype", "freq",
175            "cmonth", "table", "tdesc".
176        - finally, the following arguments will be passed to the find_abs_id()
177            and search_abs_meta() functions: ["validate_unique", "exact_match",
178            "regex", "verbose"].
179
180    Notes
181    -----
182    - if "wanted" is of type list[str] or dict[str, str], the kwargs should
183        include sufficient keys from the metacol dataclass to get the data.
184        Typically, the "cat" key, the "table" key, and the "stype" key would
185        be required. The did key would taken from the wanted list or
186        dictionary.
187    if wanted is of type dict[str, dict[str, Any]], the inner dictionary
188        must contain a "did" key. The other keys that can be used for the
189        data retrieval are the same as the metacol dataclass fileds, namely:
190        "cat", "stype", "id", "start", "end", "num", "unit", "dtype", "freq",
191        "cmonth", "table", "tdesc".
192    - if abs_dict and abs_meta are provided within the kwargs, they will be
193        used to locate and extract the selected data.
194    - if abs_dict and abs_meta are not provided, then, (1) wanted must be of
195        type dict[str, dict[str, Any]] and (2) the inner dictionary must
196        contain a "cat" key so the data can be retrieved. Other keys that
197        can be used for the data retrieval are the same as for read_abs_cat(),
198        namely ["ignore_errors", "get_zip", "get_excel_if_no_zip",
199        "get_excel", "single_excel_only", "selected_excel",
200        "single_zip_only", "cache_only"].
201
202
203    Returns
204    -------
205    Returns a tuple of two items:
206    - A dictionary of pandas Series objects, where the keys are the series
207      descriptions. The series.name attribute will be the ABS series-id.
208    - A pandas DataFrame containing the metadata for the series.
209
210    Example
211    -------
212
213    ```python
214    import readabs as ra
215    from pandas import DataFrame
216    cat_num = "5206.0"  # The ABS National Accounts
217    data, meta = ra.read_abs_cat(cat=cat_num)
218    wanted = ["Gross domestic product: Chain volume measures ;",]
219    selected, selected_meta = ra.read_abs_by_desc(
220        wanted=wanted, abs_dict=data, abs_meta=meta, table="5206001_Key_Aggregates"
221    )
222    ```
223
224    """
225    # - preparation
226    if not _work_to_do(wanted):
227        return {}, pd.DataFrame()
228    if isinstance(wanted, list):
229        wanted = _wlist_to_wdict(wanted)
230    abs_dict = kwargs.get("abs_dict", {})
231    abs_meta = kwargs.get("abs_meta", pd.DataFrame())
232    kwarg_selector = _get_search_terms(kwargs, {})
233    search_args = _get_search_args(kwargs, {})
234
235    return_dict = {}
236    return_meta = pd.DataFrame()
237    for key, value in wanted.items():
238        item_selector = kwarg_selector.copy()
239        item_search_args = search_args.copy()
240        if isinstance(value, str):
241            series, meta = _get_item_from_str(
242                item=value,
243                data_dict=abs_dict,
244                data_meta=abs_meta,
245                item_selector=item_selector,
246                search_args=item_search_args,
247            )
248
249        elif isinstance(value, dict):
250            series, meta = _get_item_from_dict(
251                item_dict=value,
252                data_dict=abs_dict,
253                data_meta=abs_meta,
254                item_selector=item_selector,
255                search_args=item_search_args,
256                **kwargs,
257            )
258        else:
259            raise TypeError(
260                "Each value in the wanted list/dictionary must be either a string " + "or a dictionary."
261            )
262
263        # save search results
264        return_dict[key] = series
265        return_meta = pd.concat([return_meta, meta])
266
267    return return_dict, return_meta

 28@cache  # minimise slowness for any repeat business
 29def read_abs_cat(
 30    cat: str,
 31    url: str = "",
 32    **kwargs: Unpack[ReadArgs],
 33) -> tuple[dict[str, DataFrame], DataFrame]:
 34    """For a specific catalogue identifier, return the complete ABS Catalogue information as DataFrames.
 35
 36    This function returns the complete ABS Catalogue information as a
 37    python dictionary of pandas DataFrames, as well as the associated metadata
 38    in a separate DataFrame. The function automates the collection of zip and
 39    excel files from the ABS website. If necessary, these files are downloaded,
 40    and saved into a cache directory. The files are then parsed to extract time
 41    series data, and the associated metadata.
 42
 43    By default, the cache directory is `./.readabs_cache/`. You can change the
 44    default directory name by setting the shell environment variable
 45    `READABS_CACHE_DIR` with the name of the preferred directory.
 46
 47    Parameters
 48    ----------
 49    cat : str
 50        The ABS Catalogue Number for the data to be downloaded and made
 51        available by this function. This argument must be specified in the
 52        function call.
 53
 54    **kwargs : Unpack[ReadArgs]
 55        The following parameters may be passed as optional keyword arguments.
 56
 57    url : str = ""
 58        The URL of an ABS landing page. Use this for discontinued series
 59        that are no longer in the ABS Time Series Directory. If provided,
 60        data will be retrieved from this URL instead of looking up the
 61        catalogue number. Example:
 62        `read_abs_cat(cat="8501.0", url="https://www.abs.gov.au/.../jun-2025")`
 63
 64    keep_non_ts : bool = False
 65        A flag for whether to keep the non-time-series tables
 66        that might form part of an ABS catalogue item. Normally, the
 67        non-time-series information is ignored, and not made available to
 68        the user.
 69
 70    history : str = ""
 71        Provide a month-year string to extract historical ABS data.
 72        For example, you can set history="dec-2023" to the get the ABS data
 73        for a catalogue identifier that was originally published in respect
 74        of Q4 of 2023. Note: not all ABS data sources are structured so that
 75        this technique works in every case; but most are.
 76
 77    verbose : bool = False
 78        Setting this to true may help diagnose why something
 79        might be going wrong with the data retrieval process.
 80
 81    ignore_errors : bool = False
 82        Normally, this function will cease downloading when
 83        an error in encountered. However, sometimes the ABS website has
 84        malformed links, and changing this setting is necessitated. (Note:
 85        if you drop a message to the ABS, they will usually fix broken
 86        links with a business day).
 87
 88    get_zip : bool = True
 89        Download the excel files in .zip files.
 90
 91    get_excel_if_no_zip : bool = True
 92        Only try to download .xlsx files if there are no zip
 93        files available to be downloaded. Only downloading individual excel
 94        files when there are no zip files to download can speed up the
 95        download process.
 96
 97    get_excel : bool = False
 98        The default value means that excel files are not
 99        automatically download. Note: at least one of `get_zip`,
100        `get_excel_if_no_zip`, or `get_excel` must be true. For most ABS
101        catalogue items, it is sufficient to just download the one zip
102        file. But note, some catalogue items do not have a zip file.
103        Others have quite a number of zip files.
104
105    single_excel_only : str = ""
106        If this argument is set to a table name (without the
107        .xlsx extension), only that excel file will be downloaded. If
108        set, and only a limited subset of available data is needed,
109        this can speed up download times significantly. Note: overrides
110        `get_zip`, `get_excel_if_no_zip`, `get_excel` and `single_zip_only`.
111
112    selected_excel : tuple[str, ...] = ()
113        If set to a tuple of table names (without the .xlsx extension),
114        only those excel files will be downloaded. Useful when several
115        specific tables are needed and downloading the full zip would
116        be wasteful. Example:
117        `selected_excel=("62020001", "62020017", "62020X28")`.
118        Must be a tuple (not a list) because `read_abs_cat` uses an
119        internal cache that requires hashable arguments. Note: overrides
120        `get_zip`, `get_excel_if_no_zip`, `get_excel` and `single_zip_only`
121        when at least one matching file is found.
122
123    single_zip_only : str = ""
124        If this argument is set to a zip file name (without
125        the .zip extension), only that zip file will be downloaded.
126        If set, and only a limited subset of available data is needed,
127        this can speed up download times significantly. Note: overrides
128        `get_zip`, `get_excel_if_no_zip`, and `get_excel`.
129
130    cache_only : bool = False
131        If set to True, this function will only access
132        data that has been previously cached. Normally, the function
133        checks the date of the cache data against the date of the data
134        on the ABS website, before deciding whether the ABS has fresher
135        data that needs to be downloaded to the cache.
136
137    zip_file: str | Path = ""
138        If set to a specific zip file name (with or without the .zip
139        extension), this function will only extract data from that zip file
140        on the local file system. This may be useful for debugging purposes.
141
142    Returns
143    -------
144    tuple[dict[str, DataFrame], DataFrame]
145        The function returns a tuple of two items. The first item is a
146        python dictionary of pandas DataFrames (which is the primary data
147        associated with the ABS catalogue item). The second item is a
148        DataFrame of ABS metadata for the ABS collection.
149
150        Note:
151        You can retrieve non-timeseries data using the grab_abs_url()
152        function. That takes the URL for the ABS landing page for the ABS
153        collection you are interested in. The read_abs_cat function is for
154        ABS catalogue identifiers which are timeseries data, for which the
155        metadata can be extracted.
156
157    Example
158    -------
159
160    ```python
161    import readabs as ra
162    from pandas import DataFrame
163    cat_num = "6202.0"  # The ABS labour force survey
164    data: tuple[dict[str, DataFrame], DataFrame] = ra.read_abs_cat(cat=cat_num)
165    abs_dict, meta = data
166    ```
167
168    """
169    # --- get the time series data ---
170    zip_file = kwargs.get("zip_file")
171    raw_abs_dict = (
172        grab_abs_zip(zip_file, **kwargs) if zip_file else grab_abs_url(cat=cat, url=url, **kwargs)
173    )
174    response = _get_time_series_data(cat, raw_abs_dict, **kwargs)
175
176    if not response:
177        response = {}, DataFrame()
178
179    return response  # dictionary of DataFrames, and a DataFrame of metadata

 15def read_abs_series(
 16    cat: str,
 17    series_id: str | Sequence[str],
 18    url: str = "",
 19    **kwargs: Unpack[ReadArgs],
 20) -> tuple[DataFrame, DataFrame]:
 21    """Get specific ABS data series by their ABS catalogue and series identifiers.
 22
 23    Parameters
 24    ----------
 25    cat : str
 26        The ABS catalogue ID.
 27
 28    series_id : str | Sequence[str]
 29        An ABS series ID or a sequence of ABS series IDs.
 30
 31    url : str = ""
 32        The URL of an ABS landing page. Use this for discontinued series
 33        that are no longer in the ABS Time Series Directory. If provided,
 34        data is retrieved from this URL instead of looking up the catalogue
 35        number. Passed through to read_abs_cat().
 36
 37    **kwargs : Any
 38        Keyword arguments for the read_abs_series function,
 39        which are the same as the keyword arguments for the
 40        read_abs_cat function.
 41
 42    Returns
 43    -------
 44    tuple[DataFrame, DataFrame]
 45        A tuple of two DataFrames, one for the primary data and one for the metadata.
 46
 47    Example
 48    -------
 49
 50    ```python
 51    import readabs as ra
 52    from pandas import DataFrame
 53    cat_num = "6202.0"  # The ABS labour force survey
 54    unemployment_rate = "A84423050A"
 55    seo = "6202001"  # The ABS table name
 56    data, meta = ra.read_abs_series(
 57        cat=cat_num, series_id=unemployment_rate, single_excel_only=seo
 58    )
 59    ```
 60
 61    """
 62    # check for unexpected keyword arguments/get defaults
 63    check_kwargs(kwargs, "read_abs_series")
 64    args = get_args(kwargs, "read_abs_series")
 65
 66    # read the ABS category data
 67    cat_data, cat_meta = read_abs_cat(cat, url=url, **args)
 68
 69    # drop repeated series_ids in the meta data,
 70    # make unique series_ids the index
 71    cat_meta.index = Index(cat_meta[metacol.id])
 72    cat_meta = cat_meta.groupby(cat_meta.index).first()
 73
 74    # get the ABS series data
 75    if isinstance(series_id, str):
 76        series_id = [series_id]
 77    return_data, return_meta = DataFrame(), DataFrame()
 78    for identifier in series_id:
 79        # confirm that the series ID is in the catalogue
 80        if identifier not in cat_meta.index:
 81            if args["verbose"]:
 82                print(f"Series ID {identifier} not found in ABS catalogue ID {cat}")
 83            if args["ignore_errors"]:
 84                continue
 85            raise ValueError(f"Series ID {identifier} not found in catalogue {cat}")
 86
 87        # confirm thay the index of the series is compatible
 88        table = str(cat_meta.loc[identifier, metacol.table])  # str for mypy
 89        data_series = cat_data[table][identifier]
 90        if (
 91            len(return_data) > 0
 92            and cast("PeriodIndex", return_data.index).freq != cast("PeriodIndex", data_series.index).freq
 93        ):
 94            if args["verbose"]:
 95                print(f"Frequency mismatch for series ID {identifier}")
 96            if args["ignore_errors"]:
 97                continue
 98            raise ValueError(f"Frequency mismatch for series ID {identifier}")
 99
100        # add the series data and meta data to the return values
101        if len(return_data) > 0:
102            return_data = return_data.reindex(return_data.index.union(data_series.index))
103        return_data[identifier] = data_series
104        return_meta = concat([return_meta, cat_meta.loc[identifier]], axis=1)
105
106    return return_data, return_meta.T

171def read_rba_ocr(*, monthly: bool = True, **kwargs: Any) -> Series:  # ignore_errors
172    """Read the Official Cash Rate (OCR) from the RBA website.
173
174    Return it in a pandas Series, with either a daily or monthly PeriodIndex,
175    depending on the value of the monthly parameter. The default is monthly.
176
177    Parameters
178    ----------
179    monthly : bool = True
180        If True, then the data will be returned with a monthly PeriodIndex.
181        If False, then the data will be returned with a daily PeriodIndex.
182    **kwargs : Any
183        Additional keyword arguments. The only keyword argument that is used is ignore_errors.
184    ignore_errors : bool = False
185        If True, then any major errors encountered will be printed and the function
186        will return an empty Series. If False, then any major errors encountered
187        will raise an exception.
188
189    Returns
190    -------
191    Series
192        The OCR data in a pandas Series, with an index of either daily or monthly Periods.
193
194    Examples
195    --------
196    ```python
197    ocr = read_rba_ocr(monthly=True)
198    ```
199
200    """
201    # read the OCR table from the RBA website, make float and sort, name the series
202    rba, _rba_meta = read_rba_table("A2", **kwargs)  # should have a daily PeriodIndex
203    ocr_series = rba.loc[lambda x: x.index >= "1990-08-02", "ARBAMPCNCRT"]
204    ocr = ocr_series.astype(float).sort_index()  # pyright: ignore[reportAttributeAccessIssue]
205    ocr.name = "RBA Official Cash Rate"
206
207    # bring up to date
208    today = Period(Timestamp.today(), freq=cast("PeriodIndex", ocr.index).freqstr)
209    last_period = cast("Period", ocr.index[-1])
210    if last_period < today:
211        ocr[today] = ocr.iloc[-1]
212
213    if not monthly:
214        # fill in missing days and return daily data
215        daily_index = period_range(start=ocr.index.min(), end=ocr.index.max(), freq="D")
216        return ocr.reindex(daily_index).ffill()
217
218    # convert to monthly data, keeping last value if duplicates in month
219    # fill in missing months
220    ocr.index = PeriodIndex(ocr.index, freq="M")
221    ocr = ocr[~ocr.index.duplicated(keep="last")]
222    monthly_index = period_range(start=ocr.index.min(), end=ocr.index.max(), freq="M")
223    return ocr.reindex(monthly_index, method="ffill")

 89def read_rba_table(table: str, **kwargs: Any) -> tuple[DataFrame, DataFrame]:  # ignore_errors
 90    """Read a table from the RBA website and return the actual data and meta data.
 91
 92    Returns the actual data and the meta data in a tuple of two DataFrames.
 93
 94    Parameters
 95    ----------
 96    table : str
 97        The table to read from the RBA website.
 98    **kwargs : Any
 99        Additional keyword arguments.
100        The only keyword argument that is used is ignore_errors.
101    ignore_errors : bool = False
102        If True, then any major errors encountered will be printed and the function
103        will return empty DataFrames. If False, then any major errors encountered
104        will raise an exception.
105
106    Returns
107    -------
108    tuple[DataFrame, DataFrame]
109        The primary data and the meta data in a tuple of two DataFrames.
110
111    Examples
112    --------
113    ```python
114    data, meta = read_rba_table("C1")
115    ```
116
117    """
118    # set-up
119    ignore_errors = kwargs.get("ignore_errors", False)
120    data, meta = DataFrame(), DataFrame()
121
122    # get the Excel file
123    excel = _get_excel_file(table, ignore_errors=ignore_errors, **kwargs)
124    if excel is None:
125        return data, meta
126
127    # read Excel file into DataFrame
128    try:
129        raw = read_excel(BytesIO(excel), header=None, index_col=None)
130    except Exception as e:
131        if ignore_errors:
132            print(f"Ignoring error: {e}")
133            return data, meta
134        raise
135
136    # extract the meta data
137    meta = raw.iloc[1:11, :].T.copy()
138    meta.columns = Index(meta.iloc[0])
139    renamer = {
140        "Mnemonic": rm.id,
141    }  # historical data is inconsistent
142    meta = meta.rename(columns=renamer)
143    meta = meta.iloc[1:, :]
144    meta.index = Index(meta[rm.id])
145    meta[rm.table] = table
146    meta[rm.tdesc] = raw.iloc[0, 0]
147    meta = meta.dropna(how="all", axis=1)  # drop columns with all NaNs
148
149    # extract the data
150    data = raw.iloc[10:, :].copy()
151    data.columns = Index(data.iloc[0])
152    data = data.iloc[1:, :]
153    data.index = DatetimeIndex(data.iloc[:, 0])
154    data = data.iloc[:, 1:]
155    data = data.dropna(how="all", axis=1)  # drop columns with all NaNs
156
157    # can we make the index into a PeriodIndex?
158    days = data.index.to_series().diff(1).dropna().dt.days
159    if days.min() >= MONTHLY_MIN_DAYS and days.max() <= MONTHLY_MAX_DAYS:
160        data.index = PeriodIndex(data.index, freq="M")
161    elif days.min() >= QUARTERLY_MIN_DAYS and days.max() <= QUARTERLY_MAX_DAYS:
162        data.index = PeriodIndex(data.index, freq="Q")
163    elif days.min() >= YEARLY_MIN_DAYS and days.max() <= YEARLY_MAX_DAYS:
164        data.index = PeriodIndex(data.index, freq="Y")
165    else:
166        data.index = PeriodIndex(data.index, freq="D")
167
168    return data, meta