readabs

39class ReadArgs(TypedDict):
40    """Type definition for ABS data reading arguments."""
41
42    verbose: NotRequired[bool]
43    ignore_errors: NotRequired[bool]
44    get_zip: NotRequired[bool]
45    get_excel_if_no_zip: NotRequired[bool]
46    get_excel: NotRequired[bool]
47    single_zip_only: NotRequired[str]
48    single_excel_only: NotRequired[str]
49    selected_excel: NotRequired[tuple[str, ...]]
50    history: NotRequired[str]
51    cache_only: NotRequired[bool]
52    keep_non_ts: NotRequired[bool]
53    zip_file: NotRequired[str]

 24@cache
 25def abs_catalogue(*, cache_only: bool = False, verbose: bool = False) -> DataFrame:
 26    """Return a DataFrame of ABS Catalogue numbers.
 27
 28    Downloads catalogue data from the ABS website on first call and caches
 29    for future use. The returned DataFrame contains catalogue numbers with
 30    their topics, themes, URLs, and status.
 31
 32    Parameters
 33    ----------
 34    cache_only : bool, default False
 35        If True, only use cached data and don't attempt to download.
 36    verbose : bool, default False
 37        If True, print progress messages.
 38
 39    Returns
 40    -------
 41    DataFrame
 42        DataFrame with columns ['Theme', 'Parent Topic', 'Topic', 'URL', 'Status']
 43        and index of catalogue IDs.
 44
 45    Raises
 46    ------
 47    CatalogueError
 48        If the catalogue data cannot be retrieved or parsed.
 49    HttpError
 50        If there's a network error downloading the catalogue.
 51    CacheError
 52        If cache_only=True but no cached data is available.
 53
 54    Example
 55    -------
 56    >>> import readabs as ra
 57    >>> catalogue = ra.abs_catalogue()
 58    >>> print(catalogue.head())
 59
 60    """
 61    try:
 62        # Download ABS catalogue page
 63        abs_bytes = get_file(ABS_CATALOGUE_URL, cache_only=cache_only, verbose=verbose)
 64
 65        if not abs_bytes:
 66            raise CatalogueError("No data retrieved from ABS catalogue URL")
 67
 68        # Parse HTML content
 69        try:
 70            html_content = abs_bytes.decode(DEFAULT_ENCODING, errors="replace")
 71        except UnicodeDecodeError as e:
 72            raise CatalogueError(f"Failed to decode HTML content: {e}") from e
 73
 74        # Extract tables from HTML
 75        try:
 76            tables = read_html(StringIO(html_content), extract_links="body")
 77            if not tables:
 78                raise CatalogueError("No tables found in HTML content")
 79            links = tables[-1]  # Get the last table
 80        except (ValueError, IndexError) as e:
 81            raise CatalogueError(f"Failed to parse HTML tables: {e}") from e
 82
 83        # Validate required columns exist
 84        required_cols = ["Catalogue number", "Topic"]
 85        missing_cols = [col for col in required_cols if col not in links.columns]
 86        if missing_cols:
 87            raise CatalogueError(f"Missing required columns: {missing_cols}")
 88
 89        # Extract catalogue numbers and URLs
 90        try:
 91            cats = links["Catalogue number"].apply(Series)[0]
 92            urls = links["Topic"].apply(Series)[1]
 93        except (KeyError, IndexError) as e:
 94            raise CatalogueError(f"Failed to extract catalogue data: {e}") from e
 95
 96        # Process topic URLs to create hierarchical structure
 97        url_snippets = _process_topic_urls(urls)
 98
 99        # Create main DataFrame with hierarchical topic structure
100        frame = _create_topic_frame(url_snippets)
101        frame["URL"] = urls
102
103        # Align catalogue numbers with processed frame
104        cats = cats[frame.index]
105
106        # Process catalogue status (active vs ceased)
107        cat_index, status = _process_catalogue_status(cats)
108
109        frame["Status"] = status
110        frame.index = Index(cat_index)
111        frame.index.name = CATALOGUE_INDEX_NAME
112
113    except (HttpError, CacheError, ValueError) as e:
114        raise CatalogueError(f"Error retrieving ABS catalogue: {e}") from e
115
116    return frame

100def annualise_percentages(data: DataT, *, periods_per_year: float) -> DataT:
101    """Annualise a growth rate (expressed as a percentage) for a period.
102
103    Args:
104        data : pandas Series or DataFrame - The growth rate (expressed as a
105            percentage) to annualise. Note a growth percentage of 5% is a growth
106            rate of 0.05.
107        periods_per_year : int or float, default 12 - The number of periods in a
108            year. For monthly data, this is 12.
109
110    Returns:
111        pandas Series or DataFrame - The annualised growth expressed as a percentage.
112            For DataFrame input, the annualised growth rate is calculated for each column.
113
114    Raises:
115        InvalidParameterError - If periods_per_year is not positive.
116        InvalidDataError - If data is not a Series or DataFrame.
117
118    """
119    if not isinstance(data, (Series, DataFrame)):
120        raise InvalidDataError("data must be a pandas Series or DataFrame")
121
122    if not isinstance(periods_per_year, (int, float)) or periods_per_year <= 0:
123        raise InvalidParameterError("periods_per_year must be a positive number")
124
125    try:
126        rates = data / 100.0
127        return annualise_rates(rates, periods_per_year=periods_per_year)
128    except Exception as e:
129        raise InvalidDataError(f"Error annualising percentages: {e}") from e

64def annualise_rates(data: DataT, *, periods_per_year: float) -> DataT:
65    """Annualise a growth rate for a period.
66
67    Note: returns a percentage value (and not a rate)!
68
69    Args:
70        data : pandas Series or DataFrame - The growth rate to annualise.
71            Note a growth rate of 0.05 is 5%.
72        periods_per_year : int or float, default 12 - The number of periods in a year.
73            For monthly data, this is 12.
74
75    Returns:
76        pandas Series or DataFrame - The annualised growth expressed as a percentage
77            (not a rate). For DataFrame input, the annualised growth rate is
78            calculated for each column.
79
80    Raises:
81        InvalidParameterError - If periods_per_year is not positive.
82    InvalidDataError - If data is not a Series or DataFrame.
83
84    """
85    if not isinstance(data, (Series, DataFrame)):
86        raise InvalidDataError("data must be a pandas Series or DataFrame")
87
88    if not isinstance(periods_per_year, (int, float)) or periods_per_year <= 0:
89        raise InvalidParameterError("periods_per_year must be a positive number")
90
91    try:
92        # identical branches: see percent_change()
93        if isinstance(data, Series):
94            return (((1 + data) ** periods_per_year) - 1) * 100
95        return (((1 + data) ** periods_per_year) - 1) * 100
96    except Exception as e:
97        raise InvalidDataError(f"Error annualising rates: {e}") from e

131def find_abs_id(
132    meta: DataFrame,
133    search_terms: dict[str, str],
134    **kwargs: Unpack[SearchArgs],
135) -> tuple[str, str, str]:  # table, series_id, units
136    """Find a unique ABS series identifier in the ABS metadata.
137
138    Parameters
139    ----------
140    meta : DataFrame
141        A pandas DataFrame of metadata from the ABS
142        (via read_abs_cat() or read_abs_series()).
143    search_terms : dict[str, str]
144        A dictionary {search_phrase: meta_column_name, ...} of search terms.
145        Note: the search terms must be unique, as a dictionary cannot hold the
146        same search term to be applied to different columns.
147    **kwargs : Unpack[SearchArgs]
148        Keyword arguments passed on to search_abs_meta(): exact_match,
149        regex, validate_unique and verbose. Unknown keyword arguments
150        are reported, and not passed on.
151    validate_unique : bool = True
152        Raise a ValueError if the search result is not a single
153        unique match. Note: the default is True for safety.
154
155    Returns
156    -------
157    tuple[str, str, str]
158        A tuple of the table, series_id and units for the unique
159        series_id that matches the search terms.
160
161    Metacol
162    -------
163    Because the meta data is a DataFrame, the columns can be referenced by either
164    their full textual name, or by the short name defined in the metacol object.
165    For example, if metacol is imported as mc, to refer to the
166    `Data Item Description` column, the user can refer to it as mc.did.
167
168    Example
169    -------
170    ```python
171    from readabs import metacol as mc  # alias for the ABS meta data column names
172    from readabs import read_abs_cat, find_abs_id, recalibrate
173    cat_num = "6202.0"  # The ABS labour force survey
174    data, meta = read_abs_cat(cat_num)
175    search_terms = {
176        "Employed total ;  Persons ;": mc.did,
177        "Seasonally Adjusted": mc.stype,
178        "62020001": mc.table,
179    }
180    table, series_id, units = find_abs_id(meta, search_terms)
181    print(f"Table: {table} Series ID: {series_id} Units: {units}")
182    recal_series, recal_units = recalibrate(data[table][series_id], units)
183    ```
184
185    """
186    check_kwargs(kwargs, "find_abs_id", _SEARCH_KWARGS)  # warn if invalid kwargs
187
188    # pass on only the known kwargs, so an unknown one is reported once
189    found = search_abs_meta(
190        meta,
191        search_terms,
192        exact_match=kwargs.get("exact_match", False),
193        regex=kwargs.get("regex", False),
194        validate_unique=kwargs.get("validate_unique", True),
195        verbose=kwargs.get("verbose", False),
196    ).iloc[0]
197    table, series_id, units = (
198        found[mc.table],
199        found[mc.id],
200        found[mc.unit],
201    )
202
203    return table, series_id, units

37@cache  # minimise slowness with repeat business
38def grab_abs_url(
39    cat: str = "",
40    url: str = "",
41    **kwargs: Unpack[ReadArgs],
42) -> dict[str, DataFrame]:
43    """For a given URL, extract the data from the Excel and ZIP file links found on that page.
44
45    The data is returned as a dictionary of DataFrames. The Excel files are converted
46    into DataFrames, with each sheet in each Excel file becoming a separate DataFrame.
47    ZIP files are examined for Excel files, which are similarly converted into
48    DataFrames. The dictionary of DataFrames is returned.
49
50    The preferred mechanism for reading ABS data is to use the `read_abs_cat()`
51    or `read_abs_series()` functions. This function is provided for those
52    cases where the data is not available in the ABS catalogue, where the
53    data is not a timeseries, or where the user wants to extract data from
54    a specific ABS landingpage.
55
56
57    Parameters
58    ----------
59    url : str = ""
60        A URL for an ABS Catalogue landing page. Either a url or
61        a catalogue number must be provided. If both are provided, the
62        URL will be used.
63
64    cat : str = ""
65        An ABS Catalogue number. If provided, and the URL is not
66        provided, then the Catalogue number will be used to get the URL.
67
68    **kwargs : Unpack[ReadArgs]
69        Accepts the same keyword arguments as `read_abs_cat()`.
70
71    Returns
72    -------
73    dict[str, DataFrame]
74        A dictionary of DataFrames.
75
76    """
77    # check/get the keyword arguments
78    url = _get_url(url, cat)
79    check_kwargs(kwargs, "grab_abs_url")  # warn if invalid kwargs
80    args = get_args(kwargs, "grab_abs_url")  # get the valid kwargs
81    if verbose := args["verbose"]:
82        print(f"grab_abs_url(): {url=}, {args=}")
83
84    # get the URL links to the relevant ABS data files on that webpage
85    links = get_abs_links(url, **args)
86    if not links:
87        print(f"No data files found at URL: {url}")
88        return {}  # return an empty Dictionary
89
90    # read the data files into a dictionary of DataFrames
91    abs_dict: dict[str, DataFrame] = {}
92
93    # Process single file requests first
94    abs_dict = _process_single_files(abs_dict, links, args, verbose=verbose)
95    if abs_dict:  # If single file was found and processed, return it
96        return abs_dict
97
98    # Process all files based on configuration
99    return _process_all_files(abs_dict, links, args)

102def grab_abs_zip(
103    zip_path: Path | str,
104    **kwargs: Unpack[ReadArgs]
105) -> dict[str, DataFrame]:
106    """Grab and process a single ABS ZIP file from a file system location.
107
108    This is a convenience function that opens an ABS ZIP file from a local
109    filesystem path. Expect to be used rarely.
110
111    Parameters
112    ----------
113    zip_path : Path | str
114        The local filesystem path of the ABS ZIP file to open and process.
115
116    **kwargs : Unpack[ReadArgs]
117        Additional keyword arguments for file retrieval and processing.
118
119    Returns
120    -------
121    dict[str, DataFrame]
122        A dictionary of DataFrames extracted from the ZIP file.
123
124    """
125    check_kwargs(kwargs, "grab_abs_zip")  # warn if invalid kwargs
126    args = get_args(kwargs, "grab_abs_zip")  # get the valid kwargs
127
128    zp: Path = zip_path if isinstance(zip_path, Path) else Path(zip_path)
129    zip_bytes = zp.read_bytes()
130    abs_dict: dict[str, DataFrame] = {}
131    return _process_zip(abs_dict, zip_bytes, **args)

199def monthly_to_qtly(data: DataT, q_ending: str = "DEC", f: str = "mean") -> DataT:
200    """Convert monthly data to quarterly data.
201
202    This is done by taking the mean (or sum) of the three months in each quarter.
203    Ignore quarters with less than or more than three months data. Drop NA items.
204    Change f to "sum" for a quarterly sum.
205
206    Args:
207        data : pandas Series or DataFrame
208            The data to convert to quarterly frequency.
209        q_ending : str, default "DEC"
210            The month in which the quarter ends. For example, "DEC" for December.
211        f : str, default "mean"
212            The function to apply to the three months in each quarter.
213            Change to "sum" for a quarterly sum. The default is a
214            quarterly mean.
215
216    Returns:
217        pandas Series or DataFrame
218            The data with a quarterly PeriodIndex. If a quarter has less than
219            three months data, the quarter is dropped. If the quarter has more
220            than three months data, the quarter is dropped. Any NA data is removed.
221        For DataFrame input, the function is applied to each column.
222
223    Raises:
224        InvalidDataError - If data is not a Series or DataFrame, or does not
225            have a monthly PeriodIndex.
226        InvalidParameterError - If q_ending or f parameters are invalid.
227
228    """
229    # Validate inputs
230    if not isinstance(data, (Series, DataFrame)):
231        raise InvalidDataError("data must be a pandas Series or DataFrame")
232
233    # non-monthly data would otherwise silently return an empty result
234    index = data.index
235    if not isinstance(index, PeriodIndex) or index.freqstr != "M":
236        raise InvalidDataError(
237            f"data must have a monthly PeriodIndex, got {type(index).__name__}"
238            f" with freq {getattr(index, 'freqstr', None)!r}"
239        )
240
241    valid_endings = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"]
242    if q_ending.upper() not in valid_endings:
243        raise InvalidParameterError(f"q_ending must be one of {valid_endings}")
244
245    valid_aggregations = ["mean", "sum", "min", "max", "std", "var"]
246    if f not in valid_aggregations:
247        raise InvalidParameterError(f"f must be one of {valid_aggregations}")
248
249    try:
250        if isinstance(data, Series):
251            return _monthly_to_qtly_series(data, q_ending, f)
252        # the guard above means this must be a DataFrame
253        return data.__class__({col: _monthly_to_qtly_series(data[col], q_ending, f) for col in data.columns})
254    except Exception as e:
255        raise InvalidDataError(f"Error converting monthly to quarterly data: {e}") from e

29def percent_change(data: DataT, n_periods: int) -> DataT:
30    """Calculate a percentage change in a contiguous, ordered series over n_periods.
31
32    Args:
33        data : pandas Series or DataFrame
34            The data to calculate the percentage change for.
35        n_periods : int
36            The number of periods to calculate the percentage change over.
37            Typically 4 for quarterly data, and 12 for monthly data.
38
39    Returns:
40        pandas Series or DataFrame - The percentage change in the data over n_periods.
41            For DataFrame input, the percentage change is calculated for each column.
42
43    Raises:
44        InvalidParameterError - If n_periods is not a positive integer.
45        InvalidDataError - If data is not a Series or DataFrame.
46
47    """
48    if not isinstance(n_periods, int) or n_periods <= 0:
49        raise InvalidParameterError("n_periods must be a positive integer")
50
51    if not isinstance(data, (Series, DataFrame)):
52        raise InvalidDataError("data must be a pandas Series or DataFrame")
53
54    try:
55        # identical branches: pandas-stubs types Series[Any] scalar arithmetic as Any,
56        # which pyright cannot match to the Series constraint of DataT unless narrowed
57        if isinstance(data, Series):
58            return (data / data.shift(n_periods) - 1) * 100
59        return (data / data.shift(n_periods) - 1) * 100
60    except Exception as e:
61        raise InvalidDataError(f"Error calculating percentage change: {e}") from e

11def print_abs_catalogue(*, cache_only: bool = False, verbose: bool = False) -> None:
12    """Print a table of ABS Catalogue Numbers with their metadata.
13
14    Displays catalogue numbers that contain time-series data along with
15    their theme, parent topic, topic, and status information. The URL
16    column is excluded from the display for readability.
17
18    This is primarily a convenience function to help users identify the
19    correct catalogue number for data retrieval functions.
20
21    Parameters
22    ----------
23    cache_only : bool, default False
24        If True, only use cached catalogue data.
25    verbose : bool, default False
26        If True, print progress messages during catalogue retrieval.
27
28    Raises
29    ------
30    CatalogueError
31        If the catalogue data cannot be retrieved or processed.
32    HttpError
33        If there's a network error downloading the catalogue.
34    CacheError
35        If cache_only=True but no cached data is available.
36
37    Example
38    -------
39    >>> import readabs as ra
40    >>> ra.print_abs_catalogue()
41
42    """
43    try:
44        # Retrieve the catalogue data
45        catalogue = abs_catalogue(cache_only=cache_only, verbose=verbose)
46
47        # Validate catalogue is not empty
48        if catalogue.empty:
49            print("No catalogue data available.")
50            return
51
52        # Select columns for display (exclude URL for readability)
53        available_columns = [col for col in DISPLAY_COLUMNS if col in catalogue.columns]
54        if not available_columns:
55            print("Catalogue data does not contain expected columns.")
56            return
57
58        display_data = catalogue[available_columns]
59
60        # Generate and print markdown table
61        try:
62            markdown_output = display_data.to_markdown()
63            print(markdown_output)
64        except Exception:  # noqa: BLE001
65            print(display_data.to_string())
66
67    except (CatalogueError, HttpError, CacheError) as e:
68        print(f"Error retrieving catalogue: {e}")

47def print_rba_catalogue(*, cache_only: bool = False, verbose: bool = False) -> None:
48    """Print to standard output a table of the RBA Catalogue Numbers.
49
50    This function prints a formatted table of RBA catalogue numbers
51    to standard output.
52
53    Parameters
54    ----------
55    cache_only : bool = False
56        If True, only use the cache.
57    verbose : bool = False
58        If True, print progress messages.
59
60    Returns
61    -------
62    None
63        This function does not return anything.
64
65    Example
66    -------
67
68    ```python
69    import readabs as ra
70    ra.print_rba_catalogue()
71    ```
72
73    """
74    rba_catalog = rba_catalogue(cache_only=cache_only, verbose=verbose)
75    print(rba_catalog.loc[:, rba_catalog.columns != "URL"].to_markdown())

132def qtly_to_monthly(
133    data: DataT,
134    *,
135    interpolate: bool = True,
136    limit: int | None = 2,  # only used if interpolate is True
137    dropna: bool = True,
138) -> DataT:
139    """Convert data from Quarterly PeriodIndex to a Monthly PeriodIndex.
140
141    Args:
142        data: Series or DataFrame with quarterly PeriodIndex. Assumes the index is unique.
143            The data to convert to monthly frequency.
144        interpolate: bool, default True
145            Whether to interpolate the missing monthly data.
146        limit: int, default 2 - The maximum number of consecutive missing months
147            to interpolate.
148        dropna: bool, default True - Whether to drop NA data
149
150    Returns:
151        pandas Series or DataFrame - The data with a Monthly PeriodIndex.
152            If interpolate is True, the missing monthly data is interpolated.
153            If dropna is True, any NA data is removed.
154
155    Raises:
156        InvalidDataError - If data index is not a quarterly PeriodIndex or has issues.
157        InvalidParameterError - If limit parameter is invalid.
158
159    """
160    # Validate input data
161    if not isinstance(data, (Series, DataFrame)):
162        raise InvalidDataError("data must be a pandas Series or DataFrame")
163
164    if not isinstance(data.index, PeriodIndex):
165        raise InvalidDataError("data index must be a PeriodIndex")
166
167    if not (data.index.freqstr and data.index.freqstr[0] == "Q"):
168        raise InvalidDataError("data index must have quarterly frequency")
169
170    if not data.index.is_unique:
171        raise InvalidDataError("data index must be unique")
172
173    if not data.index.is_monotonic_increasing:
174        raise InvalidDataError("data index must be monotonic increasing")
175
176    if limit is not None and (not isinstance(limit, int) or limit < 0):
177        raise InvalidParameterError("limit must be a non-negative integer or None")
178
179    # do the heavy lifting
180    try:
181        data = (
182            data.set_axis(labels=data.index.to_timestamp(how="end"), axis="index")
183            .resample(rule="ME")  # adds in every missing month
184            .first(min_count=1)  # generates nans for new months
185            # assumes only one value per quarter (ie. unique index)
186            .pipe(_set_axis_monthly_periods)
187        )
188    except Exception as e:
189        raise InvalidDataError(f"Error in quarterly to monthly conversion: {e}") from e
190
191    if interpolate:
192        data = data.interpolate(limit_area="inside", limit=limit)
193    if dropna:
194        data = data.dropna()
195
196    return data

17@cache
18def rba_catalogue(*, cache_only: bool = False, verbose: bool = False) -> DataFrame:
19    """Return a DataFrame of RBA Catalogue numbers.
20
21    In the first instance, this is downloaded from the RBA website, and
22    cached for future use.
23
24    Parameters
25    ----------
26    cache_only : bool = False
27        If True, only use the cache.
28    verbose : bool = False
29        If True, print progress messages.
30
31    Returns
32    -------
33    DataFrame
34        A DataFrame of RBA Catalogue numbers.
35
36    Example
37    -------
38    ```python
39    import readabs as ra
40    catalogue = ra.rba_catalogue()
41    ```
42
43    """
44    return _get_rba_links(cache_only=cache_only, verbose=verbose)

144def read_abs_by_desc(
145    wanted: list[str] | dict[str, str] | dict[str, dict[str, Any]],
146    **kwargs: Any,
147) -> tuple[dict[str, pd.Series], pd.DataFrame]:
148    """Get specific ABS data series by searching the ABS meta data.
149
150    Parameters
151    ----------
152    wanted : list of str, dict of str:str, or dict of str:dict
153        The data
154        item descriptions to search for. If a list, it will be a list of
155        descriptions to search for. If a dictionary, the keys will a name.
156        The dictionary values can be either a string (the data item
157        description to search for) or a dictionary of keyword arguments, one of
158        which would be the data item description to search for.
159    **kwargs : Any
160        Keyword arguments to control the data retrieval.
161        The keyword arguments can include the following:
162        - abs_dict : dict - the dictionary of ABS data to search (from
163            read_abs_cat()).
164        - abs_meta : DataFrame - the metadata for the ABS data (from
165            read_abs_cat()).
166        - for the retrieval of data, the "cat" argument must be present.
167            The following arguments, if present, will also be used (ie.
168            passed to read_abs_cat()): ["ignore_errors", "get_zip",
169            "get_excel_if_no_zip", "get_excel", "cache_only",
170            "single_excel_only", "selected_excel", "single_zip_only",
171            "verbose"].
172        - for the selection of data, the following metacol names, if present,
173            will be used to construct the selector: "cat", "did"
174            "stype", "id", "start", "end", "num", "unit", "dtype", "freq",
175            "cmonth", "table", "tdesc".
176        - finally, the following arguments will be passed to the find_abs_id()
177            and search_abs_meta() functions: ["validate_unique", "exact_match",
178            "regex", "verbose"].
179
180    Notes
181    -----
182    - if "wanted" is of type list[str] or dict[str, str], the kwargs should
183        include sufficient keys from the metacol dataclass to get the data.
184        Typically, the "cat" key, the "table" key, and the "stype" key would
185        be required. The did key would taken from the wanted list or
186        dictionary.
187    if wanted is of type dict[str, dict[str, Any]], the inner dictionary
188        must contain a "did" key. The other keys that can be used for the
189        data retrieval are the same as the metacol dataclass fileds, namely:
190        "cat", "stype", "id", "start", "end", "num", "unit", "dtype", "freq",
191        "cmonth", "table", "tdesc".
192    - if abs_dict and abs_meta are provided within the kwargs, they will be
193        used to locate and extract the selected data.
194    - if abs_dict and abs_meta are not provided, then, (1) wanted must be of
195        type dict[str, dict[str, Any]] and (2) the inner dictionary must
196        contain a "cat" key so the data can be retrieved. Other keys that
197        can be used for the data retrieval are the same as for read_abs_cat(),
198        namely ["ignore_errors", "get_zip", "get_excel_if_no_zip",
199        "get_excel", "single_excel_only", "selected_excel",
200        "single_zip_only", "cache_only"].
201
202
203    Returns
204    -------
205    Returns a tuple of two items:
206    - A dictionary of pandas Series objects, where the keys are the series
207      descriptions. The series.name attribute will be the ABS series-id.
208    - A pandas DataFrame containing the metadata for the series.
209
210    Example
211    -------
212
213    ```python
214    import readabs as ra
215    from pandas import DataFrame
216    cat_num = "5206.0"  # The ABS National Accounts
217    data, meta = ra.read_abs_cat(cat=cat_num)
218    wanted = ["Gross domestic product: Chain volume measures ;",]
219    selected, selected_meta = ra.read_abs_by_desc(
220        wanted=wanted, abs_dict=data, abs_meta=meta, table="5206001_Key_Aggregates"
221    )
222    ```
223
224    """
225    # - preparation
226    if not _work_to_do(wanted):
227        return {}, pd.DataFrame()
228    if isinstance(wanted, list):
229        wanted = _wlist_to_wdict(wanted)
230    abs_dict = kwargs.get("abs_dict", {})
231    abs_meta = kwargs.get("abs_meta", pd.DataFrame())
232    kwarg_selector = _get_search_terms(kwargs, {})
233    search_args = _get_search_args(kwargs, {})
234
235    return_dict = {}
236    return_meta = pd.DataFrame()
237    for key, value in wanted.items():
238        item_selector = kwarg_selector.copy()
239        item_search_args = search_args.copy()
240        if isinstance(value, str):
241            series, meta = _get_item_from_str(
242                item=value,
243                data_dict=abs_dict,
244                data_meta=abs_meta,
245                item_selector=item_selector,
246                search_args=item_search_args,
247            )
248
249        elif isinstance(value, dict):
250            series, meta = _get_item_from_dict(
251                item_dict=value,
252                data_dict=abs_dict,
253                data_meta=abs_meta,
254                item_selector=item_selector,
255                search_args=item_search_args,
256                **kwargs,
257            )
258        else:
259            raise TypeError(
260                "Each value in the wanted list/dictionary must be either a string " + "or a dictionary."
261            )
262
263        # save search results
264        return_dict[key] = series
265        return_meta = pd.concat([return_meta, meta])
266
267    return return_dict, return_meta

 28@cache  # minimise slowness for any repeat business
 29def read_abs_cat(
 30    cat: str,
 31    url: str = "",
 32    **kwargs: Unpack[ReadArgs],
 33) -> tuple[dict[str, DataFrame], DataFrame]:
 34    """For a specific catalogue identifier, return the complete ABS Catalogue information as DataFrames.
 35
 36    This function returns the complete ABS Catalogue information as a
 37    python dictionary of pandas DataFrames, as well as the associated metadata
 38    in a separate DataFrame. The function automates the collection of zip and
 39    excel files from the ABS website. If necessary, these files are downloaded,
 40    and saved into a cache directory. The files are then parsed to extract time
 41    series data, and the associated metadata.
 42
 43    By default, the cache directory is `./.readabs_cache/`. You can change the
 44    default directory name by setting the shell environment variable
 45    `READABS_CACHE_DIR` with the name of the preferred directory.
 46
 47    Parameters
 48    ----------
 49    cat : str
 50        The ABS Catalogue Number for the data to be downloaded and made
 51        available by this function. This argument must be specified in the
 52        function call.
 53
 54    **kwargs : Unpack[ReadArgs]
 55        The following parameters may be passed as optional keyword arguments.
 56
 57    url : str = ""
 58        The URL of an ABS landing page. Use this for discontinued series
 59        that are no longer in the ABS Time Series Directory. If provided,
 60        data will be retrieved from this URL instead of looking up the
 61        catalogue number. Example:
 62        `read_abs_cat(cat="8501.0", url="https://www.abs.gov.au/.../jun-2025")`
 63
 64    keep_non_ts : bool = False
 65        A flag for whether to keep the non-time-series tables
 66        that might form part of an ABS catalogue item. Normally, the
 67        non-time-series information is ignored, and not made available to
 68        the user.
 69
 70    history : str = ""
 71        Provide a month-year string to extract historical ABS data.
 72        For example, you can set history="dec-2023" to the get the ABS data
 73        for a catalogue identifier that was originally published in respect
 74        of Q4 of 2023. Note: not all ABS data sources are structured so that
 75        this technique works in every case; but most are.
 76
 77    verbose : bool = False
 78        Setting this to true may help diagnose why something
 79        might be going wrong with the data retrieval process.
 80
 81    ignore_errors : bool = False
 82        Normally, this function will cease downloading when
 83        an error in encountered. However, sometimes the ABS website has
 84        malformed links, and changing this setting is necessitated. (Note:
 85        if you drop a message to the ABS, they will usually fix broken
 86        links with a business day).
 87
 88    get_zip : bool = True
 89        Download the excel files in .zip files.
 90
 91    get_excel_if_no_zip : bool = True
 92        Only try to download .xlsx files if there are no zip
 93        files available to be downloaded. Only downloading individual excel
 94        files when there are no zip files to download can speed up the
 95        download process.
 96
 97    get_excel : bool = False
 98        The default value means that excel files are not
 99        automatically download. Note: at least one of `get_zip`,
100        `get_excel_if_no_zip`, or `get_excel` must be true. For most ABS
101        catalogue items, it is sufficient to just download the one zip
102        file. But note, some catalogue items do not have a zip file.
103        Others have quite a number of zip files.
104
105    single_excel_only : str = ""
106        If this argument is set to a table name (without the
107        .xlsx extension), only that excel file will be downloaded. If
108        set, and only a limited subset of available data is needed,
109        this can speed up download times significantly. Note: overrides
110        `get_zip`, `get_excel_if_no_zip`, `get_excel` and `single_zip_only`.
111
112    selected_excel : tuple[str, ...] = ()
113        If set to a tuple of table names (without the .xlsx extension),
114        only those excel files will be downloaded. Useful when several
115        specific tables are needed and downloading the full zip would
116        be wasteful. Example:
117        `selected_excel=("62020001", "62020017", "62020X28")`.
118        Must be a tuple (not a list) because `read_abs_cat` uses an
119        internal cache that requires hashable arguments. Note: overrides
120        `get_zip`, `get_excel_if_no_zip`, `get_excel` and `single_zip_only`
121        when at least one matching file is found.
122
123    single_zip_only : str = ""
124        If this argument is set to a zip file name (without
125        the .zip extension), only that zip file will be downloaded.
126        If set, and only a limited subset of available data is needed,
127        this can speed up download times significantly. Note: overrides
128        `get_zip`, `get_excel_if_no_zip`, and `get_excel`.
129
130    cache_only : bool = False
131        If set to True, this function will only access
132        data that has been previously cached. Normally, the function
133        checks the date of the cache data against the date of the data
134        on the ABS website, before deciding whether the ABS has fresher
135        data that needs to be downloaded to the cache.
136
137    zip_file: str | Path = ""
138        If set to a specific zip file name (with or without the .zip
139        extension), this function will only extract data from that zip file
140        on the local file system. This may be useful for debugging purposes.
141
142    Returns
143    -------
144    tuple[dict[str, DataFrame], DataFrame]
145        The function returns a tuple of two items. The first item is a
146        python dictionary of pandas DataFrames (which is the primary data
147        associated with the ABS catalogue item). The second item is a
148        DataFrame of ABS metadata for the ABS collection.
149
150        Note:
151        You can retrieve non-timeseries data using the grab_abs_url()
152        function. That takes the URL for the ABS landing page for the ABS
153        collection you are interested in. The read_abs_cat function is for
154        ABS catalogue identifiers which are timeseries data, for which the
155        metadata can be extracted.
156
157    Example
158    -------
159
160    ```python
161    import readabs as ra
162    from pandas import DataFrame
163    cat_num = "6202.0"  # The ABS labour force survey
164    data: tuple[dict[str, DataFrame], DataFrame] = ra.read_abs_cat(cat=cat_num)
165    abs_dict, meta = data
166    ```
167
168    """
169    # --- get the time series data ---
170    zip_file = kwargs.get("zip_file")
171    raw_abs_dict = (
172        grab_abs_zip(zip_file, **kwargs) if zip_file else grab_abs_url(cat=cat, url=url, **kwargs)
173    )
174    response = _get_time_series_data(cat, raw_abs_dict, **kwargs)
175
176    if not response:
177        response = {}, DataFrame()
178
179    return response  # dictionary of DataFrames, and a DataFrame of metadata

 15def read_abs_series(
 16    cat: str,
 17    series_id: str | Sequence[str],
 18    url: str = "",
 19    **kwargs: Unpack[ReadArgs],
 20) -> tuple[DataFrame, DataFrame]:
 21    """Get specific ABS data series by their ABS catalogue and series identifiers.
 22
 23    Parameters
 24    ----------
 25    cat : str
 26        The ABS catalogue ID.
 27
 28    series_id : str | Sequence[str]
 29        An ABS series ID or a sequence of ABS series IDs.
 30
 31    url : str = ""
 32        The URL of an ABS landing page. Use this for discontinued series
 33        that are no longer in the ABS Time Series Directory. If provided,
 34        data is retrieved from this URL instead of looking up the catalogue
 35        number. Passed through to read_abs_cat().
 36
 37    **kwargs : Any
 38        Keyword arguments for the read_abs_series function,
 39        which are the same as the keyword arguments for the
 40        read_abs_cat function.
 41
 42    Returns
 43    -------
 44    tuple[DataFrame, DataFrame]
 45        A tuple of two DataFrames, one for the primary data and one for the metadata.
 46
 47    Example
 48    -------
 49
 50    ```python
 51    import readabs as ra
 52    from pandas import DataFrame
 53    cat_num = "6202.0"  # The ABS labour force survey
 54    unemployment_rate = "A84423050A"
 55    seo = "62020001"  # The ABS table name
 56    data, meta = ra.read_abs_series(
 57        cat=cat_num, series_id=unemployment_rate, single_excel_only=seo
 58    )
 59    ```
 60
 61    """
 62    # check for unexpected keyword arguments/get defaults
 63    check_kwargs(kwargs, "read_abs_series")
 64    args = get_args(kwargs, "read_abs_series")
 65
 66    # read the ABS category data
 67    cat_data, cat_meta = read_abs_cat(cat, url=url, **args)
 68
 69    # drop repeated series_ids in the meta data,
 70    # make unique series_ids the index
 71    cat_meta.index = Index(cat_meta[metacol.id])
 72    cat_meta = cat_meta.groupby(cat_meta.index).first()
 73
 74    # get the ABS series data
 75    if isinstance(series_id, str):
 76        series_id = [series_id]
 77    return_data, return_meta = DataFrame(), DataFrame()
 78    for identifier in series_id:
 79        # confirm that the series ID is in the catalogue
 80        if identifier not in cat_meta.index:
 81            if args["verbose"]:
 82                print(f"Series ID {identifier} not found in ABS catalogue ID {cat}")
 83            if args["ignore_errors"]:
 84                continue
 85            raise ValueError(f"Series ID {identifier} not found in catalogue {cat}")
 86
 87        # confirm thay the index of the series is compatible
 88        table = str(cat_meta.loc[identifier, metacol.table])  # str for mypy
 89        data_series = cat_data[table][identifier]
 90        if (
 91            len(return_data) > 0
 92            and cast("PeriodIndex", return_data.index).freq != cast("PeriodIndex", data_series.index).freq
 93        ):
 94            if args["verbose"]:
 95                print(f"Frequency mismatch for series ID {identifier}")
 96            if args["ignore_errors"]:
 97                continue
 98            raise ValueError(f"Frequency mismatch for series ID {identifier}")
 99
100        # add the series data and meta data to the return values
101        if len(return_data) > 0:
102            return_data = return_data.reindex(return_data.index.union(data_series.index))
103        return_data[identifier] = data_series
104        return_meta = concat([return_meta, cat_meta.loc[identifier]], axis=1)
105
106    return return_data, return_meta.T

171def read_rba_ocr(*, monthly: bool = True, **kwargs: Any) -> Series:  # ignore_errors
172    """Read the Official Cash Rate (OCR) from the RBA website.
173
174    Return it in a pandas Series, with either a daily or monthly PeriodIndex,
175    depending on the value of the monthly parameter. The default is monthly.
176
177    Parameters
178    ----------
179    monthly : bool = True
180        If True, then the data will be returned with a monthly PeriodIndex.
181        If False, then the data will be returned with a daily PeriodIndex.
182    **kwargs : Any
183        Additional keyword arguments. The only keyword argument that is used is ignore_errors.
184    ignore_errors : bool = False
185        If True, then any major errors encountered will be printed and the function
186        will return an empty Series. If False, then any major errors encountered
187        will raise an exception.
188
189    Returns
190    -------
191    Series
192        The OCR data in a pandas Series, with an index of either daily or monthly Periods.
193
194    Examples
195    --------
196    ```python
197    ocr = read_rba_ocr(monthly=True)
198    ```
199
200    """
201    # read the OCR table from the RBA website, make float and sort, name the series
202    rba, _rba_meta = read_rba_table("A2", **kwargs)  # should have a daily PeriodIndex
203    ocr_series = rba.loc[lambda x: x.index >= "1990-08-02", "ARBAMPCNCRT"]
204    ocr = ocr_series.astype(float).sort_index()  # pyright: ignore[reportAttributeAccessIssue]
205    ocr.name = "RBA Official Cash Rate"
206
207    # bring up to date
208    today = Period(Timestamp.today(), freq=cast("PeriodIndex", ocr.index).freqstr)
209    last_period = cast("Period", ocr.index[-1])
210    if last_period < today:
211        ocr[today] = ocr.iloc[-1]
212
213    if not monthly:
214        # fill in missing days and return daily data
215        daily_index = period_range(start=ocr.index.min(), end=ocr.index.max(), freq="D")
216        return ocr.reindex(daily_index).ffill()
217
218    # convert to monthly data, keeping last value if duplicates in month
219    # fill in missing months
220    ocr.index = PeriodIndex(ocr.index, freq="M")
221    ocr = ocr[~ocr.index.duplicated(keep="last")]
222    monthly_index = period_range(start=ocr.index.min(), end=ocr.index.max(), freq="M")
223    return ocr.reindex(monthly_index, method="ffill")

 89def read_rba_table(table: str, **kwargs: Any) -> tuple[DataFrame, DataFrame]:  # ignore_errors
 90    """Read a table from the RBA website and return the actual data and meta data.
 91
 92    Returns the actual data and the meta data in a tuple of two DataFrames.
 93
 94    Parameters
 95    ----------
 96    table : str
 97        The table to read from the RBA website.
 98    **kwargs : Any
 99        Additional keyword arguments.
100        The only keyword argument that is used is ignore_errors.
101    ignore_errors : bool = False
102        If True, then any major errors encountered will be printed and the function
103        will return empty DataFrames. If False, then any major errors encountered
104        will raise an exception.
105
106    Returns
107    -------
108    tuple[DataFrame, DataFrame]
109        The primary data and the meta data in a tuple of two DataFrames.
110
111    Examples
112    --------
113    ```python
114    data, meta = read_rba_table("C1")
115    ```
116
117    """
118    # set-up
119    ignore_errors = kwargs.get("ignore_errors", False)
120    data, meta = DataFrame(), DataFrame()
121
122    # get the Excel file
123    excel = _get_excel_file(table, ignore_errors=ignore_errors, **kwargs)
124    if excel is None:
125        return data, meta
126
127    # read Excel file into DataFrame
128    try:
129        raw = read_excel(BytesIO(excel), header=None, index_col=None)
130    except Exception as e:
131        if ignore_errors:
132            print(f"Ignoring error: {e}")
133            return data, meta
134        raise
135
136    # extract the meta data
137    meta = raw.iloc[1:11, :].T.copy()
138    meta.columns = Index(meta.iloc[0])
139    renamer = {
140        "Mnemonic": rm.id,
141    }  # historical data is inconsistent
142    meta = meta.rename(columns=renamer)
143    meta = meta.iloc[1:, :]
144    meta.index = Index(meta[rm.id])
145    meta[rm.table] = table
146    meta[rm.tdesc] = raw.iloc[0, 0]
147    meta = meta.dropna(how="all", axis=1)  # drop columns with all NaNs
148
149    # extract the data
150    data = raw.iloc[10:, :].copy()
151    data.columns = Index(data.iloc[0])
152    data = data.iloc[1:, :]
153    data.index = DatetimeIndex(data.iloc[:, 0])
154    data = data.iloc[:, 1:]
155    data = data.dropna(how="all", axis=1)  # drop columns with all NaNs
156
157    # can we make the index into a PeriodIndex?
158    days = data.index.to_series().diff(1).dropna().dt.days
159    if days.min() >= MONTHLY_MIN_DAYS and days.max() <= MONTHLY_MAX_DAYS:
160        data.index = PeriodIndex(data.index, freq="M")
161    elif days.min() >= QUARTERLY_MIN_DAYS and days.max() <= QUARTERLY_MAX_DAYS:
162        data.index = PeriodIndex(data.index, freq="Q")
163    elif days.min() >= YEARLY_MIN_DAYS and days.max() <= YEARLY_MAX_DAYS:
164        data.index = PeriodIndex(data.index, freq="Y")
165    else:
166        data.index = PeriodIndex(data.index, freq="D")
167
168    return data, meta