readabs.search_abs_meta

  1"""Search a DataFrame of ABS meta data using search terms.
  2
  3Using a dictionary of search terms, identify the row or rows that match
  4all of the search terms.
  5"""
  6
  7from typing import Unpack
  8
  9from pandas import DataFrame, Index
 10
 11# local imports
 12from readabs.abs_meta_data import metacol as mc
 13from readabs.read_abs_cat import read_abs_cat
 14from readabs.read_support import _SEARCH_KWARGS, SearchArgs, VerboseArgs, check_kwargs
 15
 16
 17def search_abs_meta(
 18    meta: DataFrame,  # sourced from read_abs_series() or read_abs_cat()
 19    search_terms: dict[str, str],  # {search_term: meta_data_column_name, ...}
 20    *,
 21    exact_match: bool = False,
 22    regex: bool = False,
 23    validate_unique: bool = False,  # useful safety-net if you expect only one match
 24    **kwargs: Unpack[VerboseArgs],
 25) -> DataFrame:
 26    """Extract from the ABS meta data those rows that match the search_terms.
 27
 28    Iteratively search the meta data one search_term at a time.
 29
 30    Parameters
 31    ----------
 32    meta : DataFrame
 33        A pandas DataFrame of metadata from the ABS
 34        (via read_abs_cat() or read_abs_series()).
 35    search_terms : dict[str, str]
 36        A dictionary {search_phrase: meta_column_name, ...} of search terms.
 37        Note: the search terms must be unique, as a dictionary cannot hold the
 38        same search term to be applied to different columns.
 39    exact_match : bool = False
 40        Whether to match using == (exact) or .str.contains() (inexact).
 41    regex : bool = False
 42        Whether to use regular expressions in the search.
 43    validate_unique : bool = False
 44        Raise a ValueError if the search result is not unique.
 45    **kwargs : Unpack[VerboseArgs]
 46        Additional keyword arguments. The only keyword argument
 47        that is used is verbose. Unknown keyword arguments are reported.
 48    verbose : bool = False
 49        Print additional information while searching; which can
 50        be useful when diagnosing problems with search terms.
 51
 52    Returns
 53    -------
 54    DataFrame
 55        Returns a pandas DataFrame of matching rows (subseted from meta).
 56        Note, The index for the returned meta data will always comprise ABS
 57        series_ids. Duplicate indexes will be removed from the meta data
 58        (ie. where the same ABS series appears in more than one table, this
 59        function will only report the first match).
 60
 61    Metacol
 62    -------
 63    Because the meta data is a DataFrame, the columns can be referenced by either
 64    their full textual name, or by the short name defined in the metacol object.
 65    For example, if metacol is imported as mc, to refer to the
 66    `Data Item Description` column, the user can refer to it as mc.did.
 67
 68    Example
 69    -------
 70    ```python
 71    from readabs import metacol as mc  # alias for the ABS meta data column names
 72    from readabs import read_abs_cat, search_abs_meta
 73    cat_num = "6202.0"  # The ABS labour force survey
 74    data, meta = read_abs_cat(cat_num)
 75    search_terms = {
 76        "Unemployment rate": mc.did,  # the data item description
 77        "Persons": mc.did,
 78        "Seasonally Adjusted": mc.stype,
 79        "Percent": mc.unit,
 80        "62020001": mc.table,
 81    }
 82    rows = search_abs_meta(meta, search_terms, verbose=True)
 83    print(rows)  # should have three rows : FT/PT/All Unemployment rates
 84    ```
 85
 86    """
 87    # warn if invalid kwargs; the named parameters never reach kwargs,
 88    # so checking against all the SearchArgs names only affects the message
 89    check_kwargs(kwargs, "search_abs_meta", _SEARCH_KWARGS)
 90
 91    # get the verbose-flag from kwargs
 92    verbose = kwargs.get("verbose", False)
 93
 94    # establish the starting point
 95    meta_select = meta.copy()  # preserve the original meta data
 96    if verbose:
 97        print(f"In search_abs_meta() {exact_match=} {regex=} {verbose=}")
 98        print(f"In search_abs_meta() starting with {len(meta_select)} rows in the meta_data.")
 99
100    # iteratively search
101    for phrase, column in search_terms.items():
102        if verbose:
103            print(f"Searching {len(meta_select)}: term: {phrase} in-column: {column}")
104
105        pick_me = (
106            (meta_select[column] == phrase)
107            if (exact_match or column == mc.table)
108            else meta_select[column].str.contains(phrase, regex=regex)
109        )
110        meta_select = meta_select[pick_me]
111        if verbose:
112            print(f"In find_rows() have found {len(meta_select)}")
113
114    # search complete - check results - and return
115    meta_select.index = Index(meta_select[mc.id])
116    meta_select = meta_select[~meta_select.index.duplicated(keep="first")]
117
118    if verbose:
119        print(f"Final selection is {len(meta_select)} rows.")
120
121    elif len(meta_select) == 0:
122        print("Nothing selected?")
123
124    if validate_unique and len(meta_select) != 1:
125        raise ValueError("The selected meta data should only contain one row.")
126
127    return meta_select
128
129
130def find_abs_id(
131    meta: DataFrame,
132    search_terms: dict[str, str],
133    **kwargs: Unpack[SearchArgs],
134) -> tuple[str, str, str]:  # table, series_id, units
135    """Find a unique ABS series identifier in the ABS metadata.
136
137    Parameters
138    ----------
139    meta : DataFrame
140        A pandas DataFrame of metadata from the ABS
141        (via read_abs_cat() or read_abs_series()).
142    search_terms : dict[str, str]
143        A dictionary {search_phrase: meta_column_name, ...} of search terms.
144        Note: the search terms must be unique, as a dictionary cannot hold the
145        same search term to be applied to different columns.
146    **kwargs : Unpack[SearchArgs]
147        Keyword arguments passed on to search_abs_meta(): exact_match,
148        regex, validate_unique and verbose. Unknown keyword arguments
149        are reported, and not passed on.
150    validate_unique : bool = True
151        Raise a ValueError if the search result is not a single
152        unique match. Note: the default is True for safety.
153
154    Returns
155    -------
156    tuple[str, str, str]
157        A tuple of the table, series_id and units for the unique
158        series_id that matches the search terms.
159
160    Metacol
161    -------
162    Because the meta data is a DataFrame, the columns can be referenced by either
163    their full textual name, or by the short name defined in the metacol object.
164    For example, if metacol is imported as mc, to refer to the
165    `Data Item Description` column, the user can refer to it as mc.did.
166
167    Example
168    -------
169    ```python
170    from readabs import metacol as mc  # alias for the ABS meta data column names
171    from readabs import read_abs_cat, find_abs_id, recalibrate
172    cat_num = "6202.0"  # The ABS labour force survey
173    data, meta = read_abs_cat(cat_num)
174    search_terms = {
175        "Employed total ;  Persons ;": mc.did,
176        "Seasonally Adjusted": mc.stype,
177        "62020001": mc.table,
178    }
179    table, series_id, units = find_abs_id(meta, search_terms)
180    print(f"Table: {table} Series ID: {series_id} Units: {units}")
181    recal_series, recal_units = recalibrate(data[table][series_id], units)
182    ```
183
184    """
185    check_kwargs(kwargs, "find_abs_id", _SEARCH_KWARGS)  # warn if invalid kwargs
186
187    # pass on only the known kwargs, so an unknown one is reported once
188    found = search_abs_meta(
189        meta,
190        search_terms,
191        exact_match=kwargs.get("exact_match", False),
192        regex=kwargs.get("regex", False),
193        validate_unique=kwargs.get("validate_unique", True),
194        verbose=kwargs.get("verbose", False),
195    ).iloc[0]
196    table, series_id, units = (
197        found[mc.table],
198        found[mc.id],
199        found[mc.unit],
200    )
201
202    return table, series_id, units
203
204
205if __name__ == "__main__":
206
207    def test_search_abs_meta() -> None:
208        """Test the search_abs_meta() function."""
209        cat_num = "6202.0"  # The ABS labour force survey
210        _data, meta = read_abs_cat(cat_num)
211        search_terms = {
212            "Unemployment rate": mc.did,  # the data item description
213            "Persons": mc.did,
214            "Seasonally Adjusted": mc.stype,
215            "Percent": mc.unit,
216            "62020001": mc.table,
217        }
218        rows = search_abs_meta(meta, search_terms, verbose=True)
219        print(rows)  # should have three rows : FT/PT/All Unemplooyment rates
220
221    test_search_abs_meta()
222
223    def test_find_abs_id() -> None:
224        """Test the find_abs_id() function."""
225        cat_num = "6202.0"  # The ABS labour force survey
226        _data, meta = read_abs_cat(cat_num)
227        search_terms = {
228            "Employed total ;  Persons ;": mc.did,
229            "Seasonally Adjusted": mc.stype,
230            "62020001": mc.table,
231        }
232        table, series_id, units = find_abs_id(meta, search_terms)
233        print(f"Table: {table} Series ID: {series_id} Units: {units}")
234
235    test_find_abs_id()