sdmxabs

30class CacheError(Exception):
31    """A problem retrieving data from the cache."""

37class GetFileKwargs(TypedDict):
38    """TypedDict for acqure_url function arguments."""
39
40    verbose: NotRequired[bool]
41    """If True, print information about the data retrieval process."""
42    modality: NotRequired[ModalityType]
43    """Kind of retrieval: "prefer_cache", "prefer_url"."""

26class HttpError(Exception):
27    """A problem retrieving data from HTTP."""

17class MatchType(Enum):
18    """Enumeration for match types."""
19
20    EXACT = 1
21    PARTIAL = 2
22    REGEX = 3

 90@cache
 91def code_lists(cl_id: str, **kwargs: Unpack[GetFileKwargs]) -> FlowMetaDict:
 92    """Get the code list metadata from the ABS SDMX API.
 93
 94    Args:
 95        cl_id (str): The ID of the code list to retrieve.
 96        **kwargs: Additional keyword arguments passed to acquire_url().
 97
 98    Returns:
 99        FlowMetaDict: A dictionary containing the codes and
100            their associated key=value pairs. A "name" key should always
101            be present. A "parent" key may also be present.
102
103    Raises:
104        HttpError: If there is an issue with the HTTP request.
105        CacheError: If there is an issue with the cache.
106        ValueError: If no XML root is found in the response.
107
108    Note:
109        You will get a CacheError if the codelist is not found on the ABS SDMX API.
110        (This package tries the website first, then the cache.)
111    """
112    tree = acquire_xml(f"{URL_STEM}/codelist/ABS/{cl_id}", **kwargs)
113
114    codes: FlowMetaDict = {}
115    for code in tree.findall(".//str:Code", NAME_SPACES):
116        code_id = code.get("id", None)
117        if code_id is None:
118            continue
119        elements: dict[str, str] = {}
120        name = code.find("com:Name", NAME_SPACES)
121        elements["name"] = str(name.text) if name is not None else "(missing)"
122        parent = code.find("str:Parent", NAME_SPACES)
123        parent_id = ""
124        if parent is not None:
125            ref = parent.find("Ref", NAME_SPACES)
126            if ref is not None:
127                parent_id = str(ref.get("id", ""))
128            elements["parent"] = parent_id
129        codes[code_id] = elements
130
131    return codes

55@cache
56def data_dimensions(flow_id: str, **kwargs: Unpack[GetFileKwargs]) -> FlowMetaDict:
57    """Get the data dimensions metadata from the ABS SDMX API.
58
59    Args:
60        flow_id (str): The ID of the dataflow to retrieve dimensions for.
61        **kwargs: Additional keyword arguments passed to acquire_url().
62
63    Returns:
64        dict[str, dict[str, str]]: A dictionary containing the dimensions and
65            their metadata in key=value pairs.
66
67    Raises:
68        HttpError: If there is an issue with the HTTP request.
69        CacheError: If there is an issue with the cache.
70        ValueError: If no XML root is found in the response.
71
72    """
73    tree = acquire_xml(f"{URL_STEM}/datastructure/ABS/{flow_id}", **kwargs)
74
75    dimensions = {}
76    for dim in tree.findall(".//str:Dimension", NAME_SPACES):
77        dim_id = dim.get("id")
78        dim_pos = dim.get("position")
79        if dim_id is None or dim_pos is None:
80            continue
81        contents = {"position": dim_pos}
82        if (lr := dim.find("str:LocalRepresentation", NAME_SPACES)) is not None and (
83            enumer := lr.find("str:Enumeration/Ref", NAME_SPACES)
84        ) is not None:
85            contents = contents | enumer.attrib
86        dimensions[dim_id] = contents
87    return dimensions

22@cache
23def data_flows(flow_id: str = "all", **kwargs: Unpack[GetFileKwargs]) -> FlowMetaDict:
24    """Get the toplevel metadata from the ABS SDMX API.
25
26    Args:
27        flow_id (str): The ID of the dataflow to retrieve. Defaults to "all".
28        **kwargs: Additional keyword arguments passed to acquire_url().
29
30    Returns:
31        dict[str, dict[str, str]]: A dictionary containing the dataflow IDs
32            and their metadatain key=value pairs.
33
34    Raises:
35        HttpError: If there is an issue with the HTTP request.
36        CacheError: If there is an issue with the cache.
37        ValueError: If no XML root is found in the response.
38
39    """
40    tree = acquire_xml(f"{URL_STEM}/dataflow/ABS/{flow_id}", **kwargs)
41
42    d_flows: FlowMetaDict = {}
43    for dataflow in tree.findall(".//str:Dataflow", NAME_SPACES):
44        attributes: dict[str, str] = dataflow.attrib.copy()
45        if "id" not in attributes:
46            continue
47        df_id = attributes.pop("id")
48        name_elem = dataflow.find("com:Name", NAME_SPACES)
49        df_name = name_elem.text if name_elem is not None else "(no name)"
50        attributes["name"] = str(df_name)
51        d_flows[df_id] = attributes
52    return d_flows

139def fetch(
140    flow_id: str,
141    dims: dict[str, str] | None = None,
142    constraints: dict[str, str] | None = None,  # not implemented yet
143    *,
144    validate: bool = False,
145    **kwargs: Unpack[GetFileKwargs],
146) -> tuple[pd.DataFrame, pd.DataFrame]:
147    """Fetch data from the ABS SDMX API.
148
149    Args:
150        flow_id (str): The ID of the data flow from which to retrieve data items.
151        dims (dict[str, str], optional): A dictionary of dimensions to select the
152            data items. If None, the ABS fetch request will be for all data items,
153            which can be slow.
154        constraints (dict[str, str], optional): A dictionary of constraints to apply
155            to the data items. If None, no constraints are applied.
156        validate (bool): If True, print validation diagnostics for the proposed
157            dimensions against the metadata requirements. Defaults to False.
158        **kwargs (GetFileKwargs): Additional keyword arguments passed to acquire_xml().
159
160    Returns: a tuple of two DataFrames:
161        - The first DataFrame contains the fetched data.
162        - The second DataFrame contains the metadata.
163
164    Raises:
165        HttpError: If there is an issue with the HTTP request.
166        CacheError: If there is an issue with the cache.
167        ValueError: If no XML root is found in the response.
168
169    Notes:
170        If the `dims` argument is not valid you will get a CacheError or HttpError.
171        If the `flow_id` is not valid, you will get a ValueError.
172    """
173    # --- prepare to get the XML root from the ABS SDMX API
174    kwargs["modality"] = kwargs.get("modality", "prefer-cache")
175    key = build_key(
176        flow_id,
177        dims,
178        validate=validate,
179    )
180    _not_implemented = constraints
181    url = f"{URL_STEM}/data/{flow_id}/{key}"
182    xml_root = acquire_xml(url, **kwargs)
183    return extract(flow_id, xml_root)

 77def fetch_multi(
 78    wanted: pd.DataFrame,
 79    *,
 80    validate: bool = False,
 81    **kwargs: Unpack[GetFileKwargs],
 82) -> tuple[pd.DataFrame, pd.DataFrame]:
 83    """Fetch multiple SDMX datasets based on a DataFrame of desired datasets.
 84
 85    Args:
 86        wanted: A DataFrame with rows for each desired data set (of one or more series).
 87                Each row should contain the necessary identifiers to fetch the dataset.
 88                The columns will be 'flow_id', plus the ABS dimensions relevant to the flow.
 89                The 'flow_id' column is mandatory, and the rest are optional.
 90                Note: the DataFrame index is not used in the fetching process.
 91        validate: If True, the function will validate dimensions and values against
 92                  the ABS SDMX API codelists. Defaults to False.
 93        **kwargs: Additional keyword arguments passed to the underlying data fetching function.
 94
 95    Returns:
 96        A tuple containing two DataFrames:
 97        - The first DataFrame contains the fetched data.
 98        - The second DataFrame contains metadata about the fetched datasets.
 99
100    Raises:
101        ValueError: If the 'flow_id' column is missing from the `wanted` DataFrame.
102
103    Note:
104        CacheError and HttpError are raised by the fetch function.
105        These will be caught and reported to standard output.
106
107    Caution:
108        The selected data should all have the same index. You cannot mix (for example)
109        Quarterly and Monthly data in the same DataFrame.
110
111    """
112    # --- quick sanity checks
113    if wanted.empty:
114        print("wanted DataFrame is empty, returning empty DataFrames.")
115        return pd.DataFrame(), pd.DataFrame()
116    if "flow_id" not in wanted.columns:
117        raise ValueError("The 'flow_id' column is required in the 'wanted' DataFrame.")
118
119    # --- do the work
120    return extract(wanted, validate=validate, **kwargs)

144def fetch_selection(
145    flow_id: str,
146    criteria: MatchCriteria,
147    *,
148    validate: bool = False,
149    **kwargs: Unpack[GetFileKwargs],
150) -> tuple[pd.DataFrame, pd.DataFrame]:
151    """Fetch data based on a selection criteria for items.
152
153    Args:
154        flow_id (str): The ID of the data flow to fetch.
155        criteria (MatchCriteria): A sequence of match criteria to filter the data.
156        validate (bool, optional): If True, validate the selection against the flow's
157            required dimensions. Defaults to False.
158        **kwargs: Additional keyword arguments for the fetch_multi function.
159
160    Returns:
161        tuple[pd.DataFrame, pd.DataFrame]: A tuple containing the fetched data and metadata.
162
163    """
164    selection = select_items(flow_id, criteria)
165    return fetch_multi(selection, validate=validate, **kwargs)

73def match_criterion(
74    pattern: str,
75    dimension: str,
76    match_type: MatchType = MatchType.PARTIAL,
77) -> MatchItem:
78    """Create a new match criterion for use in selection.
79
80    Args:
81        pattern (str): The pattern to match.
82        dimension (str): The dimension to match against.
83        match_type (MatchType, optional): The type of match to perform. Defaults to MatchType.EXACT.
84
85    Returns:
86        MatchElement: A tuple representing the match element.
87
88    """
89    return (pattern, dimension, match_type)

 92def select_items(
 93    flow_id: str,
 94    criteria: MatchCriteria,
 95) -> pd.DataFrame:
 96    """Build the `wanted` Dataframe for use by fetch_multi() by matching flow metadata.
 97
 98    Args:
 99        flow_id (str): The ID of the data flow to select items from.
100        criteria (MatchElements): A sequence of tuples containing the element name,
101            the value to match, and the match type (exact, partial, or regex).
102
103    Returns:
104        pd.DataFrame: A DataFrame containing the selected items, which can be dropped
105            into the call of the function fetch_multi().
106
107    Raises:
108        ValueError: If the flow_id is not valid or if no items match the criteria.
109
110    Notes:
111    -   Should build a one line DataFrame. This Frame may select multiple data series,
112        when passed to fetch_multi. It also can be concatenated with other DataFrames
113        to build a larger selection.
114    -   If two match elements refer to the same dimension, only the `intersection` of the
115        matches will be returned.
116
117    """
118    # --- some sanity checks
119    if flow_id not in data_flows():
120        raise ValueError(f"Invalid flow_id: {flow_id}.")
121    dimensions = data_dimensions(flow_id)
122    if not dimensions:
123        raise ValueError(f"No dimensions found for flow_id: {flow_id}.")
124
125    # --- lets build the codelist dictionary
126    return_dict: dict[str, str] = {}
127    for pattern, dimension, match_type in criteria:
128        if dimension not in dimensions:
129            print(f"Dimension '{dimension}' not found for flow '{flow_id}'; (skipping)")
130            continue
131        dim_dict = dimensions[dimension]
132        if "package" not in dim_dict or dim_dict["package"] != "codelist" or "id" not in dim_dict:
133            print(f"Dimension '{dimension}' does not have a codelist; (skipping)")
134            continue
135        code_list_name = dim_dict.get("id")
136        codes = get_codes(code_lists(code_list_name), pattern, match_type)
137        package_codes(codes, dimension, return_dict)
138
139    # --- return as a (one row) `wanted` DataFrame
140    return_dict["flow_id"] = flow_id
141    return pd.DataFrame([return_dict]).astype(str)