databased.databased

  1import logging
  2import os
  3import sqlite3
  4from datetime import datetime
  5from functools import wraps
  6from typing import Any
  7
  8import pandas
  9from pathier import Pathier
 10from tabulate import tabulate
 11
 12
 13def _connect(func):
 14    """Decorator to open db connection if it isn't already open."""
 15
 16    @wraps(func)
 17    def inner(self, *args, **kwargs):
 18        if not self.connection_open:
 19            self.open()
 20        results = func(self, *args, **kwargs)
 21        return results
 22
 23    return inner
 24
 25
 26class DataBased:
 27    """Sqli wrapper so queries don't need to be written except table definitions.
 28
 29    Supports saving and reading dates as datetime objects.
 30
 31    Supports using a context manager."""
 32
 33    def __init__(
 34        self,
 35        dbpath: str | Pathier,
 36        logger_encoding: str = "utf-8",
 37        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 38    ):
 39        """
 40        :param dbpath: String or Path object to database file.
 41        If a relative path is given, it will be relative to the
 42        current working directory. The log file will be saved to the
 43        same directory.
 44
 45        :param logger_message_format: '{' style format string
 46        for the logger object."""
 47        self.dbpath = Pathier(dbpath)
 48        self.dbname = Pathier(dbpath).name
 49        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 50        self._logger_init(
 51            encoding=logger_encoding, message_format=logger_message_format
 52        )
 53        self.connection_open = False
 54
 55    def __enter__(self):
 56        self.open()
 57        return self
 58
 59    def __exit__(self, exception_type, exception_value, exception_traceback):
 60        self.close()
 61
 62    def open(self):
 63        """Open connection to db."""
 64        self.connection = sqlite3.connect(
 65            self.dbpath,
 66            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 67            timeout=10,
 68        )
 69        self.connection.execute("pragma foreign_keys = 1")
 70        self.cursor = self.connection.cursor()
 71        self.connection_open = True
 72
 73    def close(self):
 74        """Save and close connection to db.
 75
 76        Call this as soon as you are done using the database if you have
 77        multiple threads or processes using the same database."""
 78        if self.connection_open:
 79            self.connection.commit()
 80            self.connection.close()
 81            self.connection_open = False
 82
 83    def _logger_init(
 84        self,
 85        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 86        encoding: str = "utf-8",
 87    ):
 88        """:param message_format: '{' style format string"""
 89        self.logger = logging.getLogger(self.dbname)
 90        if not self.logger.hasHandlers():
 91            handler = logging.FileHandler(
 92                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
 93            )
 94            handler.setFormatter(
 95                logging.Formatter(
 96                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
 97                )
 98            )
 99            self.logger.addHandler(handler)
100            self.logger.setLevel(logging.INFO)
101
102    def _get_dict(
103        self, table: str, values: list, columns_to_return: list[str] | None = None
104    ) -> dict:
105        """Converts the values of a row into a dictionary with column names as keys.
106
107        :param table: The table that values were pulled from.
108
109        :param values: List of values expected to be the same quantity
110        and in the same order as the column names of table.
111
112        :param columns_to_return: An optional list of column names.
113        If given, only these columns will be included in the returned dictionary.
114        Otherwise all columns and values are returned."""
115        return {
116            column: value
117            for column, value in zip(self.get_column_names(table), values)
118            if not columns_to_return or column in columns_to_return
119        }
120
121    def _get_conditions(
122        self, match_criteria: list[tuple] | dict, exact_match: bool = True
123    ) -> str:
124        """Builds and returns the conditional portion of a query.
125
126        :param match_criteria: Can be a list of 2-tuples where each
127        tuple is (columnName, rowValue) or a dictionary where
128        keys are column names and values are row values.
129
130        :param exact_match: If False, the rowValue for a give column
131        will be matched as a substring.
132
133        Usage e.g.:
134
135        self.cursor.execute(f'select * from {table} where {conditions}')"""
136        if type(match_criteria) == dict:
137            match_criteria = [(k, v) for k, v in match_criteria.items()]
138        if exact_match:
139            conditions = " and ".join(
140                f'"{column_row[0]}" = "{column_row[1]}"'
141                for column_row in match_criteria
142            )
143        else:
144            conditions = " and ".join(
145                f'"{column_row[0]}" like "%{column_row[1]}%"'
146                for column_row in match_criteria
147            )
148        return f"({conditions})"
149
150    @_connect
151    def query(self, query_) -> list[Any]:
152        """Execute an arbitrary query and
153        return the results."""
154        self.cursor.execute(query_)
155        return self.cursor.fetchall()
156
157    @_connect
158    def create_tables(self, table_querys: list[str] = []):
159        """Create tables if they don't exist.
160
161        :param table_querys: Each query should be
162        in the form 'tableName(columnDefinitions)'"""
163        if len(table_querys) > 0:
164            table_names = self.get_table_names()
165            for table in table_querys:
166                if table.split("(")[0].strip() not in table_names:
167                    self.cursor.execute(f"create table {table}")
168                    self.logger.info(f'{table.split("(")[0]} table created.')
169
170    @_connect
171    def create_table(self, table: str, column_defs: list[str]):
172        """Create a table if it doesn't exist.
173
174        :param table: Name of the table to create.
175
176        :param column_defs: List of column definitions in
177        proper Sqlite3 sytax.
178        i.e. "columnName text unique" or "columnName int primary key" etc."""
179        if table not in self.get_table_names():
180            query = f"create table {table}({', '.join(column_defs)})"
181            self.cursor.execute(query)
182            self.logger.info(f"'{table}' table created.")
183
184    @_connect
185    def get_table_names(self) -> list[str]:
186        """Returns a list of table names from database."""
187        self.cursor.execute(
188            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
189        )
190        return [result[0] for result in self.cursor.fetchall()]
191
192    @_connect
193    def get_column_names(self, table: str) -> list[str]:
194        """Return a list of column names from a table."""
195        self.cursor.execute(f"select * from {table} where 1=0")
196        return [description[0] for description in self.cursor.description]
197
198    @_connect
199    def count(
200        self,
201        table: str,
202        match_criteria: list[tuple] | dict | None = None,
203        exact_match: bool = True,
204    ) -> int:
205        """Return number of items in table.
206
207        :param match_criteria: Can be a list of 2-tuples where each
208        tuple is (columnName, rowValue) or a dictionary where
209        keys are column names and values are row values.
210        If None, all rows from the table will be counted.
211
212        :param exact_match: If False, the row value for a give column
213        in match_criteria will be matched as a substring. Has no effect if
214        match_criteria is None.
215        """
216        query = f"select count(_rowid_) from {table}"
217        try:
218            if match_criteria:
219                self.cursor.execute(
220                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
221                )
222            else:
223                self.cursor.execute(f"{query}")
224            return self.cursor.fetchone()[0]
225        except:
226            return 0
227
228    @_connect
229    def add_row(
230        self, table: str, values: tuple[Any], columns: tuple[str] | None = None
231    ):
232        """Add row of values to table.
233
234        :param table: The table to insert into.
235
236        :param values: A tuple of values to be inserted into the table.
237
238        :param columns: If None, values param is expected to supply
239        a value for every column in the table. If columns is
240        provided, it should contain the same number of elements as values."""
241        parameterizer = ", ".join("?" for _ in values)
242        logger_values = ", ".join(str(value) for value in values)
243        try:
244            if columns:
245                columns_query = ", ".join(column for column in columns)
246                self.cursor.execute(
247                    f"insert into {table} ({columns_query}) values({parameterizer})",
248                    values,
249                )
250            else:
251                self.cursor.execute(
252                    f"insert into {table} values({parameterizer})", values
253                )
254            self.logger.info(f'Added "{logger_values}" to {table} table.')
255        except Exception as e:
256            if "constraint" not in str(e).lower():
257                self.logger.exception(
258                    f'Error adding "{logger_values}" to {table} table.'
259                )
260            else:
261                self.logger.debug(str(e))
262
263    @_connect
264    def get_rows(
265        self,
266        table: str,
267        match_criteria: list[tuple] | dict | None = None,
268        exact_match: bool = True,
269        sort_by_column: str | None = None,
270        columns_to_return: list[str] | None = None,
271        return_as_dataframe: bool = False,
272        values_only: bool = False,
273        order_by: str | None = None,
274        limit: str | int | None = None,
275    ) -> list[dict] | list[tuple] | pandas.DataFrame:
276        """Returns rows from table as a list of dictionaries
277        where the key-value pairs of the dictionaries are
278        column name: row value.
279
280        :param match_criteria: Can be a list of 2-tuples where each
281        tuple is (columnName, rowValue) or a dictionary where
282        keys are column names and values are row values.
283
284        :param exact_match: If False, the rowValue for a give column
285        will be matched as a substring.
286
287        :param sort_by_column: A column name to sort the results by.
288        This will sort results in Python after retrieving them from the db.
289        Use the 'order_by' param to use SQLite engine for ordering.
290
291        :param columns_to_return: Optional list of column names.
292        If provided, the elements returned by get_rows() will
293        only contain the provided columns. Otherwise every column
294        in the row is returned.
295
296        :param return_as_dataframe: If True,
297        the results will be returned as a pandas.DataFrame object.
298
299        :param values_only: Return the results as a list of tuples
300        instead of a list of dictionaries that have column names as keys.
301        The results will still be sorted according to sort_by_column if
302        one is provided.
303
304        :param order_by: If given, a 'order by {order_by}' clause
305        will be added to the select query.
306
307        :param limit: If given, a 'limit {limit}' clause will be
308        added to the select query.
309        """
310
311        if type(columns_to_return) is str:
312            columns_to_return = [columns_to_return]
313        query = f"select * from {table}"
314        matches = []
315        if match_criteria:
316            query += f" where {self._get_conditions(match_criteria, exact_match)}"
317        if order_by:
318            query += f" order by {order_by}"
319        if limit:
320            query += f" limit {limit}"
321        query += ";"
322        self.cursor.execute(query)
323        matches = self.cursor.fetchall()
324        results = [self._get_dict(table, match, columns_to_return) for match in matches]
325        if sort_by_column:
326            results = sorted(results, key=lambda x: x[sort_by_column])
327        if return_as_dataframe:
328            return pandas.DataFrame(results)
329        if values_only:
330            return [tuple(row.values()) for row in results]
331        else:
332            return results
333
334    @_connect
335    def find(
336        self, table: str, query_string: str, columns: list[str] | None = None
337    ) -> list[dict]:
338        """Search for rows that contain query_string as a substring
339        of any column.
340
341        :param table: The table to search.
342
343        :param query_string: The substring to search for in all columns.
344
345        :param columns: A list of columns to search for query_string.
346        If None, all columns in the table will be searched.
347        """
348        if type(columns) is str:
349            columns = [columns]
350        results = []
351        if not columns:
352            columns = self.get_column_names(table)
353        for column in columns:
354            results.extend(
355                [
356                    row
357                    for row in self.get_rows(
358                        table, [(column, query_string)], exact_match=False
359                    )
360                    if row not in results
361                ]
362            )
363        return results
364
365    @_connect
366    def delete(
367        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
368    ) -> int:
369        """Delete records from table.
370
371        Returns number of deleted records.
372
373        :param match_criteria: Can be a list of 2-tuples where each
374        tuple is (columnName, rowValue) or a dictionary where
375        keys are column names and values are row values.
376
377        :param exact_match: If False, the rowValue for a give column
378        will be matched as a substring.
379        """
380        num_matches = self.count(table, match_criteria, exact_match)
381        conditions = self._get_conditions(match_criteria, exact_match)
382        try:
383            self.cursor.execute(f"delete from {table} where {conditions}")
384            self.logger.info(
385                f'Deleted {num_matches} from "{table}" where {conditions}".'
386            )
387            return num_matches
388        except Exception as e:
389            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
390            return 0
391
392    @_connect
393    def update(
394        self,
395        table: str,
396        column_to_update: str,
397        new_value: Any,
398        match_criteria: list[tuple] | dict | None = None,
399    ) -> bool:
400        """Update row value for entry matched with match_criteria.
401
402        :param column_to_update: The column to be updated in the matched row.
403
404        :param new_value: The new value to insert.
405
406        :param match_criteria: Can be a list of 2-tuples where each
407        tuple is (columnName, rowValue) or a dictionary where
408        keys are column names and values are row values.
409        If None, every row will be updated.
410
411        Returns True if successful, False if not."""
412        query = f"update {table} set {column_to_update} = ?"
413        conditions = ""
414        if match_criteria:
415            if self.count(table, match_criteria) == 0:
416                self.logger.info(
417                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
418                )
419                return False
420            conditions = self._get_conditions(match_criteria)
421            query += f" where {conditions}"
422        else:
423            conditions = None
424        try:
425            self.cursor.execute(
426                query,
427                (new_value,),
428            )
429            self.logger.info(
430                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
431            )
432            return True
433        except Exception as e:
434            self.logger.error(
435                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
436            )
437            return False
438
439    @_connect
440    def drop_table(self, table: str) -> bool:
441        """Drop a table from the database.
442
443        Returns True if successful, False if not."""
444        try:
445            self.cursor.execute(f"drop Table {table}")
446            self.logger.info(f'Dropped table "{table}"')
447            return True
448        except Exception as e:
449            print(e)
450            self.logger.error(f'Failed to drop table "{table}"')
451            return False
452
453    @_connect
454    def add_column(
455        self, table: str, column: str, _type: str, default_value: str | None = None
456    ):
457        """Add a new column to table.
458
459        :param column: Name of the column to add.
460
461        :param _type: The data type of the new column.
462
463        :param default_value: Optional default value for the column."""
464        try:
465            if default_value:
466                self.cursor.execute(
467                    f"alter table {table} add column {column} {_type} default {default_value}"
468                )
469            else:
470                self.cursor.execute(f"alter table {table} add column {column} {_type}")
471            self.logger.info(f'Added column "{column}" to "{table}" table.')
472        except Exception as e:
473            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
474
475    @staticmethod
476    def data_to_string(
477        data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
478    ) -> str:
479        """Uses tabulate to produce pretty string output
480        from a list of dictionaries.
481
482        :param data: Assumes all dictionaries in list have the same set of keys.
483
484        :param sort_key: Optional dictionary key to sort data with.
485
486        :param wrap_to_terminal: If True, the table width will be wrapped
487        to fit within the current terminal window. Set to False
488        if the output is going into something like a txt file."""
489        return data_to_string(data, sort_key, wrap_to_terminal)
490
491
492def data_to_string(
493    data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
494) -> str:
495    """Use tabulate to produce grid output from a list of dictionaries.
496
497    :param data: Assumes all dictionaries in list have the same set of keys.
498
499    :param sort_key: Optional dictionary key to sort data with.
500
501    :param wrap_to_terminal: If True, the column widths will be reduced so the grid fits
502    within the current terminal window without wrapping. If the column widths have reduced to 1
503    and the grid is still too wide, str(data) will be returned."""
504    if len(data) == 0:
505        return ""
506    if sort_key:
507        data = sorted(data, key=lambda d: d[sort_key])
508    for i, d in enumerate(data):
509        for k in d:
510            data[i][k] = str(data[i][k])
511
512    too_wide = True
513    terminal_width = os.get_terminal_size().columns
514    max_col_widths = terminal_width
515    # Make an output with effectively unrestricted column widths
516    # to see if shrinking is necessary
517    output = tabulate(
518        data,
519        headers="keys",
520        disable_numparse=True,
521        tablefmt="grid",
522        maxcolwidths=max_col_widths,
523    )
524    current_width = output.index("\n")
525    if current_width < terminal_width:
526        too_wide = False
527    if wrap_to_terminal and too_wide:
528        print("Resizing grid to fit within the terminal...")
529        previous_col_widths = max_col_widths
530        acceptable_width = terminal_width - 10
531        while too_wide and max_col_widths > 1:
532            if current_width > terminal_width:
533                previous_col_widths = max_col_widths
534                max_col_widths = int(max_col_widths * 0.5)
535            elif current_width < terminal_width:
536                # Without lowering acceptable_width, this condition will cause infinite loop
537                if max_col_widths == previous_col_widths - 1:
538                    acceptable_width -= 10
539                max_col_widths = int(
540                    max_col_widths + (0.5 * (previous_col_widths - max_col_widths))
541                )
542            output = tabulate(
543                data,
544                headers="keys",
545                disable_numparse=True,
546                tablefmt="grid",
547                maxcolwidths=max_col_widths,
548            )
549            current_width = output.index("\n")
550            if acceptable_width < current_width < terminal_width:
551                too_wide = False
552        if too_wide:
553            print("Couldn't resize grid to fit within the terminal.")
554            return str(data)
555    return output
class DataBased:
 27class DataBased:
 28    """Sqli wrapper so queries don't need to be written except table definitions.
 29
 30    Supports saving and reading dates as datetime objects.
 31
 32    Supports using a context manager."""
 33
 34    def __init__(
 35        self,
 36        dbpath: str | Pathier,
 37        logger_encoding: str = "utf-8",
 38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 39    ):
 40        """
 41        :param dbpath: String or Path object to database file.
 42        If a relative path is given, it will be relative to the
 43        current working directory. The log file will be saved to the
 44        same directory.
 45
 46        :param logger_message_format: '{' style format string
 47        for the logger object."""
 48        self.dbpath = Pathier(dbpath)
 49        self.dbname = Pathier(dbpath).name
 50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 51        self._logger_init(
 52            encoding=logger_encoding, message_format=logger_message_format
 53        )
 54        self.connection_open = False
 55
 56    def __enter__(self):
 57        self.open()
 58        return self
 59
 60    def __exit__(self, exception_type, exception_value, exception_traceback):
 61        self.close()
 62
 63    def open(self):
 64        """Open connection to db."""
 65        self.connection = sqlite3.connect(
 66            self.dbpath,
 67            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 68            timeout=10,
 69        )
 70        self.connection.execute("pragma foreign_keys = 1")
 71        self.cursor = self.connection.cursor()
 72        self.connection_open = True
 73
 74    def close(self):
 75        """Save and close connection to db.
 76
 77        Call this as soon as you are done using the database if you have
 78        multiple threads or processes using the same database."""
 79        if self.connection_open:
 80            self.connection.commit()
 81            self.connection.close()
 82            self.connection_open = False
 83
 84    def _logger_init(
 85        self,
 86        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 87        encoding: str = "utf-8",
 88    ):
 89        """:param message_format: '{' style format string"""
 90        self.logger = logging.getLogger(self.dbname)
 91        if not self.logger.hasHandlers():
 92            handler = logging.FileHandler(
 93                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
 94            )
 95            handler.setFormatter(
 96                logging.Formatter(
 97                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
 98                )
 99            )
100            self.logger.addHandler(handler)
101            self.logger.setLevel(logging.INFO)
102
103    def _get_dict(
104        self, table: str, values: list, columns_to_return: list[str] | None = None
105    ) -> dict:
106        """Converts the values of a row into a dictionary with column names as keys.
107
108        :param table: The table that values were pulled from.
109
110        :param values: List of values expected to be the same quantity
111        and in the same order as the column names of table.
112
113        :param columns_to_return: An optional list of column names.
114        If given, only these columns will be included in the returned dictionary.
115        Otherwise all columns and values are returned."""
116        return {
117            column: value
118            for column, value in zip(self.get_column_names(table), values)
119            if not columns_to_return or column in columns_to_return
120        }
121
122    def _get_conditions(
123        self, match_criteria: list[tuple] | dict, exact_match: bool = True
124    ) -> str:
125        """Builds and returns the conditional portion of a query.
126
127        :param match_criteria: Can be a list of 2-tuples where each
128        tuple is (columnName, rowValue) or a dictionary where
129        keys are column names and values are row values.
130
131        :param exact_match: If False, the rowValue for a give column
132        will be matched as a substring.
133
134        Usage e.g.:
135
136        self.cursor.execute(f'select * from {table} where {conditions}')"""
137        if type(match_criteria) == dict:
138            match_criteria = [(k, v) for k, v in match_criteria.items()]
139        if exact_match:
140            conditions = " and ".join(
141                f'"{column_row[0]}" = "{column_row[1]}"'
142                for column_row in match_criteria
143            )
144        else:
145            conditions = " and ".join(
146                f'"{column_row[0]}" like "%{column_row[1]}%"'
147                for column_row in match_criteria
148            )
149        return f"({conditions})"
150
151    @_connect
152    def query(self, query_) -> list[Any]:
153        """Execute an arbitrary query and
154        return the results."""
155        self.cursor.execute(query_)
156        return self.cursor.fetchall()
157
158    @_connect
159    def create_tables(self, table_querys: list[str] = []):
160        """Create tables if they don't exist.
161
162        :param table_querys: Each query should be
163        in the form 'tableName(columnDefinitions)'"""
164        if len(table_querys) > 0:
165            table_names = self.get_table_names()
166            for table in table_querys:
167                if table.split("(")[0].strip() not in table_names:
168                    self.cursor.execute(f"create table {table}")
169                    self.logger.info(f'{table.split("(")[0]} table created.')
170
171    @_connect
172    def create_table(self, table: str, column_defs: list[str]):
173        """Create a table if it doesn't exist.
174
175        :param table: Name of the table to create.
176
177        :param column_defs: List of column definitions in
178        proper Sqlite3 sytax.
179        i.e. "columnName text unique" or "columnName int primary key" etc."""
180        if table not in self.get_table_names():
181            query = f"create table {table}({', '.join(column_defs)})"
182            self.cursor.execute(query)
183            self.logger.info(f"'{table}' table created.")
184
185    @_connect
186    def get_table_names(self) -> list[str]:
187        """Returns a list of table names from database."""
188        self.cursor.execute(
189            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
190        )
191        return [result[0] for result in self.cursor.fetchall()]
192
193    @_connect
194    def get_column_names(self, table: str) -> list[str]:
195        """Return a list of column names from a table."""
196        self.cursor.execute(f"select * from {table} where 1=0")
197        return [description[0] for description in self.cursor.description]
198
199    @_connect
200    def count(
201        self,
202        table: str,
203        match_criteria: list[tuple] | dict | None = None,
204        exact_match: bool = True,
205    ) -> int:
206        """Return number of items in table.
207
208        :param match_criteria: Can be a list of 2-tuples where each
209        tuple is (columnName, rowValue) or a dictionary where
210        keys are column names and values are row values.
211        If None, all rows from the table will be counted.
212
213        :param exact_match: If False, the row value for a give column
214        in match_criteria will be matched as a substring. Has no effect if
215        match_criteria is None.
216        """
217        query = f"select count(_rowid_) from {table}"
218        try:
219            if match_criteria:
220                self.cursor.execute(
221                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
222                )
223            else:
224                self.cursor.execute(f"{query}")
225            return self.cursor.fetchone()[0]
226        except:
227            return 0
228
229    @_connect
230    def add_row(
231        self, table: str, values: tuple[Any], columns: tuple[str] | None = None
232    ):
233        """Add row of values to table.
234
235        :param table: The table to insert into.
236
237        :param values: A tuple of values to be inserted into the table.
238
239        :param columns: If None, values param is expected to supply
240        a value for every column in the table. If columns is
241        provided, it should contain the same number of elements as values."""
242        parameterizer = ", ".join("?" for _ in values)
243        logger_values = ", ".join(str(value) for value in values)
244        try:
245            if columns:
246                columns_query = ", ".join(column for column in columns)
247                self.cursor.execute(
248                    f"insert into {table} ({columns_query}) values({parameterizer})",
249                    values,
250                )
251            else:
252                self.cursor.execute(
253                    f"insert into {table} values({parameterizer})", values
254                )
255            self.logger.info(f'Added "{logger_values}" to {table} table.')
256        except Exception as e:
257            if "constraint" not in str(e).lower():
258                self.logger.exception(
259                    f'Error adding "{logger_values}" to {table} table.'
260                )
261            else:
262                self.logger.debug(str(e))
263
264    @_connect
265    def get_rows(
266        self,
267        table: str,
268        match_criteria: list[tuple] | dict | None = None,
269        exact_match: bool = True,
270        sort_by_column: str | None = None,
271        columns_to_return: list[str] | None = None,
272        return_as_dataframe: bool = False,
273        values_only: bool = False,
274        order_by: str | None = None,
275        limit: str | int | None = None,
276    ) -> list[dict] | list[tuple] | pandas.DataFrame:
277        """Returns rows from table as a list of dictionaries
278        where the key-value pairs of the dictionaries are
279        column name: row value.
280
281        :param match_criteria: Can be a list of 2-tuples where each
282        tuple is (columnName, rowValue) or a dictionary where
283        keys are column names and values are row values.
284
285        :param exact_match: If False, the rowValue for a give column
286        will be matched as a substring.
287
288        :param sort_by_column: A column name to sort the results by.
289        This will sort results in Python after retrieving them from the db.
290        Use the 'order_by' param to use SQLite engine for ordering.
291
292        :param columns_to_return: Optional list of column names.
293        If provided, the elements returned by get_rows() will
294        only contain the provided columns. Otherwise every column
295        in the row is returned.
296
297        :param return_as_dataframe: If True,
298        the results will be returned as a pandas.DataFrame object.
299
300        :param values_only: Return the results as a list of tuples
301        instead of a list of dictionaries that have column names as keys.
302        The results will still be sorted according to sort_by_column if
303        one is provided.
304
305        :param order_by: If given, a 'order by {order_by}' clause
306        will be added to the select query.
307
308        :param limit: If given, a 'limit {limit}' clause will be
309        added to the select query.
310        """
311
312        if type(columns_to_return) is str:
313            columns_to_return = [columns_to_return]
314        query = f"select * from {table}"
315        matches = []
316        if match_criteria:
317            query += f" where {self._get_conditions(match_criteria, exact_match)}"
318        if order_by:
319            query += f" order by {order_by}"
320        if limit:
321            query += f" limit {limit}"
322        query += ";"
323        self.cursor.execute(query)
324        matches = self.cursor.fetchall()
325        results = [self._get_dict(table, match, columns_to_return) for match in matches]
326        if sort_by_column:
327            results = sorted(results, key=lambda x: x[sort_by_column])
328        if return_as_dataframe:
329            return pandas.DataFrame(results)
330        if values_only:
331            return [tuple(row.values()) for row in results]
332        else:
333            return results
334
335    @_connect
336    def find(
337        self, table: str, query_string: str, columns: list[str] | None = None
338    ) -> list[dict]:
339        """Search for rows that contain query_string as a substring
340        of any column.
341
342        :param table: The table to search.
343
344        :param query_string: The substring to search for in all columns.
345
346        :param columns: A list of columns to search for query_string.
347        If None, all columns in the table will be searched.
348        """
349        if type(columns) is str:
350            columns = [columns]
351        results = []
352        if not columns:
353            columns = self.get_column_names(table)
354        for column in columns:
355            results.extend(
356                [
357                    row
358                    for row in self.get_rows(
359                        table, [(column, query_string)], exact_match=False
360                    )
361                    if row not in results
362                ]
363            )
364        return results
365
366    @_connect
367    def delete(
368        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
369    ) -> int:
370        """Delete records from table.
371
372        Returns number of deleted records.
373
374        :param match_criteria: Can be a list of 2-tuples where each
375        tuple is (columnName, rowValue) or a dictionary where
376        keys are column names and values are row values.
377
378        :param exact_match: If False, the rowValue for a give column
379        will be matched as a substring.
380        """
381        num_matches = self.count(table, match_criteria, exact_match)
382        conditions = self._get_conditions(match_criteria, exact_match)
383        try:
384            self.cursor.execute(f"delete from {table} where {conditions}")
385            self.logger.info(
386                f'Deleted {num_matches} from "{table}" where {conditions}".'
387            )
388            return num_matches
389        except Exception as e:
390            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
391            return 0
392
393    @_connect
394    def update(
395        self,
396        table: str,
397        column_to_update: str,
398        new_value: Any,
399        match_criteria: list[tuple] | dict | None = None,
400    ) -> bool:
401        """Update row value for entry matched with match_criteria.
402
403        :param column_to_update: The column to be updated in the matched row.
404
405        :param new_value: The new value to insert.
406
407        :param match_criteria: Can be a list of 2-tuples where each
408        tuple is (columnName, rowValue) or a dictionary where
409        keys are column names and values are row values.
410        If None, every row will be updated.
411
412        Returns True if successful, False if not."""
413        query = f"update {table} set {column_to_update} = ?"
414        conditions = ""
415        if match_criteria:
416            if self.count(table, match_criteria) == 0:
417                self.logger.info(
418                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
419                )
420                return False
421            conditions = self._get_conditions(match_criteria)
422            query += f" where {conditions}"
423        else:
424            conditions = None
425        try:
426            self.cursor.execute(
427                query,
428                (new_value,),
429            )
430            self.logger.info(
431                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
432            )
433            return True
434        except Exception as e:
435            self.logger.error(
436                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
437            )
438            return False
439
440    @_connect
441    def drop_table(self, table: str) -> bool:
442        """Drop a table from the database.
443
444        Returns True if successful, False if not."""
445        try:
446            self.cursor.execute(f"drop Table {table}")
447            self.logger.info(f'Dropped table "{table}"')
448            return True
449        except Exception as e:
450            print(e)
451            self.logger.error(f'Failed to drop table "{table}"')
452            return False
453
454    @_connect
455    def add_column(
456        self, table: str, column: str, _type: str, default_value: str | None = None
457    ):
458        """Add a new column to table.
459
460        :param column: Name of the column to add.
461
462        :param _type: The data type of the new column.
463
464        :param default_value: Optional default value for the column."""
465        try:
466            if default_value:
467                self.cursor.execute(
468                    f"alter table {table} add column {column} {_type} default {default_value}"
469                )
470            else:
471                self.cursor.execute(f"alter table {table} add column {column} {_type}")
472            self.logger.info(f'Added column "{column}" to "{table}" table.')
473        except Exception as e:
474            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
475
476    @staticmethod
477    def data_to_string(
478        data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
479    ) -> str:
480        """Uses tabulate to produce pretty string output
481        from a list of dictionaries.
482
483        :param data: Assumes all dictionaries in list have the same set of keys.
484
485        :param sort_key: Optional dictionary key to sort data with.
486
487        :param wrap_to_terminal: If True, the table width will be wrapped
488        to fit within the current terminal window. Set to False
489        if the output is going into something like a txt file."""
490        return data_to_string(data, sort_key, wrap_to_terminal)

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathier.pathier.Pathier, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}')
34    def __init__(
35        self,
36        dbpath: str | Pathier,
37        logger_encoding: str = "utf-8",
38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
39    ):
40        """
41        :param dbpath: String or Path object to database file.
42        If a relative path is given, it will be relative to the
43        current working directory. The log file will be saved to the
44        same directory.
45
46        :param logger_message_format: '{' style format string
47        for the logger object."""
48        self.dbpath = Pathier(dbpath)
49        self.dbname = Pathier(dbpath).name
50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
51        self._logger_init(
52            encoding=logger_encoding, message_format=logger_message_format
53        )
54        self.connection_open = False
Parameters
  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: '{' style format string for the logger object.

def open(self):
63    def open(self):
64        """Open connection to db."""
65        self.connection = sqlite3.connect(
66            self.dbpath,
67            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
68            timeout=10,
69        )
70        self.connection.execute("pragma foreign_keys = 1")
71        self.cursor = self.connection.cursor()
72        self.connection_open = True

Open connection to db.

def close(self):
74    def close(self):
75        """Save and close connection to db.
76
77        Call this as soon as you are done using the database if you have
78        multiple threads or processes using the same database."""
79        if self.connection_open:
80            self.connection.commit()
81            self.connection.close()
82            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def query(self, query_) -> list[typing.Any]:
151    @_connect
152    def query(self, query_) -> list[Any]:
153        """Execute an arbitrary query and
154        return the results."""
155        self.cursor.execute(query_)
156        return self.cursor.fetchall()

Execute an arbitrary query and return the results.

def create_tables(self, table_querys: list[str] = []):
158    @_connect
159    def create_tables(self, table_querys: list[str] = []):
160        """Create tables if they don't exist.
161
162        :param table_querys: Each query should be
163        in the form 'tableName(columnDefinitions)'"""
164        if len(table_querys) > 0:
165            table_names = self.get_table_names()
166            for table in table_querys:
167                if table.split("(")[0].strip() not in table_names:
168                    self.cursor.execute(f"create table {table}")
169                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_querys: Each query should be in the form 'tableName(columnDefinitions)'
def create_table(self, table: str, column_defs: list[str]):
171    @_connect
172    def create_table(self, table: str, column_defs: list[str]):
173        """Create a table if it doesn't exist.
174
175        :param table: Name of the table to create.
176
177        :param column_defs: List of column definitions in
178        proper Sqlite3 sytax.
179        i.e. "columnName text unique" or "columnName int primary key" etc."""
180        if table not in self.get_table_names():
181            query = f"create table {table}({', '.join(column_defs)})"
182            self.cursor.execute(query)
183            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

Parameters
  • table: Name of the table to create.

  • column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.

def get_table_names(self) -> list[str]:
185    @_connect
186    def get_table_names(self) -> list[str]:
187        """Returns a list of table names from database."""
188        self.cursor.execute(
189            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
190        )
191        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from database.

def get_column_names(self, table: str) -> list[str]:
193    @_connect
194    def get_column_names(self, table: str) -> list[str]:
195        """Return a list of column names from a table."""
196        self.cursor.execute(f"select * from {table} where 1=0")
197        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict | None = None, exact_match: bool = True) -> int:
199    @_connect
200    def count(
201        self,
202        table: str,
203        match_criteria: list[tuple] | dict | None = None,
204        exact_match: bool = True,
205    ) -> int:
206        """Return number of items in table.
207
208        :param match_criteria: Can be a list of 2-tuples where each
209        tuple is (columnName, rowValue) or a dictionary where
210        keys are column names and values are row values.
211        If None, all rows from the table will be counted.
212
213        :param exact_match: If False, the row value for a give column
214        in match_criteria will be matched as a substring. Has no effect if
215        match_criteria is None.
216        """
217        query = f"select count(_rowid_) from {table}"
218        try:
219            if match_criteria:
220                self.cursor.execute(
221                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
222                )
223            else:
224                self.cursor.execute(f"{query}")
225            return self.cursor.fetchone()[0]
226        except:
227            return 0

Return number of items in table.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

  • exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row( self, table: str, values: tuple[typing.Any], columns: tuple[str] | None = None):
229    @_connect
230    def add_row(
231        self, table: str, values: tuple[Any], columns: tuple[str] | None = None
232    ):
233        """Add row of values to table.
234
235        :param table: The table to insert into.
236
237        :param values: A tuple of values to be inserted into the table.
238
239        :param columns: If None, values param is expected to supply
240        a value for every column in the table. If columns is
241        provided, it should contain the same number of elements as values."""
242        parameterizer = ", ".join("?" for _ in values)
243        logger_values = ", ".join(str(value) for value in values)
244        try:
245            if columns:
246                columns_query = ", ".join(column for column in columns)
247                self.cursor.execute(
248                    f"insert into {table} ({columns_query}) values({parameterizer})",
249                    values,
250                )
251            else:
252                self.cursor.execute(
253                    f"insert into {table} values({parameterizer})", values
254                )
255            self.logger.info(f'Added "{logger_values}" to {table} table.')
256        except Exception as e:
257            if "constraint" not in str(e).lower():
258                self.logger.exception(
259                    f'Error adding "{logger_values}" to {table} table.'
260                )
261            else:
262                self.logger.debug(str(e))

Add row of values to table.

Parameters
  • table: The table to insert into.

  • values: A tuple of values to be inserted into the table.

  • columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict | None = None, exact_match: bool = True, sort_by_column: str | None = None, columns_to_return: list[str] | None = None, return_as_dataframe: bool = False, values_only: bool = False, order_by: str | None = None, limit: str | int | None = None) -> list[dict] | list[tuple] | pandas.core.frame.DataFrame:
264    @_connect
265    def get_rows(
266        self,
267        table: str,
268        match_criteria: list[tuple] | dict | None = None,
269        exact_match: bool = True,
270        sort_by_column: str | None = None,
271        columns_to_return: list[str] | None = None,
272        return_as_dataframe: bool = False,
273        values_only: bool = False,
274        order_by: str | None = None,
275        limit: str | int | None = None,
276    ) -> list[dict] | list[tuple] | pandas.DataFrame:
277        """Returns rows from table as a list of dictionaries
278        where the key-value pairs of the dictionaries are
279        column name: row value.
280
281        :param match_criteria: Can be a list of 2-tuples where each
282        tuple is (columnName, rowValue) or a dictionary where
283        keys are column names and values are row values.
284
285        :param exact_match: If False, the rowValue for a give column
286        will be matched as a substring.
287
288        :param sort_by_column: A column name to sort the results by.
289        This will sort results in Python after retrieving them from the db.
290        Use the 'order_by' param to use SQLite engine for ordering.
291
292        :param columns_to_return: Optional list of column names.
293        If provided, the elements returned by get_rows() will
294        only contain the provided columns. Otherwise every column
295        in the row is returned.
296
297        :param return_as_dataframe: If True,
298        the results will be returned as a pandas.DataFrame object.
299
300        :param values_only: Return the results as a list of tuples
301        instead of a list of dictionaries that have column names as keys.
302        The results will still be sorted according to sort_by_column if
303        one is provided.
304
305        :param order_by: If given, a 'order by {order_by}' clause
306        will be added to the select query.
307
308        :param limit: If given, a 'limit {limit}' clause will be
309        added to the select query.
310        """
311
312        if type(columns_to_return) is str:
313            columns_to_return = [columns_to_return]
314        query = f"select * from {table}"
315        matches = []
316        if match_criteria:
317            query += f" where {self._get_conditions(match_criteria, exact_match)}"
318        if order_by:
319            query += f" order by {order_by}"
320        if limit:
321            query += f" limit {limit}"
322        query += ";"
323        self.cursor.execute(query)
324        matches = self.cursor.fetchall()
325        results = [self._get_dict(table, match, columns_to_return) for match in matches]
326        if sort_by_column:
327            results = sorted(results, key=lambda x: x[sort_by_column])
328        if return_as_dataframe:
329            return pandas.DataFrame(results)
330        if values_only:
331            return [tuple(row.values()) for row in results]
332        else:
333            return results

Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

  • sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.

  • columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.

  • return_as_dataframe: If True, the results will be returned as a pandas.DataFrame object.

  • values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.

  • order_by: If given, a 'order by {order_by}' clause will be added to the select query.

  • limit: If given, a 'limit {limit}' clause will be added to the select query.

def find( self, table: str, query_string: str, columns: list[str] | None = None) -> list[dict]:
335    @_connect
336    def find(
337        self, table: str, query_string: str, columns: list[str] | None = None
338    ) -> list[dict]:
339        """Search for rows that contain query_string as a substring
340        of any column.
341
342        :param table: The table to search.
343
344        :param query_string: The substring to search for in all columns.
345
346        :param columns: A list of columns to search for query_string.
347        If None, all columns in the table will be searched.
348        """
349        if type(columns) is str:
350            columns = [columns]
351        results = []
352        if not columns:
353            columns = self.get_column_names(table)
354        for column in columns:
355            results.extend(
356                [
357                    row
358                    for row in self.get_rows(
359                        table, [(column, query_string)], exact_match=False
360                    )
361                    if row not in results
362                ]
363            )
364        return results

Search for rows that contain query_string as a substring of any column.

Parameters
  • table: The table to search.

  • query_string: The substring to search for in all columns.

  • columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
366    @_connect
367    def delete(
368        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
369    ) -> int:
370        """Delete records from table.
371
372        Returns number of deleted records.
373
374        :param match_criteria: Can be a list of 2-tuples where each
375        tuple is (columnName, rowValue) or a dictionary where
376        keys are column names and values are row values.
377
378        :param exact_match: If False, the rowValue for a give column
379        will be matched as a substring.
380        """
381        num_matches = self.count(table, match_criteria, exact_match)
382        conditions = self._get_conditions(match_criteria, exact_match)
383        try:
384            self.cursor.execute(f"delete from {table} where {conditions}")
385            self.logger.info(
386                f'Deleted {num_matches} from "{table}" where {conditions}".'
387            )
388            return num_matches
389        except Exception as e:
390            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
391            return 0

Delete records from table.

Returns number of deleted records.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict | None = None) -> bool:
393    @_connect
394    def update(
395        self,
396        table: str,
397        column_to_update: str,
398        new_value: Any,
399        match_criteria: list[tuple] | dict | None = None,
400    ) -> bool:
401        """Update row value for entry matched with match_criteria.
402
403        :param column_to_update: The column to be updated in the matched row.
404
405        :param new_value: The new value to insert.
406
407        :param match_criteria: Can be a list of 2-tuples where each
408        tuple is (columnName, rowValue) or a dictionary where
409        keys are column names and values are row values.
410        If None, every row will be updated.
411
412        Returns True if successful, False if not."""
413        query = f"update {table} set {column_to_update} = ?"
414        conditions = ""
415        if match_criteria:
416            if self.count(table, match_criteria) == 0:
417                self.logger.info(
418                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
419                )
420                return False
421            conditions = self._get_conditions(match_criteria)
422            query += f" where {conditions}"
423        else:
424            conditions = None
425        try:
426            self.cursor.execute(
427                query,
428                (new_value,),
429            )
430            self.logger.info(
431                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
432            )
433            return True
434        except Exception as e:
435            self.logger.error(
436                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
437            )
438            return False

Update row value for entry matched with match_criteria.

Parameters
  • column_to_update: The column to be updated in the matched row.

  • new_value: The new value to insert.

  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.

Returns True if successful, False if not.

def drop_table(self, table: str) -> bool:
440    @_connect
441    def drop_table(self, table: str) -> bool:
442        """Drop a table from the database.
443
444        Returns True if successful, False if not."""
445        try:
446            self.cursor.execute(f"drop Table {table}")
447            self.logger.info(f'Dropped table "{table}"')
448            return True
449        except Exception as e:
450            print(e)
451            self.logger.error(f'Failed to drop table "{table}"')
452            return False

Drop a table from the database.

Returns True if successful, False if not.

def add_column( self, table: str, column: str, _type: str, default_value: str | None = None):
454    @_connect
455    def add_column(
456        self, table: str, column: str, _type: str, default_value: str | None = None
457    ):
458        """Add a new column to table.
459
460        :param column: Name of the column to add.
461
462        :param _type: The data type of the new column.
463
464        :param default_value: Optional default value for the column."""
465        try:
466            if default_value:
467                self.cursor.execute(
468                    f"alter table {table} add column {column} {_type} default {default_value}"
469                )
470            else:
471                self.cursor.execute(f"alter table {table} add column {column} {_type}")
472            self.logger.info(f'Added column "{column}" to "{table}" table.')
473        except Exception as e:
474            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

Parameters
  • column: Name of the column to add.

  • _type: The data type of the new column.

  • default_value: Optional default value for the column.

@staticmethod
def data_to_string( data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True) -> str:
476    @staticmethod
477    def data_to_string(
478        data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
479    ) -> str:
480        """Uses tabulate to produce pretty string output
481        from a list of dictionaries.
482
483        :param data: Assumes all dictionaries in list have the same set of keys.
484
485        :param sort_key: Optional dictionary key to sort data with.
486
487        :param wrap_to_terminal: If True, the table width will be wrapped
488        to fit within the current terminal window. Set to False
489        if the output is going into something like a txt file."""
490        return data_to_string(data, sort_key, wrap_to_terminal)

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.

def data_to_string( data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True) -> str:
493def data_to_string(
494    data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
495) -> str:
496    """Use tabulate to produce grid output from a list of dictionaries.
497
498    :param data: Assumes all dictionaries in list have the same set of keys.
499
500    :param sort_key: Optional dictionary key to sort data with.
501
502    :param wrap_to_terminal: If True, the column widths will be reduced so the grid fits
503    within the current terminal window without wrapping. If the column widths have reduced to 1
504    and the grid is still too wide, str(data) will be returned."""
505    if len(data) == 0:
506        return ""
507    if sort_key:
508        data = sorted(data, key=lambda d: d[sort_key])
509    for i, d in enumerate(data):
510        for k in d:
511            data[i][k] = str(data[i][k])
512
513    too_wide = True
514    terminal_width = os.get_terminal_size().columns
515    max_col_widths = terminal_width
516    # Make an output with effectively unrestricted column widths
517    # to see if shrinking is necessary
518    output = tabulate(
519        data,
520        headers="keys",
521        disable_numparse=True,
522        tablefmt="grid",
523        maxcolwidths=max_col_widths,
524    )
525    current_width = output.index("\n")
526    if current_width < terminal_width:
527        too_wide = False
528    if wrap_to_terminal and too_wide:
529        print("Resizing grid to fit within the terminal...")
530        previous_col_widths = max_col_widths
531        acceptable_width = terminal_width - 10
532        while too_wide and max_col_widths > 1:
533            if current_width > terminal_width:
534                previous_col_widths = max_col_widths
535                max_col_widths = int(max_col_widths * 0.5)
536            elif current_width < terminal_width:
537                # Without lowering acceptable_width, this condition will cause infinite loop
538                if max_col_widths == previous_col_widths - 1:
539                    acceptable_width -= 10
540                max_col_widths = int(
541                    max_col_widths + (0.5 * (previous_col_widths - max_col_widths))
542                )
543            output = tabulate(
544                data,
545                headers="keys",
546                disable_numparse=True,
547                tablefmt="grid",
548                maxcolwidths=max_col_widths,
549            )
550            current_width = output.index("\n")
551            if acceptable_width < current_width < terminal_width:
552                too_wide = False
553        if too_wide:
554            print("Couldn't resize grid to fit within the terminal.")
555            return str(data)
556    return output

Use tabulate to produce grid output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the column widths will be reduced so the grid fits within the current terminal window without wrapping. If the column widths have reduced to 1 and the grid is still too wide, str(data) will be returned.