databased.databased

  1import logging
  2import os
  3import sqlite3
  4from datetime import datetime
  5from functools import wraps
  6from typing import Any
  7
  8import pandas
  9from pathier import Pathier
 10from tabulate import tabulate
 11
 12
 13def _connect(func):
 14    """Decorator to open db connection if it isn't already open."""
 15
 16    @wraps(func)
 17    def inner(self, *args, **kwargs):
 18        if not self.connection_open:
 19            self.open()
 20        results = func(self, *args, **kwargs)
 21        return results
 22
 23    return inner
 24
 25
 26def _disconnect(func):
 27    """Decorator to commit and close db connection.
 28
 29    Primarily intended for when `DataBased` is subclassed and the inhereting class
 30    has functions that call parent class functions that are decorated with `_connect`.
 31    Decorating the child class function with `_disconnect` avoids having to manually close
 32    the connection or use a context manager in an application."""
 33
 34    @wraps(func)
 35    def inner(self, *args, **kwargs):
 36        result = func(self, *args, **kwargs)
 37        if self.connection_open:
 38            self.close()
 39        return result
 40
 41    return inner
 42
 43
 44class DataBased:
 45    """Sqli wrapper so queries don't need to be written except table definitions.
 46
 47    Supports saving and reading dates as datetime objects.
 48
 49    Supports using a context manager."""
 50
 51    def __init__(
 52        self,
 53        dbpath: str | Pathier,
 54        logger_encoding: str = "utf-8",
 55        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 56        connection_timeout: float = 10,
 57    ):
 58        """
 59        #### :params:
 60
 61        * `dbpath`: String or Path object to database file.
 62        If a relative path is given, it will be relative to the
 63        current working directory. The log file will be saved to the
 64        same directory.
 65
 66        * `logger_message_format`: `{` style format string for the logger object.
 67
 68        * `connection_timeout`: The number of seconds to wait when trying to connect to the database before throwing an error."""
 69        self.dbpath = Pathier(dbpath)
 70        self.dbname = Pathier(dbpath).name
 71        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 72        self._logger_init(
 73            encoding=logger_encoding, message_format=logger_message_format
 74        )
 75        self.connection_open = False
 76        self.connection_timeout = connection_timeout
 77
 78    def __enter__(self):
 79        self.open()
 80        return self
 81
 82    def __exit__(self, exception_type, exception_value, exception_traceback):
 83        self.close()
 84
 85    @property
 86    def connection_timeout(self) -> float:
 87        return self._connection_timeout
 88
 89    @connection_timeout.setter
 90    def connection_timeout(self, num_seconds: float):
 91        self._connection_timeout = num_seconds
 92        if self.connection_open:
 93            self.close()
 94            self.open()
 95
 96    def open(self):
 97        """Open connection to db."""
 98        self.connection = sqlite3.connect(
 99            self.dbpath,
100            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
101            timeout=self.connection_timeout,
102        )
103        self.connection.execute("pragma foreign_keys = 1;")
104        self.cursor = self.connection.cursor()
105        self.connection_open = True
106
107    def close(self):
108        """Save and close connection to db.
109
110        Call this as soon as you are done using the database if you have
111        multiple threads or processes using the same database."""
112        if self.connection_open:
113            self.connection.commit()
114            self.connection.close()
115            self.connection_open = False
116
117    def _logger_init(
118        self,
119        message_format: str = "{levelname}|-|{asctime}|-|{message}",
120        encoding: str = "utf-8",
121    ):
122        """:param `message_format`: '{' style format string"""
123        self.logger = logging.getLogger(self.dbname)
124        if not self.logger.hasHandlers():
125            handler = logging.FileHandler(
126                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
127            )
128            handler.setFormatter(
129                logging.Formatter(
130                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
131                )
132            )
133            self.logger.addHandler(handler)
134            self.logger.setLevel(logging.INFO)
135
136    def _get_dict(
137        self, table: str, values: list, columns_to_return: list[str] | None = None
138    ) -> dict:
139        """Converts the values of a row into a dictionary with column names as keys.
140
141        #### :params:
142
143        `table`: The table that values were pulled from.
144
145        `values`: List of values expected to be the same quantity
146        and in the same order as the column names of table.
147
148        `columns_to_return`: An optional list of column names.
149        If given, only these columns will be included in the returned dictionary.
150        Otherwise all columns and values are returned."""
151        return {
152            column: value
153            for column, value in zip(self.get_column_names(table), values)
154            if not columns_to_return or column in columns_to_return
155        }
156
157    def _get_conditions(
158        self, match_criteria: list[tuple] | dict, exact_match: bool = True
159    ) -> str:
160        """Builds and returns the conditional portion of a query.
161
162        #### :params:
163
164        `match_criteria`: Can be a list of 2-tuples where each
165        tuple is `(columnName, rowValue)` or a dictionary where
166        keys are column names and values are row values.
167
168        `exact_match`: If `False`, the row value for a given column
169        will be matched as a substring.
170
171        Usage e.g.:
172
173        >>> self.cursor.execute(f'select * from {table} where {conditions};')"""
174        if type(match_criteria) == dict:
175            match_criteria = [(k, v) for k, v in match_criteria.items()]
176        if exact_match:
177            conditions = " and ".join(
178                f'"{column_row[0]}" = "{column_row[1]}"'
179                for column_row in match_criteria
180            )
181        else:
182            conditions = " and ".join(
183                f'"{column_row[0]}" like "%{column_row[1]}%"'
184                for column_row in match_criteria
185            )
186        return f"({conditions})"
187
188    def vacuum(self):
189        """Reduce disk size of the database with a `VACUUM` query."""
190        self.query("VACUUM;")
191
192    @_connect
193    def query(self, query_) -> list[Any]:
194        """Execute an arbitrary query and return the results."""
195        self.cursor.execute(query_)
196        return self.cursor.fetchall()
197
198    @_connect
199    def create_tables(self, table_defs: list[str] = []):
200        """Create tables if they don't exist.
201
202        :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`"""
203        if len(table_defs) > 0:
204            table_names = self.get_table_names()
205            for table in table_defs:
206                if table.split("(")[0].strip() not in table_names:
207                    self.cursor.execute(f"create table [{table}];")
208                    self.logger.info(f'{table.split("(")[0]} table created.')
209
210    @_connect
211    def create_table(self, table: str, column_defs: list[str]):
212        """Create a table if it doesn't exist.
213
214        #### :params:
215
216        `table`: Name of the table to create.
217
218        `column_defs`: List of column definitions in proper Sqlite3 sytax.
219        i.e. `"column_name text unique"` or `"column_name int primary key"` etc."""
220        if table not in self.get_table_names():
221            query = f"create table [{table}]({', '.join(column_defs)});"
222            self.cursor.execute(query)
223            self.logger.info(f"'{table}' table created.")
224
225    @_connect
226    def get_table_names(self) -> list[str]:
227        """Returns a list of table names from the database."""
228        self.cursor.execute(
229            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";'
230        )
231        return [result[0] for result in self.cursor.fetchall()]
232
233    @_connect
234    def get_column_names(self, table: str) -> list[str]:
235        """Return a list of column names from a table."""
236        self.cursor.execute(f"select * from [{table}] where 1=0;")
237        return [description[0] for description in self.cursor.description]
238
239    @_connect
240    def count(
241        self,
242        table: str,
243        match_criteria: list[tuple] | dict | None = None,
244        exact_match: bool = True,
245    ) -> int:
246        """Return number of items in `table`.
247
248        #### :params:
249
250        `match_criteria`: Can be a list of 2-tuples where each
251        tuple is `(columnName, rowValue)` or a dictionary where
252        keys are column names and values are row values.
253        If `None`, all rows from the table will be counted.
254
255        `exact_match`: If `False`, the row value for a given column
256        in `match_criteria` will be matched as a substring.
257        Has no effect if `match_criteria` is `None`.
258        """
259        query = f"select count(_rowid_) from [{table}]"
260        try:
261            if match_criteria:
262                self.cursor.execute(
263                    f"{query} where {self._get_conditions(match_criteria, exact_match)};"
264                )
265            else:
266                self.cursor.execute(f"{query}")
267            return self.cursor.fetchone()[0]
268        except:
269            return 0
270
271    @_connect
272    def add_row(
273        self, table: str, values: tuple[Any], columns: tuple[str] | None = None
274    ) -> bool:
275        """Add a row of values to a table.
276
277        Returns whether the addition was successful or not.
278
279        #### :params:
280
281        `table`: The table to insert values into.
282
283        `values`: A tuple of values to be inserted into the table.
284
285        `columns`: If `None`, `values` is expected to supply a value for every column in the table.
286        If `columns` is provided, it should contain the same number of elements as `values`."""
287        parameterizer = ", ".join("?" for _ in values)
288        logger_values = ", ".join(str(value) for value in values)
289        try:
290            if columns:
291                columns_query = ", ".join(column for column in columns)
292                self.cursor.execute(
293                    f"insert into [{table}] ({columns_query}) values({parameterizer});",
294                    values,
295                )
296            else:
297                self.cursor.execute(
298                    f"insert into [{table}] values({parameterizer});", values
299                )
300            self.logger.info(f'Added "{logger_values}" to {table} table.')
301            return True
302        except Exception as e:
303            if "constraint" not in str(e).lower():
304                self.logger.exception(
305                    f'Error adding "{logger_values}" to {table} table.'
306                )
307            else:
308                self.logger.debug(str(e))
309            return False
310
311    @_connect
312    def add_rows(
313        self, table: str, values: list[tuple[Any]], columns: tuple[str] | None = None
314    ) -> tuple[int, int]:
315        """Add multiple rows of values to a table.
316
317        Returns a tuple containing the number of successful additions and the number of failed additions.
318
319        #### :params:
320
321        `table`: The table to insert values into.
322
323        `values`: A list of tuples of values to be inserted into the table.
324        Each tuple constitutes a single row to be inserted
325
326        `columns`: If `None`, `values` is expected to supply a value for every column in the table.
327        If `columns` is provided, it should contain the same number of elements as `values`."""
328        successes = 0
329        failures = 0
330        for row in values:
331            if self.add_row(table, row, columns):
332                successes += 1
333            else:
334                failures += 1
335        return (successes, failures)
336
337    @_connect
338    def get_rows(
339        self,
340        table: str,
341        match_criteria: list[tuple] | dict | None = None,
342        exact_match: bool = True,
343        sort_by_column: str | None = None,
344        columns_to_return: list[str] | None = None,
345        return_as_dataframe: bool = False,
346        values_only: bool = False,
347        order_by: str | None = None,
348        limit: str | int | None = None,
349    ) -> list[dict] | list[tuple] | pandas.DataFrame:
350        """Return matching rows from `table`.
351
352        By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]`
353
354
355        #### :params:
356
357        `match_criteria`: Can be a list of 2-tuples where each
358        tuple is `(columnName, rowValue)` or a dictionary where
359        keys are column names and values are row values.
360
361        `exact_match`: If `False`, the row value for a given column will be matched as a substring.
362
363        `sort_by_column`: A column name to sort the results by.
364        This will sort results in Python after retrieving them from the db.
365        Use the 'order_by' param to use SQLite engine for ordering.
366
367        `columns_to_return`: Optional list of column names.
368        If provided, the elements returned by this function will only contain the provided columns.
369        Otherwise every column in the row is returned.
370
371        `return_as_dataframe`: Return the results as a `pandas.DataFrame` object.
372
373        `values_only`: Return the results as a list of tuples.
374
375        `order_by`: If given, a `order by {order_by}` clause will be added to the select query.
376
377        `limit`: If given, a `limit {limit}` clause will be added to the select query.
378        """
379
380        if type(columns_to_return) is str:
381            columns_to_return = [columns_to_return]
382        query = f"select * from [{table}]"
383        matches = []
384        if match_criteria:
385            query += f" where {self._get_conditions(match_criteria, exact_match)}"
386        if order_by:
387            query += f" order by {order_by}"
388        if limit:
389            query += f" limit {limit}"
390        query += ";"
391        self.cursor.execute(query)
392        matches = self.cursor.fetchall()
393        results = [self._get_dict(table, match, columns_to_return) for match in matches]
394        if sort_by_column:
395            results = sorted(results, key=lambda x: x[sort_by_column])
396        if return_as_dataframe:
397            return pandas.DataFrame(results)
398        if values_only:
399            return [tuple(row.values()) for row in results]
400        else:
401            return results
402
403    @_connect
404    def find(
405        self, table: str, query_string: str, columns: list[str] | None = None
406    ) -> list[dict]:
407        """Search for rows that contain `query_string` as a substring of any column.
408
409        #### :params:
410
411        `table`: The table to search.
412
413        `query_string`: The substring to search for in all columns.
414
415        `columns`: A list of columns to search for query_string.
416        If None, all columns in the table will be searched.
417        """
418        if type(columns) is str:
419            columns = [columns]
420        results = []
421        if not columns:
422            columns = self.get_column_names(table)
423        for column in columns:
424            results.extend(
425                [
426                    row
427                    for row in self.get_rows(
428                        table, [(column, query_string)], exact_match=False
429                    )
430                    if row not in results
431                ]
432            )
433        return results
434
435    @_connect
436    def delete(
437        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
438    ) -> int:
439        """Delete records from `table`.
440
441        Returns the number of deleted records.
442
443        #### :params:
444
445        `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)`
446        or a dictionary where keys are column names and values are corresponding values.
447
448        `exact_match`: If `False`, the value for a given column will be matched as a substring.
449        """
450        conditions = self._get_conditions(match_criteria, exact_match)
451        try:
452            self.cursor.execute(f"delete from [{table}] where {conditions};")
453            num_deletions = self.cursor.rowcount
454            self.logger.info(
455                f'Deleted {num_deletions} rows from "{table}" where {conditions}".'
456            )
457            return num_deletions
458        except Exception as e:
459            self.logger.debug(
460                f'Error deleting rows from "{table}" where {conditions}.\n{e}'
461            )
462            return 0
463
464    @_connect
465    def update(
466        self,
467        table: str,
468        column_to_update: str,
469        new_value: Any,
470        match_criteria: list[tuple] | dict | None = None,
471        exact_match: bool = True,
472    ) -> int:
473        """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`.
474
475        #### :params:
476
477        `table`: The table to update rows in.
478
479        `column_to_update`: The column to be updated in the matched rows.
480
481        `new_value`: The new value to insert.
482
483        `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)`
484        or a dictionary where keys are column names and values are corresponding values.
485        If `None`, every row in `table` will be updated.
486
487        `exact_match`: If `False`, `match_criteria` values will be treated as substrings.
488
489        Returns the number of updated rows."""
490        query = f"update [{table}] set {column_to_update} = ?"
491        conditions = ""
492        if match_criteria:
493            conditions = self._get_conditions(match_criteria, exact_match)
494            query += f" where {conditions}"
495        else:
496            conditions = None
497        query += ";"
498        try:
499            self.cursor.execute(
500                query,
501                (new_value,),
502            )
503            num_updates = self.cursor.rowcount
504            self.logger.info(
505                f'In {num_updates} rows, updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
506            )
507            return num_updates
508        except Exception as e:
509            self.logger.error(
510                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
511            )
512            return 0
513
514    @_connect
515    def drop_table(self, table: str) -> bool:
516        """Drop `table` from the database.
517
518        Returns `True` if successful, `False` if not."""
519        try:
520            self.cursor.execute(f"drop Table [{table}];")
521            self.logger.info(f'Dropped table "{table}"')
522            return True
523        except Exception as e:
524            print(e)
525            self.logger.error(f'Failed to drop table "{table}"')
526            return False
527
528    @_connect
529    def add_column(
530        self, table: str, column: str, _type: str, default_value: str | None = None
531    ):
532        """Add a new column to `table`.
533
534        #### :params:
535
536        `column`: Name of the column to add.
537
538        `_type`: The data type of the new column.
539
540        `default_value`: Optional default value for the column."""
541        try:
542            if default_value:
543                self.cursor.execute(
544                    f"alter table [{table}] add column {column} {_type} default {default_value};"
545                )
546                self.update(table, column, default_value)
547            else:
548                self.cursor.execute(
549                    f"alter table [{table}] add column {column} {_type};"
550                )
551            self.logger.info(f'Added column "{column}" to "{table}" table.')
552        except Exception as e:
553            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
554
555    @staticmethod
556    def data_to_string(
557        data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
558    ) -> str:
559        """Uses tabulate to produce pretty string output from a list of dictionaries.
560
561        #### :params:
562
563        `data`: The list of dictionaries to create a grid from.
564        Assumes all dictionaries in list have the same set of keys.
565
566        `sort_key`: Optional dictionary key to sort data with.
567
568        `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window.
569        Pass as `False` if the output is going into something like a `.txt` file."""
570        return data_to_string(data, sort_key, wrap_to_terminal)
571
572
573def data_to_string(
574    data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
575) -> str:
576    """Uses tabulate to produce pretty string output from a list of dictionaries.
577
578    #### :params:
579
580    `data`: The list of dictionaries to create a grid from.
581    Assumes all dictionaries in list have the same set of keys.
582
583    `sort_key`: Optional dictionary key to sort data with.
584
585    `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window.
586    Pass as `False` if the output is going into something like a `.txt` file."""
587    if len(data) == 0:
588        return ""
589    if sort_key:
590        data = sorted(data, key=lambda d: d[sort_key])
591    for i, d in enumerate(data):
592        for k in d:
593            data[i][k] = str(data[i][k])
594
595    too_wide = True
596    terminal_width = os.get_terminal_size().columns
597    max_col_widths = terminal_width
598    # Make an output with effectively unrestricted column widths
599    # to see if shrinking is necessary
600    output = tabulate(
601        data,
602        headers="keys",
603        disable_numparse=True,
604        tablefmt="grid",
605        maxcolwidths=max_col_widths,
606    )
607    current_width = output.index("\n")
608    if current_width < terminal_width:
609        too_wide = False
610    if wrap_to_terminal and too_wide:
611        print("Resizing grid to fit within the terminal...\n")
612        previous_col_widths = max_col_widths
613        acceptable_width = terminal_width - 10
614        while too_wide and max_col_widths > 1:
615            if current_width >= terminal_width:
616                previous_col_widths = max_col_widths
617                max_col_widths = int(max_col_widths * 0.5)
618            elif current_width < terminal_width:
619                # Without lowering acceptable_width, this condition will cause infinite loop
620                if max_col_widths == previous_col_widths - 1:
621                    acceptable_width -= 10
622                max_col_widths = int(
623                    max_col_widths + (0.5 * (previous_col_widths - max_col_widths))
624                )
625            output = tabulate(
626                data,
627                headers="keys",
628                disable_numparse=True,
629                tablefmt="grid",
630                maxcolwidths=max_col_widths,
631            )
632            current_width = output.index("\n")
633            if acceptable_width < current_width < terminal_width:
634                too_wide = False
635        if too_wide:
636            print("Couldn't resize grid to fit within the terminal.")
637            return str(data)
638    return output
class DataBased:
 45class DataBased:
 46    """Sqli wrapper so queries don't need to be written except table definitions.
 47
 48    Supports saving and reading dates as datetime objects.
 49
 50    Supports using a context manager."""
 51
 52    def __init__(
 53        self,
 54        dbpath: str | Pathier,
 55        logger_encoding: str = "utf-8",
 56        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 57        connection_timeout: float = 10,
 58    ):
 59        """
 60        #### :params:
 61
 62        * `dbpath`: String or Path object to database file.
 63        If a relative path is given, it will be relative to the
 64        current working directory. The log file will be saved to the
 65        same directory.
 66
 67        * `logger_message_format`: `{` style format string for the logger object.
 68
 69        * `connection_timeout`: The number of seconds to wait when trying to connect to the database before throwing an error."""
 70        self.dbpath = Pathier(dbpath)
 71        self.dbname = Pathier(dbpath).name
 72        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 73        self._logger_init(
 74            encoding=logger_encoding, message_format=logger_message_format
 75        )
 76        self.connection_open = False
 77        self.connection_timeout = connection_timeout
 78
 79    def __enter__(self):
 80        self.open()
 81        return self
 82
 83    def __exit__(self, exception_type, exception_value, exception_traceback):
 84        self.close()
 85
 86    @property
 87    def connection_timeout(self) -> float:
 88        return self._connection_timeout
 89
 90    @connection_timeout.setter
 91    def connection_timeout(self, num_seconds: float):
 92        self._connection_timeout = num_seconds
 93        if self.connection_open:
 94            self.close()
 95            self.open()
 96
 97    def open(self):
 98        """Open connection to db."""
 99        self.connection = sqlite3.connect(
100            self.dbpath,
101            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
102            timeout=self.connection_timeout,
103        )
104        self.connection.execute("pragma foreign_keys = 1;")
105        self.cursor = self.connection.cursor()
106        self.connection_open = True
107
108    def close(self):
109        """Save and close connection to db.
110
111        Call this as soon as you are done using the database if you have
112        multiple threads or processes using the same database."""
113        if self.connection_open:
114            self.connection.commit()
115            self.connection.close()
116            self.connection_open = False
117
118    def _logger_init(
119        self,
120        message_format: str = "{levelname}|-|{asctime}|-|{message}",
121        encoding: str = "utf-8",
122    ):
123        """:param `message_format`: '{' style format string"""
124        self.logger = logging.getLogger(self.dbname)
125        if not self.logger.hasHandlers():
126            handler = logging.FileHandler(
127                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
128            )
129            handler.setFormatter(
130                logging.Formatter(
131                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
132                )
133            )
134            self.logger.addHandler(handler)
135            self.logger.setLevel(logging.INFO)
136
137    def _get_dict(
138        self, table: str, values: list, columns_to_return: list[str] | None = None
139    ) -> dict:
140        """Converts the values of a row into a dictionary with column names as keys.
141
142        #### :params:
143
144        `table`: The table that values were pulled from.
145
146        `values`: List of values expected to be the same quantity
147        and in the same order as the column names of table.
148
149        `columns_to_return`: An optional list of column names.
150        If given, only these columns will be included in the returned dictionary.
151        Otherwise all columns and values are returned."""
152        return {
153            column: value
154            for column, value in zip(self.get_column_names(table), values)
155            if not columns_to_return or column in columns_to_return
156        }
157
158    def _get_conditions(
159        self, match_criteria: list[tuple] | dict, exact_match: bool = True
160    ) -> str:
161        """Builds and returns the conditional portion of a query.
162
163        #### :params:
164
165        `match_criteria`: Can be a list of 2-tuples where each
166        tuple is `(columnName, rowValue)` or a dictionary where
167        keys are column names and values are row values.
168
169        `exact_match`: If `False`, the row value for a given column
170        will be matched as a substring.
171
172        Usage e.g.:
173
174        >>> self.cursor.execute(f'select * from {table} where {conditions};')"""
175        if type(match_criteria) == dict:
176            match_criteria = [(k, v) for k, v in match_criteria.items()]
177        if exact_match:
178            conditions = " and ".join(
179                f'"{column_row[0]}" = "{column_row[1]}"'
180                for column_row in match_criteria
181            )
182        else:
183            conditions = " and ".join(
184                f'"{column_row[0]}" like "%{column_row[1]}%"'
185                for column_row in match_criteria
186            )
187        return f"({conditions})"
188
189    def vacuum(self):
190        """Reduce disk size of the database with a `VACUUM` query."""
191        self.query("VACUUM;")
192
193    @_connect
194    def query(self, query_) -> list[Any]:
195        """Execute an arbitrary query and return the results."""
196        self.cursor.execute(query_)
197        return self.cursor.fetchall()
198
199    @_connect
200    def create_tables(self, table_defs: list[str] = []):
201        """Create tables if they don't exist.
202
203        :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`"""
204        if len(table_defs) > 0:
205            table_names = self.get_table_names()
206            for table in table_defs:
207                if table.split("(")[0].strip() not in table_names:
208                    self.cursor.execute(f"create table [{table}];")
209                    self.logger.info(f'{table.split("(")[0]} table created.')
210
211    @_connect
212    def create_table(self, table: str, column_defs: list[str]):
213        """Create a table if it doesn't exist.
214
215        #### :params:
216
217        `table`: Name of the table to create.
218
219        `column_defs`: List of column definitions in proper Sqlite3 sytax.
220        i.e. `"column_name text unique"` or `"column_name int primary key"` etc."""
221        if table not in self.get_table_names():
222            query = f"create table [{table}]({', '.join(column_defs)});"
223            self.cursor.execute(query)
224            self.logger.info(f"'{table}' table created.")
225
226    @_connect
227    def get_table_names(self) -> list[str]:
228        """Returns a list of table names from the database."""
229        self.cursor.execute(
230            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";'
231        )
232        return [result[0] for result in self.cursor.fetchall()]
233
234    @_connect
235    def get_column_names(self, table: str) -> list[str]:
236        """Return a list of column names from a table."""
237        self.cursor.execute(f"select * from [{table}] where 1=0;")
238        return [description[0] for description in self.cursor.description]
239
240    @_connect
241    def count(
242        self,
243        table: str,
244        match_criteria: list[tuple] | dict | None = None,
245        exact_match: bool = True,
246    ) -> int:
247        """Return number of items in `table`.
248
249        #### :params:
250
251        `match_criteria`: Can be a list of 2-tuples where each
252        tuple is `(columnName, rowValue)` or a dictionary where
253        keys are column names and values are row values.
254        If `None`, all rows from the table will be counted.
255
256        `exact_match`: If `False`, the row value for a given column
257        in `match_criteria` will be matched as a substring.
258        Has no effect if `match_criteria` is `None`.
259        """
260        query = f"select count(_rowid_) from [{table}]"
261        try:
262            if match_criteria:
263                self.cursor.execute(
264                    f"{query} where {self._get_conditions(match_criteria, exact_match)};"
265                )
266            else:
267                self.cursor.execute(f"{query}")
268            return self.cursor.fetchone()[0]
269        except:
270            return 0
271
272    @_connect
273    def add_row(
274        self, table: str, values: tuple[Any], columns: tuple[str] | None = None
275    ) -> bool:
276        """Add a row of values to a table.
277
278        Returns whether the addition was successful or not.
279
280        #### :params:
281
282        `table`: The table to insert values into.
283
284        `values`: A tuple of values to be inserted into the table.
285
286        `columns`: If `None`, `values` is expected to supply a value for every column in the table.
287        If `columns` is provided, it should contain the same number of elements as `values`."""
288        parameterizer = ", ".join("?" for _ in values)
289        logger_values = ", ".join(str(value) for value in values)
290        try:
291            if columns:
292                columns_query = ", ".join(column for column in columns)
293                self.cursor.execute(
294                    f"insert into [{table}] ({columns_query}) values({parameterizer});",
295                    values,
296                )
297            else:
298                self.cursor.execute(
299                    f"insert into [{table}] values({parameterizer});", values
300                )
301            self.logger.info(f'Added "{logger_values}" to {table} table.')
302            return True
303        except Exception as e:
304            if "constraint" not in str(e).lower():
305                self.logger.exception(
306                    f'Error adding "{logger_values}" to {table} table.'
307                )
308            else:
309                self.logger.debug(str(e))
310            return False
311
312    @_connect
313    def add_rows(
314        self, table: str, values: list[tuple[Any]], columns: tuple[str] | None = None
315    ) -> tuple[int, int]:
316        """Add multiple rows of values to a table.
317
318        Returns a tuple containing the number of successful additions and the number of failed additions.
319
320        #### :params:
321
322        `table`: The table to insert values into.
323
324        `values`: A list of tuples of values to be inserted into the table.
325        Each tuple constitutes a single row to be inserted
326
327        `columns`: If `None`, `values` is expected to supply a value for every column in the table.
328        If `columns` is provided, it should contain the same number of elements as `values`."""
329        successes = 0
330        failures = 0
331        for row in values:
332            if self.add_row(table, row, columns):
333                successes += 1
334            else:
335                failures += 1
336        return (successes, failures)
337
338    @_connect
339    def get_rows(
340        self,
341        table: str,
342        match_criteria: list[tuple] | dict | None = None,
343        exact_match: bool = True,
344        sort_by_column: str | None = None,
345        columns_to_return: list[str] | None = None,
346        return_as_dataframe: bool = False,
347        values_only: bool = False,
348        order_by: str | None = None,
349        limit: str | int | None = None,
350    ) -> list[dict] | list[tuple] | pandas.DataFrame:
351        """Return matching rows from `table`.
352
353        By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]`
354
355
356        #### :params:
357
358        `match_criteria`: Can be a list of 2-tuples where each
359        tuple is `(columnName, rowValue)` or a dictionary where
360        keys are column names and values are row values.
361
362        `exact_match`: If `False`, the row value for a given column will be matched as a substring.
363
364        `sort_by_column`: A column name to sort the results by.
365        This will sort results in Python after retrieving them from the db.
366        Use the 'order_by' param to use SQLite engine for ordering.
367
368        `columns_to_return`: Optional list of column names.
369        If provided, the elements returned by this function will only contain the provided columns.
370        Otherwise every column in the row is returned.
371
372        `return_as_dataframe`: Return the results as a `pandas.DataFrame` object.
373
374        `values_only`: Return the results as a list of tuples.
375
376        `order_by`: If given, a `order by {order_by}` clause will be added to the select query.
377
378        `limit`: If given, a `limit {limit}` clause will be added to the select query.
379        """
380
381        if type(columns_to_return) is str:
382            columns_to_return = [columns_to_return]
383        query = f"select * from [{table}]"
384        matches = []
385        if match_criteria:
386            query += f" where {self._get_conditions(match_criteria, exact_match)}"
387        if order_by:
388            query += f" order by {order_by}"
389        if limit:
390            query += f" limit {limit}"
391        query += ";"
392        self.cursor.execute(query)
393        matches = self.cursor.fetchall()
394        results = [self._get_dict(table, match, columns_to_return) for match in matches]
395        if sort_by_column:
396            results = sorted(results, key=lambda x: x[sort_by_column])
397        if return_as_dataframe:
398            return pandas.DataFrame(results)
399        if values_only:
400            return [tuple(row.values()) for row in results]
401        else:
402            return results
403
404    @_connect
405    def find(
406        self, table: str, query_string: str, columns: list[str] | None = None
407    ) -> list[dict]:
408        """Search for rows that contain `query_string` as a substring of any column.
409
410        #### :params:
411
412        `table`: The table to search.
413
414        `query_string`: The substring to search for in all columns.
415
416        `columns`: A list of columns to search for query_string.
417        If None, all columns in the table will be searched.
418        """
419        if type(columns) is str:
420            columns = [columns]
421        results = []
422        if not columns:
423            columns = self.get_column_names(table)
424        for column in columns:
425            results.extend(
426                [
427                    row
428                    for row in self.get_rows(
429                        table, [(column, query_string)], exact_match=False
430                    )
431                    if row not in results
432                ]
433            )
434        return results
435
436    @_connect
437    def delete(
438        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
439    ) -> int:
440        """Delete records from `table`.
441
442        Returns the number of deleted records.
443
444        #### :params:
445
446        `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)`
447        or a dictionary where keys are column names and values are corresponding values.
448
449        `exact_match`: If `False`, the value for a given column will be matched as a substring.
450        """
451        conditions = self._get_conditions(match_criteria, exact_match)
452        try:
453            self.cursor.execute(f"delete from [{table}] where {conditions};")
454            num_deletions = self.cursor.rowcount
455            self.logger.info(
456                f'Deleted {num_deletions} rows from "{table}" where {conditions}".'
457            )
458            return num_deletions
459        except Exception as e:
460            self.logger.debug(
461                f'Error deleting rows from "{table}" where {conditions}.\n{e}'
462            )
463            return 0
464
465    @_connect
466    def update(
467        self,
468        table: str,
469        column_to_update: str,
470        new_value: Any,
471        match_criteria: list[tuple] | dict | None = None,
472        exact_match: bool = True,
473    ) -> int:
474        """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`.
475
476        #### :params:
477
478        `table`: The table to update rows in.
479
480        `column_to_update`: The column to be updated in the matched rows.
481
482        `new_value`: The new value to insert.
483
484        `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)`
485        or a dictionary where keys are column names and values are corresponding values.
486        If `None`, every row in `table` will be updated.
487
488        `exact_match`: If `False`, `match_criteria` values will be treated as substrings.
489
490        Returns the number of updated rows."""
491        query = f"update [{table}] set {column_to_update} = ?"
492        conditions = ""
493        if match_criteria:
494            conditions = self._get_conditions(match_criteria, exact_match)
495            query += f" where {conditions}"
496        else:
497            conditions = None
498        query += ";"
499        try:
500            self.cursor.execute(
501                query,
502                (new_value,),
503            )
504            num_updates = self.cursor.rowcount
505            self.logger.info(
506                f'In {num_updates} rows, updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
507            )
508            return num_updates
509        except Exception as e:
510            self.logger.error(
511                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
512            )
513            return 0
514
515    @_connect
516    def drop_table(self, table: str) -> bool:
517        """Drop `table` from the database.
518
519        Returns `True` if successful, `False` if not."""
520        try:
521            self.cursor.execute(f"drop Table [{table}];")
522            self.logger.info(f'Dropped table "{table}"')
523            return True
524        except Exception as e:
525            print(e)
526            self.logger.error(f'Failed to drop table "{table}"')
527            return False
528
529    @_connect
530    def add_column(
531        self, table: str, column: str, _type: str, default_value: str | None = None
532    ):
533        """Add a new column to `table`.
534
535        #### :params:
536
537        `column`: Name of the column to add.
538
539        `_type`: The data type of the new column.
540
541        `default_value`: Optional default value for the column."""
542        try:
543            if default_value:
544                self.cursor.execute(
545                    f"alter table [{table}] add column {column} {_type} default {default_value};"
546                )
547                self.update(table, column, default_value)
548            else:
549                self.cursor.execute(
550                    f"alter table [{table}] add column {column} {_type};"
551                )
552            self.logger.info(f'Added column "{column}" to "{table}" table.')
553        except Exception as e:
554            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
555
556    @staticmethod
557    def data_to_string(
558        data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
559    ) -> str:
560        """Uses tabulate to produce pretty string output from a list of dictionaries.
561
562        #### :params:
563
564        `data`: The list of dictionaries to create a grid from.
565        Assumes all dictionaries in list have the same set of keys.
566
567        `sort_key`: Optional dictionary key to sort data with.
568
569        `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window.
570        Pass as `False` if the output is going into something like a `.txt` file."""
571        return data_to_string(data, sort_key, wrap_to_terminal)

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathier.pathier.Pathier, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}', connection_timeout: float = 10)
52    def __init__(
53        self,
54        dbpath: str | Pathier,
55        logger_encoding: str = "utf-8",
56        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
57        connection_timeout: float = 10,
58    ):
59        """
60        #### :params:
61
62        * `dbpath`: String or Path object to database file.
63        If a relative path is given, it will be relative to the
64        current working directory. The log file will be saved to the
65        same directory.
66
67        * `logger_message_format`: `{` style format string for the logger object.
68
69        * `connection_timeout`: The number of seconds to wait when trying to connect to the database before throwing an error."""
70        self.dbpath = Pathier(dbpath)
71        self.dbname = Pathier(dbpath).name
72        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
73        self._logger_init(
74            encoding=logger_encoding, message_format=logger_message_format
75        )
76        self.connection_open = False
77        self.connection_timeout = connection_timeout

:params:

  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: { style format string for the logger object.

  • connection_timeout: The number of seconds to wait when trying to connect to the database before throwing an error.

def open(self):
 97    def open(self):
 98        """Open connection to db."""
 99        self.connection = sqlite3.connect(
100            self.dbpath,
101            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
102            timeout=self.connection_timeout,
103        )
104        self.connection.execute("pragma foreign_keys = 1;")
105        self.cursor = self.connection.cursor()
106        self.connection_open = True

Open connection to db.

def close(self):
108    def close(self):
109        """Save and close connection to db.
110
111        Call this as soon as you are done using the database if you have
112        multiple threads or processes using the same database."""
113        if self.connection_open:
114            self.connection.commit()
115            self.connection.close()
116            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def vacuum(self):
189    def vacuum(self):
190        """Reduce disk size of the database with a `VACUUM` query."""
191        self.query("VACUUM;")

Reduce disk size of the database with a VACUUM query.

def query(self, query_) -> list[typing.Any]:
193    @_connect
194    def query(self, query_) -> list[Any]:
195        """Execute an arbitrary query and return the results."""
196        self.cursor.execute(query_)
197        return self.cursor.fetchall()

Execute an arbitrary query and return the results.

def create_tables(self, table_defs: list[str] = []):
199    @_connect
200    def create_tables(self, table_defs: list[str] = []):
201        """Create tables if they don't exist.
202
203        :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`"""
204        if len(table_defs) > 0:
205            table_names = self.get_table_names()
206            for table in table_defs:
207                if table.split("(")[0].strip() not in table_names:
208                    self.cursor.execute(f"create table [{table}];")
209                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_defs: Each definition should be in the form table_name(column_definitions)
def create_table(self, table: str, column_defs: list[str]):
211    @_connect
212    def create_table(self, table: str, column_defs: list[str]):
213        """Create a table if it doesn't exist.
214
215        #### :params:
216
217        `table`: Name of the table to create.
218
219        `column_defs`: List of column definitions in proper Sqlite3 sytax.
220        i.e. `"column_name text unique"` or `"column_name int primary key"` etc."""
221        if table not in self.get_table_names():
222            query = f"create table [{table}]({', '.join(column_defs)});"
223            self.cursor.execute(query)
224            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

:params:

table: Name of the table to create.

column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "column_name text unique" or "column_name int primary key" etc.

def get_table_names(self) -> list[str]:
226    @_connect
227    def get_table_names(self) -> list[str]:
228        """Returns a list of table names from the database."""
229        self.cursor.execute(
230            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";'
231        )
232        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from the database.

def get_column_names(self, table: str) -> list[str]:
234    @_connect
235    def get_column_names(self, table: str) -> list[str]:
236        """Return a list of column names from a table."""
237        self.cursor.execute(f"select * from [{table}] where 1=0;")
238        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict | None = None, exact_match: bool = True) -> int:
240    @_connect
241    def count(
242        self,
243        table: str,
244        match_criteria: list[tuple] | dict | None = None,
245        exact_match: bool = True,
246    ) -> int:
247        """Return number of items in `table`.
248
249        #### :params:
250
251        `match_criteria`: Can be a list of 2-tuples where each
252        tuple is `(columnName, rowValue)` or a dictionary where
253        keys are column names and values are row values.
254        If `None`, all rows from the table will be counted.
255
256        `exact_match`: If `False`, the row value for a given column
257        in `match_criteria` will be matched as a substring.
258        Has no effect if `match_criteria` is `None`.
259        """
260        query = f"select count(_rowid_) from [{table}]"
261        try:
262            if match_criteria:
263                self.cursor.execute(
264                    f"{query} where {self._get_conditions(match_criteria, exact_match)};"
265                )
266            else:
267                self.cursor.execute(f"{query}")
268            return self.cursor.fetchone()[0]
269        except:
270            return 0

Return number of items in table.

:params:

match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

exact_match: If False, the row value for a given column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row( self, table: str, values: tuple[typing.Any], columns: tuple[str] | None = None) -> bool:
272    @_connect
273    def add_row(
274        self, table: str, values: tuple[Any], columns: tuple[str] | None = None
275    ) -> bool:
276        """Add a row of values to a table.
277
278        Returns whether the addition was successful or not.
279
280        #### :params:
281
282        `table`: The table to insert values into.
283
284        `values`: A tuple of values to be inserted into the table.
285
286        `columns`: If `None`, `values` is expected to supply a value for every column in the table.
287        If `columns` is provided, it should contain the same number of elements as `values`."""
288        parameterizer = ", ".join("?" for _ in values)
289        logger_values = ", ".join(str(value) for value in values)
290        try:
291            if columns:
292                columns_query = ", ".join(column for column in columns)
293                self.cursor.execute(
294                    f"insert into [{table}] ({columns_query}) values({parameterizer});",
295                    values,
296                )
297            else:
298                self.cursor.execute(
299                    f"insert into [{table}] values({parameterizer});", values
300                )
301            self.logger.info(f'Added "{logger_values}" to {table} table.')
302            return True
303        except Exception as e:
304            if "constraint" not in str(e).lower():
305                self.logger.exception(
306                    f'Error adding "{logger_values}" to {table} table.'
307                )
308            else:
309                self.logger.debug(str(e))
310            return False

Add a row of values to a table.

Returns whether the addition was successful or not.

:params:

table: The table to insert values into.

values: A tuple of values to be inserted into the table.

columns: If None, values is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def add_rows( self, table: str, values: list[tuple[typing.Any]], columns: tuple[str] | None = None) -> tuple[int, int]:
312    @_connect
313    def add_rows(
314        self, table: str, values: list[tuple[Any]], columns: tuple[str] | None = None
315    ) -> tuple[int, int]:
316        """Add multiple rows of values to a table.
317
318        Returns a tuple containing the number of successful additions and the number of failed additions.
319
320        #### :params:
321
322        `table`: The table to insert values into.
323
324        `values`: A list of tuples of values to be inserted into the table.
325        Each tuple constitutes a single row to be inserted
326
327        `columns`: If `None`, `values` is expected to supply a value for every column in the table.
328        If `columns` is provided, it should contain the same number of elements as `values`."""
329        successes = 0
330        failures = 0
331        for row in values:
332            if self.add_row(table, row, columns):
333                successes += 1
334            else:
335                failures += 1
336        return (successes, failures)

Add multiple rows of values to a table.

Returns a tuple containing the number of successful additions and the number of failed additions.

:params:

table: The table to insert values into.

values: A list of tuples of values to be inserted into the table. Each tuple constitutes a single row to be inserted

columns: If None, values is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict | None = None, exact_match: bool = True, sort_by_column: str | None = None, columns_to_return: list[str] | None = None, return_as_dataframe: bool = False, values_only: bool = False, order_by: str | None = None, limit: str | int | None = None) -> list[dict] | list[tuple] | pandas.core.frame.DataFrame:
338    @_connect
339    def get_rows(
340        self,
341        table: str,
342        match_criteria: list[tuple] | dict | None = None,
343        exact_match: bool = True,
344        sort_by_column: str | None = None,
345        columns_to_return: list[str] | None = None,
346        return_as_dataframe: bool = False,
347        values_only: bool = False,
348        order_by: str | None = None,
349        limit: str | int | None = None,
350    ) -> list[dict] | list[tuple] | pandas.DataFrame:
351        """Return matching rows from `table`.
352
353        By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]`
354
355
356        #### :params:
357
358        `match_criteria`: Can be a list of 2-tuples where each
359        tuple is `(columnName, rowValue)` or a dictionary where
360        keys are column names and values are row values.
361
362        `exact_match`: If `False`, the row value for a given column will be matched as a substring.
363
364        `sort_by_column`: A column name to sort the results by.
365        This will sort results in Python after retrieving them from the db.
366        Use the 'order_by' param to use SQLite engine for ordering.
367
368        `columns_to_return`: Optional list of column names.
369        If provided, the elements returned by this function will only contain the provided columns.
370        Otherwise every column in the row is returned.
371
372        `return_as_dataframe`: Return the results as a `pandas.DataFrame` object.
373
374        `values_only`: Return the results as a list of tuples.
375
376        `order_by`: If given, a `order by {order_by}` clause will be added to the select query.
377
378        `limit`: If given, a `limit {limit}` clause will be added to the select query.
379        """
380
381        if type(columns_to_return) is str:
382            columns_to_return = [columns_to_return]
383        query = f"select * from [{table}]"
384        matches = []
385        if match_criteria:
386            query += f" where {self._get_conditions(match_criteria, exact_match)}"
387        if order_by:
388            query += f" order by {order_by}"
389        if limit:
390            query += f" limit {limit}"
391        query += ";"
392        self.cursor.execute(query)
393        matches = self.cursor.fetchall()
394        results = [self._get_dict(table, match, columns_to_return) for match in matches]
395        if sort_by_column:
396            results = sorted(results, key=lambda x: x[sort_by_column])
397        if return_as_dataframe:
398            return pandas.DataFrame(results)
399        if values_only:
400            return [tuple(row.values()) for row in results]
401        else:
402            return results

Return matching rows from table.

By default, rows will be returned as a list of dictionaries of the form [{"column_name": value, ...}, ...]

:params:

match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

exact_match: If False, the row value for a given column will be matched as a substring.

sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.

columns_to_return: Optional list of column names. If provided, the elements returned by this function will only contain the provided columns. Otherwise every column in the row is returned.

return_as_dataframe: Return the results as a pandas.DataFrame object.

values_only: Return the results as a list of tuples.

order_by: If given, a order by {order_by} clause will be added to the select query.

limit: If given, a limit {limit} clause will be added to the select query.

def find( self, table: str, query_string: str, columns: list[str] | None = None) -> list[dict]:
404    @_connect
405    def find(
406        self, table: str, query_string: str, columns: list[str] | None = None
407    ) -> list[dict]:
408        """Search for rows that contain `query_string` as a substring of any column.
409
410        #### :params:
411
412        `table`: The table to search.
413
414        `query_string`: The substring to search for in all columns.
415
416        `columns`: A list of columns to search for query_string.
417        If None, all columns in the table will be searched.
418        """
419        if type(columns) is str:
420            columns = [columns]
421        results = []
422        if not columns:
423            columns = self.get_column_names(table)
424        for column in columns:
425            results.extend(
426                [
427                    row
428                    for row in self.get_rows(
429                        table, [(column, query_string)], exact_match=False
430                    )
431                    if row not in results
432                ]
433            )
434        return results

Search for rows that contain query_string as a substring of any column.

:params:

table: The table to search.

query_string: The substring to search for in all columns.

columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
436    @_connect
437    def delete(
438        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
439    ) -> int:
440        """Delete records from `table`.
441
442        Returns the number of deleted records.
443
444        #### :params:
445
446        `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)`
447        or a dictionary where keys are column names and values are corresponding values.
448
449        `exact_match`: If `False`, the value for a given column will be matched as a substring.
450        """
451        conditions = self._get_conditions(match_criteria, exact_match)
452        try:
453            self.cursor.execute(f"delete from [{table}] where {conditions};")
454            num_deletions = self.cursor.rowcount
455            self.logger.info(
456                f'Deleted {num_deletions} rows from "{table}" where {conditions}".'
457            )
458            return num_deletions
459        except Exception as e:
460            self.logger.debug(
461                f'Error deleting rows from "{table}" where {conditions}.\n{e}'
462            )
463            return 0

Delete records from table.

Returns the number of deleted records.

:params:

match_criteria: Can be a list of 2-tuples where each tuple is (column_name, value) or a dictionary where keys are column names and values are corresponding values.

exact_match: If False, the value for a given column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict | None = None, exact_match: bool = True) -> int:
465    @_connect
466    def update(
467        self,
468        table: str,
469        column_to_update: str,
470        new_value: Any,
471        match_criteria: list[tuple] | dict | None = None,
472        exact_match: bool = True,
473    ) -> int:
474        """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`.
475
476        #### :params:
477
478        `table`: The table to update rows in.
479
480        `column_to_update`: The column to be updated in the matched rows.
481
482        `new_value`: The new value to insert.
483
484        `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)`
485        or a dictionary where keys are column names and values are corresponding values.
486        If `None`, every row in `table` will be updated.
487
488        `exact_match`: If `False`, `match_criteria` values will be treated as substrings.
489
490        Returns the number of updated rows."""
491        query = f"update [{table}] set {column_to_update} = ?"
492        conditions = ""
493        if match_criteria:
494            conditions = self._get_conditions(match_criteria, exact_match)
495            query += f" where {conditions}"
496        else:
497            conditions = None
498        query += ";"
499        try:
500            self.cursor.execute(
501                query,
502                (new_value,),
503            )
504            num_updates = self.cursor.rowcount
505            self.logger.info(
506                f'In {num_updates} rows, updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
507            )
508            return num_updates
509        except Exception as e:
510            self.logger.error(
511                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
512            )
513            return 0

Update the value in column_to_update to new_value for rows matched with match_criteria.

:params:

table: The table to update rows in.

column_to_update: The column to be updated in the matched rows.

new_value: The new value to insert.

match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are corresponding values. If None, every row in table will be updated.

exact_match: If False, match_criteria values will be treated as substrings.

Returns the number of updated rows.

def drop_table(self, table: str) -> bool:
515    @_connect
516    def drop_table(self, table: str) -> bool:
517        """Drop `table` from the database.
518
519        Returns `True` if successful, `False` if not."""
520        try:
521            self.cursor.execute(f"drop Table [{table}];")
522            self.logger.info(f'Dropped table "{table}"')
523            return True
524        except Exception as e:
525            print(e)
526            self.logger.error(f'Failed to drop table "{table}"')
527            return False

Drop table from the database.

Returns True if successful, False if not.

def add_column( self, table: str, column: str, _type: str, default_value: str | None = None):
529    @_connect
530    def add_column(
531        self, table: str, column: str, _type: str, default_value: str | None = None
532    ):
533        """Add a new column to `table`.
534
535        #### :params:
536
537        `column`: Name of the column to add.
538
539        `_type`: The data type of the new column.
540
541        `default_value`: Optional default value for the column."""
542        try:
543            if default_value:
544                self.cursor.execute(
545                    f"alter table [{table}] add column {column} {_type} default {default_value};"
546                )
547                self.update(table, column, default_value)
548            else:
549                self.cursor.execute(
550                    f"alter table [{table}] add column {column} {_type};"
551                )
552            self.logger.info(f'Added column "{column}" to "{table}" table.')
553        except Exception as e:
554            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

:params:

column: Name of the column to add.

_type: The data type of the new column.

default_value: Optional default value for the column.

@staticmethod
def data_to_string( data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True) -> str:
556    @staticmethod
557    def data_to_string(
558        data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
559    ) -> str:
560        """Uses tabulate to produce pretty string output from a list of dictionaries.
561
562        #### :params:
563
564        `data`: The list of dictionaries to create a grid from.
565        Assumes all dictionaries in list have the same set of keys.
566
567        `sort_key`: Optional dictionary key to sort data with.
568
569        `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window.
570        Pass as `False` if the output is going into something like a `.txt` file."""
571        return data_to_string(data, sort_key, wrap_to_terminal)

Uses tabulate to produce pretty string output from a list of dictionaries.

:params:

data: The list of dictionaries to create a grid from. Assumes all dictionaries in list have the same set of keys.

sort_key: Optional dictionary key to sort data with.

wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Pass as False if the output is going into something like a .txt file.

def data_to_string( data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True) -> str:
574def data_to_string(
575    data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True
576) -> str:
577    """Uses tabulate to produce pretty string output from a list of dictionaries.
578
579    #### :params:
580
581    `data`: The list of dictionaries to create a grid from.
582    Assumes all dictionaries in list have the same set of keys.
583
584    `sort_key`: Optional dictionary key to sort data with.
585
586    `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window.
587    Pass as `False` if the output is going into something like a `.txt` file."""
588    if len(data) == 0:
589        return ""
590    if sort_key:
591        data = sorted(data, key=lambda d: d[sort_key])
592    for i, d in enumerate(data):
593        for k in d:
594            data[i][k] = str(data[i][k])
595
596    too_wide = True
597    terminal_width = os.get_terminal_size().columns
598    max_col_widths = terminal_width
599    # Make an output with effectively unrestricted column widths
600    # to see if shrinking is necessary
601    output = tabulate(
602        data,
603        headers="keys",
604        disable_numparse=True,
605        tablefmt="grid",
606        maxcolwidths=max_col_widths,
607    )
608    current_width = output.index("\n")
609    if current_width < terminal_width:
610        too_wide = False
611    if wrap_to_terminal and too_wide:
612        print("Resizing grid to fit within the terminal...\n")
613        previous_col_widths = max_col_widths
614        acceptable_width = terminal_width - 10
615        while too_wide and max_col_widths > 1:
616            if current_width >= terminal_width:
617                previous_col_widths = max_col_widths
618                max_col_widths = int(max_col_widths * 0.5)
619            elif current_width < terminal_width:
620                # Without lowering acceptable_width, this condition will cause infinite loop
621                if max_col_widths == previous_col_widths - 1:
622                    acceptable_width -= 10
623                max_col_widths = int(
624                    max_col_widths + (0.5 * (previous_col_widths - max_col_widths))
625                )
626            output = tabulate(
627                data,
628                headers="keys",
629                disable_numparse=True,
630                tablefmt="grid",
631                maxcolwidths=max_col_widths,
632            )
633            current_width = output.index("\n")
634            if acceptable_width < current_width < terminal_width:
635                too_wide = False
636        if too_wide:
637            print("Couldn't resize grid to fit within the terminal.")
638            return str(data)
639    return output

Uses tabulate to produce pretty string output from a list of dictionaries.

:params:

data: The list of dictionaries to create a grid from. Assumes all dictionaries in list have the same set of keys.

sort_key: Optional dictionary key to sort data with.

wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Pass as False if the output is going into something like a .txt file.